diff --git a/README.md b/README.md index 8743d686..6f3bf04c 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ public static class RunAlarm implements Functional { } ``` #### Chat Completion Service with Vision -Example to call the Chat Completion service to allow the model to take in images and answer questions about them: +Example to call the Chat Completion service to allow the model to take in external images and answer questions about them: ```java var chatRequest = ChatRequest.builder() .model("gpt-4-vision-preview") @@ -270,6 +270,37 @@ chatResponse.filter(chatResp -> chatResp.firstContent() != null) .forEach(System.out::print); System.out.println(); ``` +Example to call the Chat Completion service to allow the model to take in local images and answer questions about them: +```java +var chatRequest = ChatRequest.builder() + .model("gpt-4-vision-preview") + .messages(List.of( + new ChatMsgUser(List.of( + new ContentPartText( + "What do you see in the image? Give in details in no more than 100 words."), + new ContentPartImage(loadImageAsBase64("src/demo/resources/machupicchu.jpg")))))) + .temperature(0.0) + .maxTokens(500) + .build(); +var chatResponse = openai.chatCompletions().createStream(chatRequest).join(); +chatResponse.filter(chatResp -> chatResp.firstContent() != null) + .map(chatResp -> chatResp.firstContent()) + .forEach(System.out::print); +System.out.println(); + +private static ImageUrl loadImageAsBase64(String imagePath) { + try { + Path path = Paths.get(imagePath); + byte[] imageBytes = Files.readAllBytes(path); + String base64String = Base64.getEncoder().encodeToString(imageBytes); + var extension = imagePath.substring(imagePath.lastIndexOf(".") + 1); + var prefix = "data:image/" + extension + ";base64,"; + return new ImageUrl(prefix + base64String); + } catch (Exception e) { + return null; + } +} +``` ## ✳ Run Examples Examples for each OpenAI service have been created in the folder [demo](https://github.com/sashirestela/simple-openai/tree/main/src/demo/java/io/github/sashirestela/openai/demo) and you can follow the next steps to execute them: diff --git a/pom.xml b/pom.xml index 9d79efad..de8699e6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.github.sashirestela simple-openai - 1.3.0 + 1.4.0 jar simple-openai diff --git a/src/demo/java/io/github/sashirestela/openai/demo/ChatServiceDemo.java b/src/demo/java/io/github/sashirestela/openai/demo/ChatServiceDemo.java index 5192b8bb..fd18303a 100644 --- a/src/demo/java/io/github/sashirestela/openai/demo/ChatServiceDemo.java +++ b/src/demo/java/io/github/sashirestela/openai/demo/ChatServiceDemo.java @@ -1,6 +1,10 @@ package io.github.sashirestela.openai.demo; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; @@ -94,7 +98,7 @@ public void demoCallChatWithFunctions() { System.out.println(chatResponse.firstContent()); } - public void demoCallChatWithVision() { + public void demoCallChatWithVisionExternalImage() { var chatRequest = ChatRequest.builder() .model("gpt-4-vision-preview") .messages(List.of( @@ -113,6 +117,38 @@ public void demoCallChatWithVision() { System.out.println(); } + public void demoCallChatWithVisionLocalImage() { + var chatRequest = ChatRequest.builder() + .model("gpt-4-vision-preview") + .messages(List.of( + new ChatMsgUser(List.of( + new ContentPartText( + "What do you see in the image? Give in details in no more than 100 words."), + new ContentPartImage(loadImageAsBase64("src/demo/resources/machupicchu.jpg")))))) + .temperature(0.0) + .maxTokens(500) + .build(); + var chatResponse = openAI.chatCompletions().createStream(chatRequest).join(); + chatResponse.filter(chatResp -> chatResp.firstContent() != null) + .map(chatResp -> chatResp.firstContent()) + .forEach(System.out::print); + System.out.println(); + } + + private static ImageUrl loadImageAsBase64(String imagePath) { + try { + Path path = Paths.get(imagePath); + byte[] imageBytes = Files.readAllBytes(path); + String base64String = Base64.getEncoder().encodeToString(imageBytes); + var extension = imagePath.substring(imagePath.lastIndexOf(".") + 1); + var prefix = "data:image/" + extension + ";base64,"; + return new ImageUrl(prefix + base64String); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + public static class Weather implements Functional { @JsonPropertyDescription("City and state, for example: León, Guanajuato") public String location; @@ -155,7 +191,8 @@ public static void main(String[] args) { demo.addTitleAction("Call Chat (Streaming Approach)", demo::demoCallChatStreaming); demo.addTitleAction("Call Chat (Blocking Approach)", demo::demoCallChatBlocking); demo.addTitleAction("Call Chat with Functions", demo::demoCallChatWithFunctions); - demo.addTitleAction("Call Chat with Vision", demo::demoCallChatWithVision); + demo.addTitleAction("Call Chat with Vision (External image)", demo::demoCallChatWithVisionExternalImage); + demo.addTitleAction("Call Chat with Vision (Local image)", demo::demoCallChatWithVisionLocalImage); demo.run(); } diff --git a/src/demo/resources/machupicchu.jpg b/src/demo/resources/machupicchu.jpg new file mode 100644 index 00000000..e04123bd Binary files /dev/null and b/src/demo/resources/machupicchu.jpg differ diff --git a/src/main/java/io/github/sashirestela/openai/SimpleOpenAI.java b/src/main/java/io/github/sashirestela/openai/SimpleOpenAI.java index 0ccd7e6a..928f1f74 100644 --- a/src/main/java/io/github/sashirestela/openai/SimpleOpenAI.java +++ b/src/main/java/io/github/sashirestela/openai/SimpleOpenAI.java @@ -90,7 +90,7 @@ public SimpleOpenAI( this.apiKey = apiKey; this.organizationId = organizationId; this.baseUrl = Optional.ofNullable(baseUrl) - .orElse(Optional.ofNullable(urlBase).orElse(OPENAI_BASE_URL)); + .orElse(Optional.ofNullable(urlBase).orElse(OPENAI_BASE_URL)); this.httpClient = Optional.ofNullable(httpClient).orElse(HttpClient.newHttpClient()); @@ -102,11 +102,11 @@ public SimpleOpenAI( headers.add(organizationId); } this.cleverClient = CleverClient.builder() - .httpClient(this.httpClient) - .baseUrl(this.baseUrl) - .headers(headers) - .endOfStream(END_OF_STREAM) - .build(); + .httpClient(this.httpClient) + .baseUrl(this.baseUrl) + .headers(headers) + .endOfStream(END_OF_STREAM) + .build(); } public void setCleverClient(CleverClient cleverClient) { diff --git a/src/test/java/io/github/sashirestela/openai/SimpleOpenAITest.java b/src/test/java/io/github/sashirestela/openai/SimpleOpenAITest.java index c966d8dc..12ef68db 100644 --- a/src/test/java/io/github/sashirestela/openai/SimpleOpenAITest.java +++ b/src/test/java/io/github/sashirestela/openai/SimpleOpenAITest.java @@ -62,8 +62,8 @@ void shouldSetPropertiesWhenBuilderIsCalledWithThoseProperties() { void shouldSetBaseUrlWhenBuilderIsCalledWithBaseUrlOnly() { var someUrl = "https://exmaple.org/api"; var openAI = SimpleOpenAI.builder() - .baseUrl(someUrl) - .build(); + .baseUrl(someUrl) + .build(); assertEquals(someUrl, openAI.getBaseUrl()); } @@ -71,8 +71,8 @@ void shouldSetBaseUrlWhenBuilderIsCalledWithBaseUrlOnly() { void shouldSetBaseUrlWhenBuilderIsCalledWithUrlBaseOnly() { var someUrl = "https://exmaple.org/api"; var openAI = SimpleOpenAI.builder() - .urlBase(someUrl) - .build(); + .urlBase(someUrl) + .build(); assertEquals(someUrl, openAI.getBaseUrl()); } @@ -81,16 +81,16 @@ void shouldSetBaseUrlWhenBuilderIsCalledWithBothBaseUrlAndUrlBase() { var someUrl = "https://exmaple.org/api"; var otherUrl = "https://exmaple.org/other-api"; var openAI = SimpleOpenAI.builder() - .baseUrl(someUrl) - .urlBase(otherUrl) - .build(); + .baseUrl(someUrl) + .urlBase(otherUrl) + .build(); assertEquals(someUrl, openAI.getBaseUrl()); } @Test void shouldSetDefaultBaseUrlWhenBuilderIsCalledWithoutBaseUrlOrUrlBase() { var openAI = SimpleOpenAI.builder() - .build(); + .build(); assertEquals(OPENAI_BASE_URL, openAI.getBaseUrl()); } @@ -163,9 +163,9 @@ void init() { @Test void shouldInstanceAudioServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Audios.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Audios.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.audios()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -173,9 +173,9 @@ void shouldInstanceAudioServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceChatCompletionServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.ChatCompletions.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.ChatCompletions.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.chatCompletions()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -183,9 +183,9 @@ void shouldInstanceChatCompletionServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceCompletionServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Completions.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Completions.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.completions()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -193,9 +193,9 @@ void shouldInstanceCompletionServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceEmbeddingServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Embeddings.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Embeddings.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.embeddings()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -203,9 +203,9 @@ void shouldInstanceEmbeddingServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceFilesServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Files.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Files.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.files()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -213,9 +213,9 @@ void shouldInstanceFilesServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceFineTunningServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.FineTunings.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.FineTunings.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.fineTunings()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -223,9 +223,9 @@ void shouldInstanceFineTunningServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceImageServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Images.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Images.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.images()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -233,9 +233,9 @@ void shouldInstanceImageServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceModelsServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Models.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Models.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.models()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); } @@ -243,9 +243,9 @@ void shouldInstanceModelsServiceOnlyOnceWhenItIsCalledSeveralTimes() { @Test void shouldInstanceModerationServiceOnlyOnceWhenItIsCalledSeveralTimes() { when(cleverClient.create(any())) - .thenReturn(ReflectUtil.createProxy( - OpenAI.Moderations.class, - new HttpProcessor(null, null, null))); + .thenReturn(ReflectUtil.createProxy( + OpenAI.Moderations.class, + new HttpProcessor(null, null, null))); repeat(NUMBER_CALLINGS, () -> openAI.moderations()); verify(cleverClient, times(NUMBER_INVOCATIONS)).create(any()); }