project to use utf-8 encoding to support all languages

Author: jetkai

build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 group = "io.github.jetkai"
-version = "1.1.1"
+version = "1.1.2"
 
 java {
     //withSourcesJar()
@@ -26,11 +26,26 @@ dependencies {
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.9.2")
 }
 
+/**
+ * Character Encoding
+ * --- START ---
+ */
+tasks.withType<JavaExec> {
+    jvmArgs = listOf("-Dfile.encoding=UTF-8", "-Dconsole.encoding=UTF-8")
+}
+
+tasks.compileJava {
+    options.encoding = "UTF-8"
+}
+/**
+ * Character Encoding
+ * --- END ---
+ */
+
 /**
  * Set Environmental Variables for the OPEN_AI_API_KEY
  * Called by System.getenv("OPEN_AI_API_KEY"); in source-code
  */
-
 tasks.register("setEnvironmentVariable") {
     doFirst {
         val env = System.getenv().toMutableMap()
@@ -45,11 +60,8 @@ tasks.register("setEnvironmentVariable") {
 
 tasks.withType<Jar> {
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-
     archiveFileName.set("openai-binary.jar")
-
     from(sourceSets.main.get().output)
-
     dependsOn(configurations.runtimeClasspath)
     from({
         configurations.runtimeClasspath.get().filter { it.name.endsWith("jar") }.map { zipTree(it) }
@@ -64,7 +76,6 @@ tasks.getByName<Test>("test") {
  * Jacoco Testing Plugin
  * --- START ---
  */
-
 tasks.test {
     finalizedBy(tasks.jacocoTestReport)
 }
@@ -75,7 +86,6 @@ tasks.jacocoTestReport {
     }
     dependsOn(tasks.test)
 }
-
 /**
  * Jacoco Testing Plugin
  * --- END ---

src/main/java/examples/ExampleChatGPT.java

@@ -14,7 +14,7 @@
  * ExampleChatGPT
  *
  * @author <a href="https://github.com/jetkai">Kai</a>
- * @version 1.1.0
+ * @version 1.1.2
  * {@code - 07/03/2023}
  * @since 1.0.0
  * {@code - 05/03/2023}
@@ -32,8 +32,10 @@ final class ExampleChatGPT {
     public static void main(String[] args) {
         ExampleChatGPT gpt = new ExampleChatGPT();
 
+        System.setProperty("console.encoding", "UTF-8");
+
         //The first message that we want to send
-        String message1 = "Hello ChatGPT!";
+        String message1 = "こんにちは";
         //The second message that we want to send
         String message2 = "What was the first thing I said?";
 
@@ -81,8 +83,8 @@ private String exampleBuilders(String message) {
         //Including both our messages and the AI's messages
         messageHistory.addAll(createChatCompletion.asChatResponseDataList());
 
-        //Parse the response back as plain-text & replace \n & ascii characters (Šťŕĭńġ -> String)
-        return createChatCompletion.asNormalizedText();
+        //Parse the response back as plain-text
+        return createChatCompletion.asText();
     }
 
     private String exampleInstanceOrEnum(String message) {
@@ -110,8 +112,8 @@ private String exampleInstanceOrEnum(String message) {
         //Including both our messages and the AI's messages
         messageHistory.addAll(createChatCompletion.asChatResponseDataList());
 
-        //Parse the response back as plain-text & replace \n & ascii characters (Šťŕĭńġ -> String)
-        return createChatCompletion.asNormalizedText();
+        //Parse the response back as plain-text
+        return createChatCompletion.asText();
     }
 
     private String exampleHttpClientProxy(String message) {
@@ -143,8 +145,8 @@ private String exampleHttpClientProxy(String message) {
         //Including both our messages and the AI's messages
         messageHistory.addAll(createChatCompletion.asChatResponseDataList());
 
-        //Parse the response back as plain-text & replace \n & ascii characters (Šťŕĭńġ -> String)
-        return createChatCompletion.asNormalizedText();
+        //Parse the response back as plain-text
+        return createChatCompletion.asText();
     }
 
 }

src/main/java/io/github/jetkai/openai/api/CreateCompletion.java

@@ -10,6 +10,7 @@
 import java.text.Normalizer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * CreateCompletion
@@ -41,12 +42,9 @@ public CreateCompletion(Object completion, OpenAIEndpoints endpoint) {
      */
     public List<String> asSentences() {
         List<String> sentences = new ArrayList<>();
-        StringBuilder sentenceBuilder = new StringBuilder();
         String text = asText();
+        Objects.requireNonNull(text, "asText() can not be null");
 
-        if(text == null) {
-            return null;
-        }
         if(text.contains("\n")) {
             String[] words = text.split("\n");
             sentences.addAll(List.of(words));

src/main/java/io/github/jetkai/openai/net/request/RequestBuilder.java

@@ -40,7 +40,7 @@ public abstract class RequestBuilder {
 
     public HttpRequest createGetRequest(URI uri, String apiKey, String organization) {
         return HttpRequest.newBuilder()
-                .headers("Content-Type", "application/json")
+                .headers("Content-Type", "application/json; charset=utf-8")
                 .uri(uri)
                 .header("User-Agent", USER_AGENT)
                 .header("Authorization", "Bearer " + apiKey)
@@ -53,13 +53,13 @@ public HttpRequest createGetRequest(URI uri, String apiKey, String organization)
     public HttpRequest createPostRequest(URI uri, Object data, String apiKey, String organization) {
         Object postData = data == null ? "" : data;
         return HttpRequest.newBuilder()
-                .headers("Content-Type", "application/json")
+                .headers("Content-Type", "application/json; charset=utf-8")
                 .uri(uri)
                 .header("User-Agent", USER_AGENT)
                 .header("Authorization", "Bearer " + apiKey)
                 .header("OpenAI-Organization", organization)
                 .version(HttpClient.Version.HTTP_2)
-                .POST(HttpRequest.BodyPublishers.ofString(String.valueOf(postData)))
+                .POST(HttpRequest.BodyPublishers.ofString(String.valueOf(postData), StandardCharsets.UTF_8))
                 .build();
     }
 

src/test/java/CreateInstanceTest.java

@@ -1,5 +1,4 @@
 import io.github.jetkai.openai.api.ListModel;
-import io.github.jetkai.openai.api.ListModels;
 import io.github.jetkai.openai.net.OpenAIEndpoints;
 import io.github.jetkai.openai.openai.OpenAI;
 import org.junit.jupiter.api.Test;