Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions src/main/java/com/google/genai/LocalTokenizerLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,21 @@ public String toString() {
private static final Map<String, ModelProto> modelProtoCache = new ConcurrentHashMap<>();
private static final Map<String, LocalTokenizerProcessor> localTokenizerProcessorCache =
new ConcurrentHashMap<>();
private static OkHttpClient httpClient = new OkHttpClient();
// Non-null only when replaced by tests.
private static OkHttpClient httpClient;

private LocalTokenizerLoader() {}

private static final class HttpClientHolder {
private static final OkHttpClient INSTANCE = new OkHttpClient();

private HttpClientHolder() {}
}

private static OkHttpClient getHttpClient() {
return httpClient != null ? httpClient : HttpClientHolder.INSTANCE;
}

/** Gets the tokenizer name for the given model name. */
public static String getTokenizerName(String modelName) {
if (GEMINI_MODELS_TO_TOKENIZER_NAMES.containsKey(modelName)) {
Expand Down Expand Up @@ -240,7 +251,7 @@ private static void maybeSaveToCache(Path cacheDir, Path cachePath, byte[] conte

private static byte[] loadFromUrl(String fileUrl, String expectedHash) throws IOException {
Request request = new Request.Builder().url(fileUrl).build();
try (Response response = httpClient.newCall(request).execute()) {
try (Response response = getHttpClient().newCall(request).execute()) {
if (response == null) {
throw new GenAiIOException("HTTP request failed: response is null");
}
Expand Down
12 changes: 12 additions & 0 deletions src/test/java/com/google/genai/LocalTokenizerLoaderTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand Down Expand Up @@ -98,11 +99,22 @@ void setUp() throws Exception {
@AfterEach
void tearDown() throws Exception {
System.setProperty("java.io.tmpdir", originalTmpDir);
Field clientField = LocalTokenizerLoader.class.getDeclaredField("httpClient");
setStatic(clientField, null);
// Restore original tokenizers map
Field tokenizersField = LocalTokenizerLoader.class.getDeclaredField("TOKENIZERS");
setStatic(tokenizersField, originalTokenizers);
}

@Test
void getTokenizerName_doesNotInitializeHttpClient() throws Exception {
Field clientField = LocalTokenizerLoader.class.getDeclaredField("httpClient");
setStatic(clientField, null);

assertEquals("gemma3", LocalTokenizerLoader.getTokenizerName("gemini-2.5-pro"));
assertNull(clientField.get(null));
}

@Test
void getTokenizerName_supportedModels() {
assertEquals("gemma3", LocalTokenizerLoader.getTokenizerName("gemini-2.5-pro"));
Expand Down