fix: add 'clearMergeCache'

Author: niieani

README.md

@@ -393,6 +393,14 @@ setMergeCacheSize(5000)
 setMergeCacheSize(0)
 ```
 
+The cache is persisted between encoding calls. To explicitly clear the cache (e.g. to free up memory), use the `clearMergeCache` function:
+
+```ts
+import { clearMergeCache } from 'gpt-tokenizer'
+
+clearMergeCache()
+```
+
 ## Testing and Validation
 
 `gpt-tokenizer` includes a set of test cases in the [TestPlans.txt](./data/TestPlans.txt) file to ensure its compatibility with OpenAI's Python `tiktoken` library. These test cases validate the functionality and behavior of `gpt-tokenizer`, providing a reliable reference for developers.

src/BytePairEncodingCore.ts

@@ -104,6 +104,10 @@ export class BytePairEncodingCore {
     }
   }
 
+  clearMergeCache(): void {
+    this.mergeCache?.clear()
+  }
+
   *encodeNativeGenerator(
     text: string,
     allowedSpecial?: Set<string>,

src/GptEncoding.ts

@@ -128,6 +128,7 @@ export class GptEncoding {
     this.encodeChatGenerator = this.encodeChatGenerator.bind(this)
     this.countTokens = this.countTokens.bind(this)
     this.setMergeCacheSize = this.setMergeCacheSize.bind(this)
+    this.clearMergeCache = this.clearMergeCache.bind(this)
     this.modelName = modelName
   }
 
@@ -366,6 +367,10 @@ export class GptEncoding {
     this.bytePairEncodingCoreProcessor.setMergeCacheSize(size)
   }
 
+  clearMergeCache(): void {
+    this.bytePairEncodingCoreProcessor.clearMergeCache()
+  }
+
   decode(inputTokensToDecode: Iterable<number>): string {
     return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode)
   }

src/encoding/cl100k_base.ts

@@ -18,8 +18,10 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,

src/encoding/o200k_base.ts

@@ -18,8 +18,10 @@ const {
   encodeChatGenerator,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,

src/encoding/p50k_base.ts

@@ -16,8 +16,10 @@ const {
   countTokens,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,

src/encoding/p50k_edit.ts

@@ -16,8 +16,10 @@ const {
   countTokens,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,

src/encoding/r50k_base.ts

@@ -16,8 +16,10 @@ const {
   countTokens,
   vocabularySize,
   setMergeCacheSize,
+  clearMergeCache,
 } = api
 export {
+  clearMergeCache,
   countTokens,
   decode,
   decodeAsyncGenerator,