feat: add new models and ability to estimate cost (#72)

BREAKING CHANGE: changes the default encoding to `o200k_base` as that is what most modern models use now

fixes #71
fixes #70

Author: niieani

.config/beemo/eslint.ts

@@ -4,7 +4,12 @@ const config: ESLintConfig = {
   rules: {
     'import/no-unresolved': 'off',
   },
-  ignorePatterns: ['**/models/*.js'],
+  ignorePatterns: [
+    '**/models/*.js',
+    'src/model/*.ts',
+    'benchmark/**/*.ts',
+    'src/codegen/*.js',
+  ],
 }
 
 export default config

.prettierignore

@@ -7,4 +7,6 @@ dts/
 esm/
 lib/
 mjs/
-umd/
+umd/
+**/*.gen.ts
+src/models.ts

.yarn/releases/yarn-4.5.0.cjs

@@ -8,4 +8,4 @@ plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-postinstall-dev.cjs
     spec: "https://raw.githubusercontent.com/sachinraja/yarn-plugin-postinstall-dev/main/bundles/%40yarnpkg/plugin-postinstall-dev.js"
 
-yarnPath: .yarn/releases/yarn-4.5.0.cjs
+yarnPath: .yarn/releases/yarn-4.9.2.cjs

.yarn/releases/yarn-4.9.2.cjs

@@ -2,7 +2,7 @@
 
 [![Play with gpt-tokenizer](https://codesandbox.io/static/img/play-codesandbox.svg)](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark)
 
-`gpt-tokenizer` is a Token Byte Pair Encoder/Decoder supporting all OpenAI's models (including GPT-3.5, GPT-4, GPT-4o, and o1).
+`gpt-tokenizer` is a Token Byte Pair Encoder/Decoder supporting all OpenAI's models (including GPT-4o, o1, o3, o4, GPT-4.1 and older models like GPT-3.5, GPT-4).
 It's the [_fastest, smallest and lowest footprint_](#benchmarks) GPT tokenizer available for all JavaScript environments. It's written in TypeScript.
 
 This library has been trusted by:
@@ -17,7 +17,7 @@ Please consider [🩷 sponsoring](https://github.com/sponsors/niieani) the proje
 
 #### Features
 
-As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional, unique features sprinkled on top:
+It is the most feature-complete, open-source GPT tokenizer on NPM. This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional, unique features sprinkled on top:
 
 - Support for easily tokenizing chats thanks to the `encodeChat` function
 - Support for all current OpenAI models (available encodings: `r50k_base`, `p50k_base`, `p50k_edit`, `cl100k_base` and `o200k_base`)
@@ -26,6 +26,8 @@ As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. T
 - Provides the ability to decode an asynchronous stream of data (using `decodeAsyncGenerator` and `decodeGenerator` with any iterable input)
 - No global cache (no accidental memory leaks, as with the original GPT-3-Encoder implementation)
 - Includes a highly performant `isWithinTokenLimit` function to assess token limit without encoding the entire text/chat
+- Built-in cost estimation with the `estimateCost` function for calculating API usage costs
+- Full library of OpenAI models with comprehensive pricing information (see [`src/models.ts`](./src/models.ts) and [`src/models.gen.ts`](./src/models.gen.ts))
 - Improves overall performance by eliminating transitive arrays
 - Type-safe (written in TypeScript)
 - Works in the browser out-of-the-box
@@ -51,8 +53,8 @@ npm install gpt-tokenizer
 
 If you wish to use a custom encoding, fetch the relevant script.
 
-- https://unpkg.com/gpt-tokenizer/dist/o200k_base.js (for `gpt-4o` and `o1`)
-- https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js (for `gpt-4-*` and `gpt-3.5-turbo`)
+- https://unpkg.com/gpt-tokenizer/dist/o200k_base.js (for all modern models, such as `gpt-4o`, `gpt-4.1`, `o1` and others)
+- https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js (for `gpt-4` and `gpt-3.5`)
 - https://unpkg.com/gpt-tokenizer/dist/p50k_base.js
 - https://unpkg.com/gpt-tokenizer/dist/p50k_edit.js
 - https://unpkg.com/gpt-tokenizer/dist/r50k_base.js
@@ -130,7 +132,7 @@ for await (const textChunk of decodeAsyncGenerator(asyncTokens)) {
 }
 ```
 
-By default, importing from `gpt-tokenizer` uses `cl100k_base` encoding, used by `gpt-3.5-turbo` and `gpt-4`.
+By default, importing from `gpt-tokenizer` uses `o200k_base` encoding, used by all modern OpenAI models, including `gpt-4o`, `gpt-4.1`, `o1`, etc.
 
 To get a tokenizer for a different model, import it directly, for example:
 
@@ -182,16 +184,18 @@ import {
 
 ### Supported models and their encodings
 
-- `o1-*` (`o200k_base`)
+We support all OpenAI models, including the latest ones, with the following encodings:
+
+- `o`-series models, like `o1-*`, `o3-*` and `o4-*` (`o200k_base`)
 - `gpt-4o` (`o200k_base`)
 - `gpt-4-*` (`cl100k_base`)
-- `gpt-3.5-turbo` (`cl100k_base`)
+- `gpt-3.5-*` (`cl100k_base`)
 - `text-davinci-003` (`p50k_base`)
 - `text-davinci-002` (`p50k_base`)
 - `text-davinci-001` (`r50k_base`)
 - ...and many other models, see [models.ts](./src/models.ts) for an up-to-date list of supported models and their encodings.
 
-Note: if you're using `gpt-3.5-*` or `gpt-4-*` and don't see the model you're looking for, use the `cl100k_base` encoding directly.
+If you don't see the model you're looking for, the default encoding is probably the one you want.
 
 ## API
 
@@ -326,6 +330,31 @@ async function processTokens(asyncTokensIterator) {
 }
 ```
 
+### `estimateCost(tokenCount: number, modelSpec?: ModelSpec): PriceData`
+
+Estimates the cost of processing a given number of tokens using the model's pricing data. This function calculates costs for different API usage types (main API, batch API) and cached tokens when available.
+
+The function returns a `PriceData` object with the following structure:
+- `main`: Main API pricing with `input`, `output`, `cached_input`, and `cached_output` costs
+- `batch`: Batch API pricing with the same cost categories
+
+All costs are calculated in USD based on the token count provided.
+
+Example:
+
+```typescript
+import { estimateCost } from 'gpt-tokenizer/model/gpt-4o'
+
+const tokenCount = 1000
+const costEstimate = estimateCost(tokenCount)
+
+console.log('Main API input cost:', costEstimate.main?.input)
+console.log('Main API output cost:', costEstimate.main?.output)
+console.log('Batch API input cost:', costEstimate.batch?.input)
+```
+
+Note: The model spec must be available either through the model-specific import or by passing it as the second parameter. Cost information may not be available for all models.
+
 ## Special tokens
 
 There are a few special tokens that are used by the GPT models.

.yarnrc.yml

@@ -1,4 +1,3 @@
-// benchmarkWorker.ts
 import type {
   BenchmarkResult,
   WorkerInput,

README.md

@@ -73,8 +73,10 @@
     "dist"
   ],
   "scripts": {
-    "codegen:models": "rm -rf src/model && yarn tsx src/codegen/generateByModel.ts",
-    "codegen:bpe": "rm -rf src/bpeRanks && yarn tsx src/codegen/generateJsBpe.ts",
+    "codegen": "yarn codegen:bpe && yarn codegen:chat-enabled && yarn codegen:models",
+    "codegen:models": "rm -rf src/model && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateByModel.ts",
+    "codegen:bpe": "rm -rf src/bpeRanks && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateJsBpe.ts",
+    "codegen:chat-enabled": "rm -rf src/chat && node --experimental-transform-types --import node-resolve-ts/register src/codegen/generateChatEnabled.ts",
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
     "build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --target es2022",
@@ -87,11 +89,11 @@
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
     "postinstallDev": "yarn prepare",
-    "prepare": "rrun husky install .config/husky && beemo create-config",
+    "prepare": "rrun husky install .config/husky && beemo create-config && echo '\n**/*.gen.ts\nsrc/models.ts' >> .prettierignore",
     "release": "beemo run-script release",
     "test": "yarn test:format && yarn test:types && yarn test:lint && yarn test:code",
     "test:code": "vitest",
-    "test:format": "yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
+    "test:format": "yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\" --ignore-path .prettierignore",
     "test:lint": "rrun eslint 'src/*.{js,jsx,ts,tsx}'",
     "test:types": "yarn rrun tsc --noEmit"
   },
@@ -117,17 +119,19 @@
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",
-    "@niieani/scaffold": "^1.7.39",
-    "@swc/cli": "^0.5.2",
-    "@swc/core": "^1.10.4",
-    "tsx": "^4.19.2",
-    "typescript": "^5.7.2",
-    "vitest": "^2.1.8"
+    "@niieani/scaffold": "^1.7.49",
+    "@swc/cli": "^0.7.7",
+    "@swc/core": "^1.11.31",
+    "devalue": "^5.1.1",
+    "node-resolve-ts": "^1.0.2",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.2"
   },
   "resolutions": {
-    "typescript": "5.7.2"
+    "typescript": "5.8.3",
+    "prettier": "^3"
   },
-  "packageManager": "yarn@4.5.0",
+  "packageManager": "yarn@4.9.2",
   "publishConfig": {
     "access": "public"
   }

benchmark/src/benchmarkWorker.ts

@@ -7,10 +7,14 @@ import { type ChatMessage, GptEncoding } from './GptEncoding.js'
 import {
   type ChatModelName,
   type EncodingName,
+  type ModelName,
   chatModelParams,
+  DEFAULT_ENCODING,
   encodingNames,
+  modelToEncodingMap,
 } from './mapping.js'
-import { models } from './models.js'
+import * as models from './models.js'
+import * as modelsMap from './modelsMap.js'
 import { resolveEncoding } from './resolveEncoding.js'
 import { EndOfText } from './specialTokens.js'
 
@@ -250,22 +254,23 @@ const exampleMessages: ChatMessage[] = [
   },
 ] as const
 
-describe.each(chatModelNames)('%s', (modelName) => {
-  const encoding = GptEncoding.getEncodingApiForModel(
-    modelName,
-    resolveEncoding,
+describe.each(chatModelNames)('%s', async (modelName) => {
+  const encoding: GptEncoding = await import(`./model/${modelName}.ts`).then(
+    (mod) => mod.default,
   )
-  const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
+  const expectedEncodedLength = modelName.startsWith('gpt-3.5')
     ? 127
-    : modelName.startsWith('gpt-4o')
-    ? 120
-    : 121
+    : modelName.startsWith('gpt-4') &&
+        !modelName.startsWith('gpt-4o') &&
+        !modelName.startsWith('gpt-4.')
+      ? 121
+      : 120
 
   describe('chat functionality', () => {
     test('encodes a chat correctly', () => {
       const encoded = encoding.encodeChat(exampleMessages)
-      expect(encoded).toMatchSnapshot()
       expect(encoded).toHaveLength(expectedEncodedLength)
+      expect(encoded).toMatchSnapshot()
 
       const decoded = encoding.decode(encoded)
       expect(decoded).toMatchSnapshot()
@@ -288,45 +293,77 @@ describe.each(chatModelNames)('%s', (modelName) => {
   })
 })
 
-describe('estimateCost functionality', () => {
-  const gpt4oEncoding = GptEncoding.getEncodingApiForModel(
-    'gpt-4o',
-    resolveEncoding,
+describe('estimateCost functionality', async () => {
+  const gpt4oEncoding = await import(`./model/gpt-4o.js`).then(
+    (mod) => mod.default,
   )
-  const gpt35Encoding = GptEncoding.getEncodingApiForModel(
-    'gpt-3.5-turbo',
-    resolveEncoding,
+  const gpt35Encoding = await import(`./model/gpt-3.5-turbo.js`).then(
+    (mod) => mod.default,
   )
 
   test('estimates cost correctly for gpt-4o model', () => {
     const tokenCount = 1_000
     const cost = gpt4oEncoding.estimateCost(tokenCount)
 
-    // gpt-4o has $2.5 per million tokens for input and $10 per million tokens for output
-    expect(cost.input).toBeCloseTo(0.002_5, 6) // 1000/1M * $2.5
-    expect(cost.output).toBeCloseTo(0.01, 6) // 1000/1M * $10
-    expect(cost.batchInput).toBeCloseTo(0.001_25, 6) // 1000/1M * $1.25
-    expect(cost.batchOutput).toBeCloseTo(0.005, 6) // 1000/1M * $5
+    expect(cost).toMatchInlineSnapshot(`
+      {
+        "batch": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.005,
+          "output": 0.015,
+        },
+        "main": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.01,
+          "output": 0.03,
+        },
+      }
+    `)
   })
 
   test('estimates cost correctly for gpt-3.5-turbo model', () => {
     const tokenCount = 1_000
     const cost = gpt35Encoding.estimateCost(tokenCount)
-
-    // gpt-3.5-turbo has $0.5 per million tokens for input and $1.5 per million tokens for output
-    expect(cost.input).toBeCloseTo(0.000_5, 6) // 1000/1M * $0.5
-    expect(cost.output).toBeCloseTo(0.001_5, 6) // 1000/1M * $1.5
-    expect(cost.batchInput).toBeCloseTo(0.000_25, 6) // 1000/1M * $0.25
-    expect(cost.batchOutput).toBeCloseTo(0.000_75, 6) // 1000/1M * $0.75
+    expect(cost).toMatchInlineSnapshot(`
+      {
+        "batch": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.00025,
+          "output": 0.00075,
+        },
+        "main": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.0005,
+          "output": 0.0015,
+        },
+      }
+    `)
   })
 
   test('allows overriding model name', () => {
     const tokenCount = 1_000
     // Use gpt-4o encoding but override with gpt-3.5-turbo model name
-    const cost = gpt4oEncoding.estimateCost(tokenCount, 'gpt-3.5-turbo')
-
-    expect(cost.input).toBeCloseTo(0.000_5, 6) // 1000/1M * $0.5
-    expect(cost.output).toBeCloseTo(0.001_5, 6) // 1000/1M * $1.5
+    const cost = gpt4oEncoding.estimateCost(tokenCount, models['gpt-3.5-turbo'])
+    expect(cost).toMatchInlineSnapshot(`
+      {
+        "batch": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.00025,
+          "output": 0.00075,
+        },
+        "main": {
+          "cached_input": undefined,
+          "cached_output": undefined,
+          "input": 0.0005,
+          "output": 0.0015,
+        },
+      }
+    `)
   })
 
   test('throws error when model name is not provided', () => {
@@ -335,30 +372,29 @@ describe('estimateCost functionality', () => {
 
     // No model name was provided during initialization or function call
     expect(() => encoding.estimateCost(tokenCount)).toThrow(
-      'Model name must be provided either during initialization or passed in to the method.',
+      'Model spec must be provided either during initialization or passed in to the method.',
     )
   })
 
-  test('throws error for unknown model', () => {
-    const tokenCount = 1_000
-    expect(() =>
-      gpt4oEncoding.estimateCost(tokenCount, 'non-existent-model' as any),
-    ).toThrow('Unknown model: non-existent-model')
-  })
-
   test('only includes properties that exist for the model', () => {
     // Find a model that only has input cost but no output cost
     const modelWithInputOnly = Object.entries(models).find(
       ([_, model]) =>
-        model.cost?.input !== undefined && model.cost?.output === undefined,
+        'price_data' in model &&
+        model.price_data?.main?.input !== undefined &&
+        (!('output' in model.price_data.main) ||
+          model.price_data?.main?.output === undefined),
     )
 
     if (modelWithInputOnly) {
       const [modelName] = modelWithInputOnly
-      const cost = gpt4oEncoding.estimateCost(1_000, modelName as any)
+      const cost = gpt4oEncoding.estimateCost(
+        1_000,
+        models[modelName as ModelName],
+      )
 
-      expect(cost.input).toBeDefined()
-      expect(cost.output).toBeUndefined()
+      expect(cost.main?.input).toBeDefined()
+      expect(cost.main?.output).toBeUndefined()
     } else {
       // Skip test if we can't find an appropriate model
       console.log('Skipping test: no model with input-only cost found')