refactor: use tokenizer to accurately count tokens (#114)

Co-authored-by: hiroki osame <hiroki.osame@gmail.com>

package.json

@@ -28,6 +28,9 @@
 	"lint-staged": {
 		"*.ts": "eslint --cache"
 	},
+	"dependencies": {
+		"@dqbd/tiktoken": "^0.4.0"
+	},
 	"devDependencies": {
 		"@clack/prompts": "^0.2.2",
 		"@pvtnbr/eslint-config": "^0.33.0",

pnpm-lock.yaml

@@ -2,6 +2,7 @@ lockfileVersion: 5.4
 
 specifiers:
   '@clack/prompts': ^0.2.2
+  '@dqbd/tiktoken': ^0.4.0
   '@pvtnbr/eslint-config': ^0.33.0
   '@types/ini': ^1.3.31
   '@types/inquirer': ^9.0.3
@@ -17,6 +18,9 @@ specifiers:
   simple-git-hooks: ^2.8.1
   typescript: ^4.9.5
 
+dependencies:
+  '@dqbd/tiktoken': 0.4.0
+
 devDependencies:
   '@clack/prompts': 0.2.2
   '@pvtnbr/eslint-config': 0.33.0_7kw3g6rralp5ps6mg3uyzz6azm
@@ -72,6 +76,10 @@ packages:
       sisteransi: 1.0.5
     dev: true
 
+  /@dqbd/tiktoken/0.4.0:
+    resolution: {integrity: sha512-iaHgmwKAOqowBFZKxelyszoeGLoNw62eOULcmyme1aA1Ymr3JgYl0V7jwpuUm7fksalycZajx3loFn9TRUaviw==}
+    dev: false
+
   /@esbuild/android-arm/0.17.8:
     resolution: {integrity: sha512-0/rb91GYKhrtbeglJXOhAv9RuYimgI8h623TplY2X+vA4EXnk3Zj1fXZreJ0J3OJJu1bwmb0W7g+2cT/d8/l/w==}
     engines: {node: '>=12'}

src/utils/openai.ts

@@ -1,5 +1,6 @@
 import https from 'https';
 import type { CreateCompletionRequest, CreateCompletionResponse } from 'openai';
+import { encoding_for_model as encodingForModel } from '@dqbd/tiktoken';
 
 const createCompletion = (
 	apiKey: string,
@@ -58,6 +59,9 @@ const deduplicateMessages = (array: string[]) => Array.from(new Set(array));
 
 const promptTemplate = 'Write an insightful but concise Git commit message in a complete sentence in present tense for the following diff without prefacing it with anything:';
 
+const model = 'text-davinci-003';
+const encoder = encodingForModel(model);
+
 export const generateCommitMessage = async (
 	apiKey: string,
 	diff: string,
@@ -65,14 +69,17 @@ export const generateCommitMessage = async (
 ) => {
 	const prompt = `${promptTemplate}\n${diff}`;
 
-	// Accounting for GPT-3's input req of 4k tokens (approx 8k chars)
-	if (prompt.length > 8000) {
+	/**
+	 * text-davinci-003 has a token limit of 4000
+	 * https://platform.openai.com/docs/models/overview#:~:text=to%20Sep%202021-,text%2Ddavinci%2D003,-Can%20do%20any
+	 */
+	if (encoder.encode(prompt).length > 4000) {
 		throw new Error('The diff is too large for the OpenAI API. Try reducing the number of staged changes, or write your own commit message.');
 	}
 
 	try {
 		const completion = await createCompletion(apiKey, {
-			model: 'text-davinci-003',
+			model,
 			prompt,
 			temperature: 0.7,
 			top_p: 1,