refactor: use tokenizer to accurately count tokens (#114)

Co-authored-by: hiroki osame <hiroki.osame@gmail.com>
2023-03-02 10:00:52 +08:00
parent 5f2efc83f5
commit c194c917d7
3 changed files with 21 additions and 3 deletions
--- a/package.json
+++ b/package.json
@@ -28,6 +28,9 @@
 	"lint-staged": {
 		"*.ts": "eslint --cache"
 	},
+	"dependencies": {
+		"@dqbd/tiktoken": "^0.4.0"
+	},
 	"devDependencies": {
 		"@clack/prompts": "^0.2.2",
 		"@pvtnbr/eslint-config": "^0.33.0",