refactor: use tokenizer to accurately count tokens (#114)

Co-authored-by: hiroki osame <hiroki.osame@gmail.com>

pnpm-lock.yaml

@@ -2,6 +2,7 @@ lockfileVersion: 5.4
 
 specifiers:
   '@clack/prompts': ^0.2.2
+  '@dqbd/tiktoken': ^0.4.0
   '@pvtnbr/eslint-config': ^0.33.0
   '@types/ini': ^1.3.31
   '@types/inquirer': ^9.0.3
@@ -17,6 +18,9 @@ specifiers:
   simple-git-hooks: ^2.8.1
   typescript: ^4.9.5
 
+dependencies:
+  '@dqbd/tiktoken': 0.4.0
+
 devDependencies:
   '@clack/prompts': 0.2.2
   '@pvtnbr/eslint-config': 0.33.0_7kw3g6rralp5ps6mg3uyzz6azm
@@ -72,6 +76,10 @@ packages:
       sisteransi: 1.0.5
     dev: true
 
+  /@dqbd/tiktoken/0.4.0:
+    resolution: {integrity: sha512-iaHgmwKAOqowBFZKxelyszoeGLoNw62eOULcmyme1aA1Ymr3JgYl0V7jwpuUm7fksalycZajx3loFn9TRUaviw==}
+    dev: false
+
   /@esbuild/android-arm/0.17.8:
     resolution: {integrity: sha512-0/rb91GYKhrtbeglJXOhAv9RuYimgI8h623TplY2X+vA4EXnk3Zj1fXZreJ0J3OJJu1bwmb0W7g+2cT/d8/l/w==}
     engines: {node: '>=12'}