first commit

packages/@n8n/ai-utilities/scripts/copy-tokenizer-json.js

@@ -0,0 +1,22 @@
+const glob = require('fast-glob');
+const fs = require('fs');
+const path = require('path');
+
+function copyTokenizerJsonFiles(baseDir) {
+	const targetBaseDir = process.argv[3] || 'dist';
+	// Make sure the target directory exists
+	const targetDir = path.resolve(baseDir, targetBaseDir, 'utils', 'tokenizer');
+	if (!fs.existsSync(targetDir)) {
+		fs.mkdirSync(targetDir, { recursive: true });
+	}
+	// Copy all tokenizer JSON files
+	const files = glob.sync('src/utils/tokenizer/*.json', { cwd: baseDir });
+	for (const file of files) {
+		const sourcePath = path.resolve(baseDir, file);
+		const targetPath = path.resolve(baseDir, targetBaseDir, file.replace('src/', ''));
+		fs.copyFileSync(sourcePath, targetPath);
+		console.log(`Copied: ${file} -> ${targetPath.replace(baseDir, '')}`);
+	}
+}
+
+copyTokenizerJsonFiles(process.argv[2] || '.');