alighasami
3d5eaf9445
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
23 lines
832 B
JavaScript
23 lines
832 B
JavaScript
const glob = require('fast-glob');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
function copyTokenizerJsonFiles(baseDir) {
|
|
const targetBaseDir = process.argv[3] || 'dist';
|
|
// Make sure the target directory exists
|
|
const targetDir = path.resolve(baseDir, targetBaseDir, 'utils', 'tokenizer');
|
|
if (!fs.existsSync(targetDir)) {
|
|
fs.mkdirSync(targetDir, { recursive: true });
|
|
}
|
|
// Copy all tokenizer JSON files
|
|
const files = glob.sync('src/utils/tokenizer/*.json', { cwd: baseDir });
|
|
for (const file of files) {
|
|
const sourcePath = path.resolve(baseDir, file);
|
|
const targetPath = path.resolve(baseDir, targetBaseDir, file.replace('src/', ''));
|
|
fs.copyFileSync(sourcePath, targetPath);
|
|
console.log(`Copied: ${file} -> ${targetPath.replace(baseDir, '')}`);
|
|
}
|
|
}
|
|
|
|
copyTokenizerJsonFiles(process.argv[2] || '.');
|