/** * Tests for CSV prompt loader. */ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { loadTestCasesFromCsv } from '../cli/csv-prompt-loader'; describe('csv-prompt-loader', () => { const tempDirs: string[] = []; function writeTempCsv(filename: string, content: string): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-csv-')); tempDirs.push(dir); const csvPath = path.join(dir, filename); fs.writeFileSync(csvPath, content, 'utf8'); return csvPath; } afterEach(() => { while (tempDirs.length > 0) { const dir = tempDirs.pop(); if (dir && fs.existsSync(dir)) { fs.rmSync(dir, { recursive: true, force: true }); } } }); it('should load dos/donts into context when present', () => { const csvPath = writeTempCsv( 'pairwise.csv', 'id,prompt,dos,donts\npw-1,"Create a workflow","Must use Notion","No HTTP Request"\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([ { id: 'pw-1', prompt: 'Create a workflow', context: { dos: 'Must use Notion', donts: 'No HTTP Request' }, }, ]); }); it('should load prompts without context when dos/donts are absent', () => { const csvPath = writeTempCsv('llm.csv', 'id,prompt\nllm-1,"Create a workflow"\n'); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]); }); it('should support header column re-ordering', () => { const csvPath = writeTempCsv( 'reorder.csv', 'prompt,id,donts,dos\n"Create a workflow","pw-1","No HTTP Request","Must use Notion"\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([ { id: 'pw-1', prompt: 'Create a workflow', context: { dos: 'Must use Notion', donts: 'No HTTP Request' }, }, ]); }); it('should support do/dont header aliases', () => { const csvPath = writeTempCsv( 'aliases.csv', 'id,prompt,do,dont\npw-1,"Create a workflow","Must use Notion","No HTTP Request"\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([ { id: 'pw-1', prompt: 'Create a workflow', context: { dos: 'Must use Notion', donts: 'No HTTP Request' }, }, ]); }); it('should not create context when dos/donts columns are present but empty', () => { const csvPath = writeTempCsv( 'empty-context.csv', 'id,prompt,dos,donts\npw-1,"Create a workflow",,\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([{ id: 'pw-1', prompt: 'Create a workflow' }]); }); it('should allow headerless CSV (treat first column as prompt)', () => { const csvPath = writeTempCsv('no-header.csv', '"Create a workflow"\n"Second prompt"\n'); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([ { id: 'csv-case-1', prompt: 'Create a workflow' }, { id: 'csv-case-2', prompt: 'Second prompt' }, ]); }); it('should ignore rows with empty prompts', () => { const csvPath = writeTempCsv('empty-rows.csv', 'id,prompt\nrow-1,\nrow-2,"Valid prompt"\n'); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([{ id: 'row-2', prompt: 'Valid prompt' }]); }); it('should handle UTF-8 BOM', () => { const csvPath = writeTempCsv('bom.csv', '\ufeffid,prompt\nllm-1,"Create a workflow"\n'); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]); }); it('should resolve relative paths from process.cwd()', () => { const csvPath = writeTempCsv('relative.csv', 'id,prompt\nllm-1,"Create a workflow"\n'); const relativePath = path.relative(process.cwd(), csvPath); const testCases = loadTestCasesFromCsv(relativePath); expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]); }); it('should throw when file does not exist', () => { expect(() => loadTestCasesFromCsv('/definitely-not-a-real-path.csv')).toThrow( /CSV file not found/, ); }); it('should throw when CSV is empty', () => { const csvPath = writeTempCsv('empty.csv', ''); expect(() => loadTestCasesFromCsv(csvPath)).toThrow('The provided CSV file is empty'); }); it('should throw when no valid prompts exist', () => { const csvPath = writeTempCsv('no-prompts.csv', 'id,prompt\nrow-1,\nrow-2," "\n'); expect(() => loadTestCasesFromCsv(csvPath)).toThrow( 'No valid prompts found in the provided CSV file', ); }); it('should parse annotations column as JSON into context', () => { const csvPath = writeTempCsv( 'annotations.csv', 'id,prompt,annotations\nann-1,"Create a workflow","{""code_necessary"":true}"\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([ { id: 'ann-1', prompt: 'Create a workflow', context: { annotations: { code_necessary: true } }, }, ]); }); it('should skip invalid JSON in annotations column with warning', () => { const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); const csvPath = writeTempCsv( 'bad-annotations.csv', 'id,prompt,annotations\nbad-1,"Create a workflow","not-json"\n', ); const testCases = loadTestCasesFromCsv(csvPath); expect(testCases).toEqual([{ id: 'bad-1', prompt: 'Create a workflow', context: {} }]); expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('invalid JSON')); warnSpy.mockRestore(); }); });