first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/support/test-case-generator.ts

@@ -0,0 +1,179 @@
+/**
+ * Test Case Generator
+ *
+ * Generates test cases for workflow evaluation using LLM with structured output.
+ * For default test cases, see fixtures/default-prompts.csv.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import { z } from 'zod';
+
+/**
+ * Options for test case generation.
+ */
+export interface TestCaseGeneratorOptions {
+	/** Number of test cases to generate (default: 10) */
+	count?: number;
+	/** Focus areas for generation */
+	focus?: string;
+	/** Complexity distribution */
+	complexity?: 'balanced' | 'simple' | 'complex';
+}
+
+/**
+ * A test case generated by the LLM.
+ */
+export interface GeneratedTestCase {
+	id: string;
+	name: string;
+	summary: string;
+	prompt: string;
+}
+
+/**
+ * Test case generator interface.
+ */
+export interface TestCaseGenerator {
+	/** Generate test cases */
+	generate(): Promise<GeneratedTestCase[]>;
+}
+
+/**
+ * Zod schema for structured output.
+ */
+const generatedTestCasesSchema = z.object({
+	testCases: z.array(
+		z.object({
+			id: z.string().describe('Unique identifier (e.g., "test_001")'),
+			name: z.string().describe('Short descriptive title'),
+			summary: z.string().describe('Brief description of what the workflow does'),
+			prompt: z.string().describe('User-facing prompt for workflow generation'),
+		}),
+	),
+});
+
+/** Inferred type from the Zod schema */
+type GeneratedTestCasesOutput = z.infer<typeof generatedTestCasesSchema>;
+
+/** Parse and validate LLM output using the Zod schema */
+function parseTestCasesOutput(value: unknown): GeneratedTestCasesOutput {
+	const parsed = generatedTestCasesSchema.safeParse(value);
+	if (!parsed.success) {
+		throw new Error(`Invalid LLM output: ${parsed.error.message}`);
+	}
+	return parsed.data;
+}
+
+/**
+ * System prompt for test case generation.
+ */
+const systemPrompt = `You are an expert at generating diverse test cases for an n8n workflow builder AI. Create test cases that cover various real-world scenarios and complexity levels.
+
+## Test Case Requirements:
+
+1. **Simple Test Cases**: Single operation workflows
+   - API calls
+   - Data transformations
+   - File operations
+   - Basic integrations
+
+2. **Medium Test Cases**: Multi-step workflows with logic
+   - Conditional logic (IF nodes)
+   - Data filtering and transformation
+   - Multiple API integrations
+   - Error handling
+
+3. **Complex Test Cases**: Advanced workflows
+   - Parallel execution branches
+   - Complex error handling and retry logic
+   - Multiple integrations with data synchronization
+   - Webhooks and event-driven flows
+
+## Guidelines:
+- Create realistic business scenarios
+- Include specific requirements that can be evaluated
+- Vary the domains (e-commerce, HR, marketing, DevOps, etc.)
+- Include both common and edge-case scenarios
+- Make prompts clear and unambiguous
+- Specify expected node types when possible
+
+## Output Format:
+Each test case should have:
+- Unique ID (e.g., "test_001")
+- Descriptive name
+- Brief description
+- Clear prompt that a user would give`;
+
+/**
+ * Get default focus based on complexity option.
+ */
+function getFocus(options?: TestCaseGeneratorOptions): string {
+	const complexity = options?.complexity ?? 'balanced';
+
+	if (options?.focus) {
+		return options.focus;
+	}
+
+	switch (complexity) {
+		case 'simple':
+			return 'simple, single-operation workflows like basic API calls, data transformations, and file operations';
+		case 'complex':
+			return 'complex, multi-step workflows with parallel execution, error handling, and multiple integrations';
+		case 'balanced':
+		default:
+			return 'balanced mix of API integrations, data processing, and automation scenarios';
+	}
+}
+
+/**
+ * Build the human message content.
+ */
+function buildHumanMessage(count: number, focus: string): string {
+	return `Generate ${count} diverse test cases for workflow generation evaluation.
+
+Focus on:
+${focus}
+
+Ensure a good mix of complexity levels and use cases.`;
+}
+
+/**
+ * Create a test case generator that uses LLM to generate test cases.
+ *
+ * @param llm - Language model to use for generation
+ * @param options - Generation options
+ * @returns A test case generator
+ *
+ * @example
+ * ```typescript
+ * const generator = createTestCaseGenerator(llm, { count: 20 });
+ * const testCases = await generator.generate();
+ * ```
+ */
+export function createTestCaseGenerator(
+	llm: BaseChatModel,
+	options?: TestCaseGeneratorOptions,
+): TestCaseGenerator {
+	const count = options?.count ?? 10;
+	const focus = getFocus(options);
+
+	// Create LLM with structured output
+	const llmWithStructuredOutput = llm.withStructuredOutput(generatedTestCasesSchema);
+
+	return {
+		async generate(): Promise<GeneratedTestCase[]> {
+			const humanMessage = buildHumanMessage(count, focus);
+
+			const rawResult = await llmWithStructuredOutput.invoke([
+				new SystemMessage(systemPrompt),
+				new HumanMessage(humanMessage),
+			]);
+
+			// Validate and parse the LLM output using Zod
+			const result = parseTestCasesOutput(rawResult);
+
+			return result.testCases;
+		},
+	};
+}