first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/argument-parser.test.ts

@@ -0,0 +1,182 @@
+import { parseEvaluationArgs, type EvaluationSuite } from '../cli/argument-parser';
+
+describe('argument-parser', () => {
+	describe('suite options', () => {
+		it('accepts all valid suite options', () => {
+			const validSuites: EvaluationSuite[] = [
+				'llm-judge',
+				'pairwise',
+				'programmatic',
+				'similarity',
+			];
+
+			for (const suite of validSuites) {
+				const args = parseEvaluationArgs(['--suite', suite]);
+				expect(args.suite).toBe(suite);
+			}
+		});
+	});
+
+	it('parses numeric flags like --max-examples and --concurrency', () => {
+		const args = parseEvaluationArgs([
+			'--suite',
+			'pairwise',
+			'--backend',
+			'langsmith',
+			'--max-examples',
+			'5',
+			'--concurrency',
+			'3',
+		]);
+
+		expect(args.maxExamples).toBe(5);
+		expect(args.concurrency).toBe(3);
+	});
+
+	it('supports inline --max-examples= syntax', () => {
+		const args = parseEvaluationArgs(['--max-examples=7']);
+		expect(args.maxExamples).toBe(7);
+	});
+
+	it('parses filters for pairwise suite', () => {
+		const args = parseEvaluationArgs([
+			'--suite',
+			'pairwise',
+			'--backend',
+			'langsmith',
+			'--filter',
+			'do:Slack',
+			'--filter',
+			'technique:content_generation',
+		]);
+
+		expect(args.filters).toEqual({
+			doSearch: 'Slack',
+			technique: 'content_generation',
+		});
+	});
+
+	it('accepts prompt values that start with "-"', () => {
+		const args = parseEvaluationArgs(['--prompt', '-starts-with-dash']);
+		expect(args.prompt).toBe('-starts-with-dash');
+	});
+
+	it('rejects conflicting backend/local when --langsmith is set', () => {
+		expect(() => parseEvaluationArgs(['--langsmith', '--backend', 'local'])).toThrow(
+			'Cannot combine `--langsmith` with `--backend local`',
+		);
+	});
+
+	it('treats --langsmith as backend=langsmith', () => {
+		const args = parseEvaluationArgs(['--langsmith']);
+		expect(args.backend).toBe('langsmith');
+	});
+
+	it('rejects do/dont filters for non-pairwise suite', () => {
+		expect(() => parseEvaluationArgs(['--suite', 'llm-judge', '--filter', 'do:Slack'])).toThrow(
+			'only supported for `--suite pairwise`',
+		);
+	});
+
+	it('rejects malformed filters', () => {
+		expect(() =>
+			parseEvaluationArgs(['--suite', 'pairwise', '--backend', 'langsmith', '--filter', 'nope']),
+		).toThrow('Invalid `--filter` format');
+	});
+
+	describe('--webhook-url', () => {
+		it('parses valid HTTPS webhook URL', () => {
+			const args = parseEvaluationArgs(['--webhook-url', 'https://example.com/webhook']);
+			expect(args.webhookUrl).toBe('https://example.com/webhook');
+		});
+
+		it('parses webhook URL with inline = syntax', () => {
+			const args = parseEvaluationArgs(['--webhook-url=https://api.example.com/hook']);
+			expect(args.webhookUrl).toBe('https://api.example.com/hook');
+		});
+
+		it('rejects invalid URL format', () => {
+			expect(() => parseEvaluationArgs(['--webhook-url', 'not-a-url'])).toThrow();
+		});
+
+		it('rejects non-URL strings', () => {
+			expect(() => parseEvaluationArgs(['--webhook-url', 'just-some-text'])).toThrow();
+		});
+
+		it('allows webhook URL to be undefined when not provided', () => {
+			const args = parseEvaluationArgs([]);
+			expect(args.webhookUrl).toBeUndefined();
+		});
+
+		it('parses webhook URL with path and query params', () => {
+			const args = parseEvaluationArgs([
+				'--webhook-url',
+				'https://hooks.example.com/api/v1/notify?token=abc123',
+			]);
+			expect(args.webhookUrl).toBe('https://hooks.example.com/api/v1/notify?token=abc123');
+		});
+	});
+
+	it('accepts binary-checks suite', () => {
+		const args = parseEvaluationArgs(['--suite', 'binary-checks']);
+		expect(args.suite).toBe('binary-checks');
+	});
+
+	it('parses --checks flag as comma-separated list', () => {
+		const args = parseEvaluationArgs([
+			'--suite',
+			'binary-checks',
+			'--checks',
+			'has_nodes,has_trigger,all_nodes_connected',
+		]);
+		expect(args.checks).toEqual(['has_nodes', 'has_trigger', 'all_nodes_connected']);
+	});
+
+	it('trims whitespace in --checks values', () => {
+		const args = parseEvaluationArgs([
+			'--suite',
+			'binary-checks',
+			'--checks',
+			' has_nodes , has_trigger ',
+		]);
+		expect(args.checks).toEqual(['has_nodes', 'has_trigger']);
+	});
+
+	it('throws when --checks is used with non-binary-checks suite', () => {
+		expect(() => parseEvaluationArgs(['--suite', 'llm-judge', '--checks', 'has_nodes'])).toThrow(
+			'`--checks` is only supported for `--suite binary-checks`',
+		);
+	});
+
+	describe('--webhook-secret', () => {
+		it('parses valid webhook secret', () => {
+			const args = parseEvaluationArgs(['--webhook-secret', 'my-secure-secret-key-1234567890']);
+			expect(args.webhookSecret).toBe('my-secure-secret-key-1234567890');
+		});
+
+		it('parses webhook secret with inline = syntax', () => {
+			const args = parseEvaluationArgs(['--webhook-secret=another-secret-key-12345678']);
+			expect(args.webhookSecret).toBe('another-secret-key-12345678');
+		});
+
+		it('rejects secret shorter than 16 characters', () => {
+			expect(() => parseEvaluationArgs(['--webhook-secret', 'short'])).toThrow();
+		});
+
+		it('allows webhook secret to be undefined when not provided', () => {
+			const args = parseEvaluationArgs([]);
+			expect(args.webhookSecret).toBeUndefined();
+		});
+
+		it('can be combined with webhook URL', () => {
+			const args = parseEvaluationArgs([
+				'--webhook-url',
+				'https://example.com/webhook',
+				'--webhook-secret',
+				'my-secure-secret-key-1234567890',
+			]);
+			expect(args.webhookUrl).toBe('https://example.com/webhook');
+			expect(args.webhookSecret).toBe('my-secure-secret-key-1234567890');
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/cli.test.ts

@@ -0,0 +1,619 @@
+/**
+ * Tests for V2 CLI entry point.
+ *
+ * These tests mock all external dependencies and verify that
+ * the CLI correctly orchestrates evaluation runs.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+import type { Client } from 'langsmith/client';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+// Store mocks for dependencies
+const mockParseEvaluationArgs = jest.fn();
+const mockArgsToStageModels = jest.fn();
+const mockSetupTestEnvironment = jest.fn();
+const mockCreateAgent = jest.fn();
+const mockGenerateRunId = jest.fn();
+const mockIsWorkflowStateValues = jest.fn();
+const mockLoadTestCasesFromCsv = jest.fn();
+const mockConsumeGenerator = jest.fn();
+const mockGetChatPayload = jest.fn();
+const mockRunEvaluation = jest.fn();
+const mockCreateConsoleLifecycle = jest.fn();
+const mockCreateLLMJudgeEvaluator = jest.fn();
+const mockCreateProgrammaticEvaluator = jest.fn();
+const mockCreatePairwiseEvaluator = jest.fn();
+const mockCreateExecutionEvaluator = jest.fn();
+const mockSendWebhookNotification = jest.fn();
+
+// Mock all external modules
+jest.mock('../cli/argument-parser', () => ({
+	parseEvaluationArgs: (): unknown => mockParseEvaluationArgs(),
+	argsToStageModels: (...args: unknown[]): unknown => mockArgsToStageModels(...args),
+	getDefaultDatasetName: (suite: unknown): unknown =>
+		suite === 'pairwise' ? 'notion-pairwise-workflows' : 'workflow-builder-canvas-prompts',
+	getDefaultExperimentName: (suite: unknown): unknown =>
+		suite === 'pairwise' ? 'pairwise-evals' : 'workflow-builder-evaluation',
+}));
+
+jest.mock('../support/environment', () => ({
+	setupTestEnvironment: (): unknown => mockSetupTestEnvironment(),
+	createAgent: (...args: unknown[]): unknown => mockCreateAgent(...args),
+	resolveNodesBasePath: (): string => '/mock/nodes-base',
+}));
+
+jest.mock('../langsmith/types', () => ({
+	generateRunId: (): unknown => mockGenerateRunId(),
+	isWorkflowStateValues: (...args: unknown[]): unknown => mockIsWorkflowStateValues(...args),
+}));
+
+jest.mock('../cli/csv-prompt-loader', () => ({
+	loadTestCasesFromCsv: (...args: unknown[]): unknown => mockLoadTestCasesFromCsv(...args),
+	loadDefaultTestCases: () => [
+		{ id: 'test-case-1', prompt: 'Create a workflow that sends a daily email summary' },
+	],
+	getDefaultTestCaseIds: () => ['test-case-1'],
+}));
+
+jest.mock('../cli/webhook', () => ({
+	sendWebhookNotification: (...args: unknown[]): unknown => mockSendWebhookNotification(...args),
+}));
+
+jest.mock('../harness/evaluation-helpers', () => ({
+	consumeGenerator: (...args: unknown[]): unknown => mockConsumeGenerator(...args),
+	getChatPayload: (...args: unknown[]): unknown => mockGetChatPayload(...args),
+	createWorkflowGenerator: () =>
+		jest.fn().mockResolvedValue({ name: 'Test', nodes: [], connections: {} }),
+}));
+
+jest.mock('../lifecycles/introspection-analysis', () => ({
+	createIntrospectionAnalysisLifecycle: () => ({}),
+}));
+
+jest.mock('../index', () => ({
+	runEvaluation: (...args: unknown[]): unknown => mockRunEvaluation(...args),
+	createConsoleLifecycle: (...args: unknown[]): unknown => mockCreateConsoleLifecycle(...args),
+	mergeLifecycles: (...lifecycles: unknown[]): unknown => {
+		// Simple merge implementation for tests - just return the first non-empty lifecycle
+		const valid = (lifecycles as Array<Record<string, unknown> | undefined>).filter(
+			(lc) => lc !== undefined,
+		);
+		if (valid.length === 0) return {};
+		// Return a merged object
+		return Object.assign({}, ...valid);
+	},
+	createLLMJudgeEvaluator: (...args: unknown[]): unknown => mockCreateLLMJudgeEvaluator(...args),
+	createProgrammaticEvaluator: (...args: unknown[]): unknown =>
+		mockCreateProgrammaticEvaluator(...args),
+	createPairwiseEvaluator: (...args: unknown[]): unknown => mockCreatePairwiseEvaluator(...args),
+	createSimilarityEvaluator: () => ({ name: 'similarity', evaluate: jest.fn() }),
+	createExecutionEvaluator: (...args: unknown[]): unknown => mockCreateExecutionEvaluator(...args),
+}));
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create default args */
+function createMockArgs(overrides: Record<string, unknown> = {}) {
+	return {
+		suite: 'llm-judge',
+		backend: 'local',
+		verbose: false,
+		timeoutMs: 60_000,
+		datasetName: undefined,
+		prompt: undefined,
+		testCase: undefined,
+		promptsCsv: undefined,
+		maxExamples: undefined,
+		dos: undefined,
+		donts: undefined,
+		numJudges: 3,
+		experimentName: undefined,
+		repetitions: 1,
+		concurrency: 4,
+		featureFlags: undefined,
+		...overrides,
+	};
+}
+
+/** Helper to create mock environment */
+function createMockEnvironment() {
+	const mockLlm = mock<BaseChatModel>();
+	return {
+		parsedNodeTypes: [] as INodeTypeDescription[],
+		llms: {
+			default: mockLlm,
+			supervisor: mockLlm,
+			responder: mockLlm,
+			discovery: mockLlm,
+			builder: mockLlm,
+			parameterUpdater: mockLlm,
+			judge: mockLlm,
+		},
+		lsClient: mock<Client>(),
+	};
+}
+
+/** Helper to create mock agent */
+function createMockAgentInstance(workflowJSON: SimpleWorkflow = createMockWorkflow()) {
+	return {
+		chat: jest.fn().mockReturnValue((async function* () {})()),
+		getState: jest.fn().mockResolvedValue({
+			values: {
+				workflowJSON,
+				messages: [],
+			},
+		}),
+	};
+}
+
+/** Helper to create mock run summary */
+function createMockSummary(overrides: Record<string, unknown> = {}) {
+	return {
+		totalExamples: 10,
+		passed: 8,
+		failed: 2,
+		errors: 0,
+		averageScore: 0.85,
+		totalDurationMs: 5000,
+		...overrides,
+	};
+}
+
+describe('CLI', () => {
+	// Mock process.exit to prevent test termination
+	let mockExit: jest.SpyInstance;
+	const originalEnv = process.env;
+
+	beforeEach(() => {
+		jest.clearAllMocks();
+		mockExit = jest.spyOn(process, 'exit').mockImplementation((code) => {
+			throw new Error(`process.exit(${code})`);
+		});
+
+		// Reset environment
+		process.env = { ...originalEnv };
+
+		// Setup default mocks
+		mockParseEvaluationArgs.mockReturnValue(createMockArgs());
+		mockArgsToStageModels.mockReturnValue({ default: 'claude-sonnet-4.5' });
+		mockSetupTestEnvironment.mockResolvedValue(createMockEnvironment());
+		mockCreateAgent.mockReturnValue(createMockAgentInstance());
+		mockGenerateRunId.mockReturnValue('test-run-id');
+		mockIsWorkflowStateValues.mockReturnValue(true);
+		mockConsumeGenerator.mockResolvedValue(undefined);
+		mockGetChatPayload.mockReturnValue({});
+		mockRunEvaluation.mockResolvedValue(createMockSummary());
+		mockCreateConsoleLifecycle.mockReturnValue({});
+		mockCreateLLMJudgeEvaluator.mockReturnValue({ name: 'llm-judge', evaluate: jest.fn() });
+		mockCreateProgrammaticEvaluator.mockReturnValue({ name: 'programmatic', evaluate: jest.fn() });
+		mockCreatePairwiseEvaluator.mockReturnValue({ name: 'pairwise', evaluate: jest.fn() });
+		mockCreateExecutionEvaluator.mockReturnValue({ name: 'execution', evaluate: jest.fn() });
+	});
+
+	afterEach(() => {
+		mockExit.mockRestore();
+		process.env = originalEnv;
+	});
+
+	describe('runV2Evaluation()', () => {
+		describe('loadTestCases', () => {
+			it('should load test cases from CSV when promptsCsv is set', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ promptsCsv: '/path/to/prompts.csv' }),
+				);
+				mockLoadTestCasesFromCsv.mockReturnValue([
+					{ prompt: 'CSV prompt 1', id: '1' },
+					{ prompt: 'CSV prompt 2', id: '2' },
+				]);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockLoadTestCasesFromCsv).toHaveBeenCalledWith('/path/to/prompts.csv');
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: [
+							{ prompt: 'CSV prompt 1', id: '1' },
+							{ prompt: 'CSV prompt 2', id: '2' },
+						],
+					}),
+				);
+			});
+
+			it('should create single test case when prompt is set', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({
+						prompt: 'Create a workflow',
+						dos: 'Use Slack',
+						donts: 'No HTTP',
+					}),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: [
+							{
+								prompt: 'Create a workflow',
+								context: { dos: 'Use Slack', donts: 'No HTTP' },
+							},
+						],
+					}),
+				);
+			});
+
+			it('should use default test case when no prompt source specified', async () => {
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: [
+							{ id: 'test-case-1', prompt: 'Create a workflow that sends a daily email summary' },
+						],
+					}),
+				);
+			});
+		});
+
+		describe('mode selection', () => {
+			it('should create LLM-judge + programmatic evaluators for llm-judge suite (local)', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ suite: 'llm-judge', backend: 'local' }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockCreateLLMJudgeEvaluator).toHaveBeenCalled();
+				expect(mockCreateProgrammaticEvaluator).toHaveBeenCalled();
+				expect(mockCreatePairwiseEvaluator).not.toHaveBeenCalled();
+			});
+
+			it('should create LLM-judge + programmatic evaluators for llm-judge suite (langsmith)', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ suite: 'llm-judge', backend: 'langsmith' }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockCreateLLMJudgeEvaluator).toHaveBeenCalled();
+				expect(mockCreateProgrammaticEvaluator).toHaveBeenCalled();
+				expect(mockCreatePairwiseEvaluator).not.toHaveBeenCalled();
+			});
+
+			it('should create pairwise + programmatic evaluators for pairwise suite (local)', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ suite: 'pairwise', backend: 'local', numJudges: 5 }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockCreatePairwiseEvaluator).toHaveBeenCalled();
+				// Verify numJudges was passed correctly
+				const callArgs = mockCreatePairwiseEvaluator.mock.calls[0] as [
+					unknown,
+					{ numJudges: number },
+				];
+				expect(callArgs[1]).toEqual({ numJudges: 5 });
+				expect(mockCreateProgrammaticEvaluator).toHaveBeenCalled();
+				expect(mockCreateLLMJudgeEvaluator).not.toHaveBeenCalled();
+			});
+
+			it('should create pairwise + programmatic evaluators for pairwise suite (langsmith)', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ suite: 'pairwise', backend: 'langsmith' }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockCreatePairwiseEvaluator).toHaveBeenCalled();
+				expect(mockCreateProgrammaticEvaluator).toHaveBeenCalled();
+				expect(mockCreateLLMJudgeEvaluator).not.toHaveBeenCalled();
+			});
+		});
+
+		describe('config building', () => {
+			it('should use local mode for backend=local', async () => {
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs({ backend: 'local' }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						mode: 'local',
+					}),
+				);
+			});
+
+			it('should use langsmith mode for backend=langsmith', async () => {
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs({ backend: 'langsmith' }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						mode: 'langsmith',
+						langsmithOptions: expect.objectContaining({
+							experimentName: 'workflow-builder-evaluation',
+						}),
+					}),
+				);
+			});
+
+			it('should use datasetName from args for langsmith mode', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ backend: 'langsmith', datasetName: 'custom-dataset' }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: 'custom-dataset',
+					}),
+				);
+			});
+
+			it('should fall back to default dataset name when env var not set', async () => {
+				delete process.env.LANGSMITH_DATASET_NAME;
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs({ backend: 'langsmith' }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: 'workflow-builder-canvas-prompts',
+					}),
+				);
+			});
+
+			it('should include langsmithOptions with custom experiment name', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({
+						backend: 'langsmith',
+						experimentName: 'my-experiment',
+						repetitions: 3,
+						concurrency: 8,
+					}),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						langsmithOptions: expect.objectContaining({
+							experimentName: 'my-experiment',
+							repetitions: 3,
+							concurrency: 8,
+						}),
+					}),
+				);
+			});
+		});
+
+		describe('exit codes', () => {
+			it('should always exit with 0 on successful completion (pass/fail is informational)', async () => {
+				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 10, passed: 7 }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+			});
+
+			it('should exit with 0 even when pass rate is low', async () => {
+				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 10, passed: 5 }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+			});
+
+			it('should exit with 0 even when no examples', async () => {
+				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 0, passed: 0 }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+			});
+		});
+
+		describe('workflow generator', () => {
+			it('should create agent with correct config', async () => {
+				const env = createMockEnvironment();
+				mockSetupTestEnvironment.mockResolvedValue(env);
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ featureFlags: { testFlag: true } }),
+				);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				// Verify generateWorkflow was passed to config
+				expect(mockRunEvaluation).toHaveBeenCalledWith(
+					expect.objectContaining({
+						generateWorkflow: expect.any(Function),
+					}),
+				);
+			});
+
+			it('should setup test environment', async () => {
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockSetupTestEnvironment).toHaveBeenCalled();
+			});
+
+			it('should create console lifecycle with verbose option', async () => {
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs({ verbose: true }));
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit');
+
+				expect(mockCreateConsoleLifecycle).toHaveBeenCalledWith(
+					expect.objectContaining({ verbose: true }),
+				);
+			});
+		});
+
+		describe('webhook notification', () => {
+			it('should send webhook notification when webhookUrl is provided', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({
+						webhookUrl: 'https://example.com/webhook',
+						suite: 'llm-judge',
+						backend: 'local',
+					}),
+				);
+				mockRunEvaluation.mockResolvedValue(createMockSummary());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).toHaveBeenCalledTimes(1);
+				expect(mockSendWebhookNotification).toHaveBeenCalledWith(
+					expect.objectContaining({
+						webhookUrl: 'https://example.com/webhook',
+						suite: 'llm-judge',
+						dataset: 'local-dataset',
+					}),
+				);
+			});
+
+			it('should NOT send webhook notification when webhookUrl is not provided', async () => {
+				mockParseEvaluationArgs.mockReturnValue(createMockArgs({ webhookUrl: undefined }));
+				mockRunEvaluation.mockResolvedValue(createMockSummary());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).not.toHaveBeenCalled();
+			});
+
+			it('should use dataset name from args for langsmith backend', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({
+						webhookUrl: 'https://example.com/webhook',
+						backend: 'langsmith',
+						experimentName: 'my-custom-experiment',
+						datasetName: 'my-custom-dataset',
+						suite: 'pairwise',
+					}),
+				);
+				mockRunEvaluation.mockResolvedValue(createMockSummary());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: 'my-custom-dataset',
+						suite: 'pairwise',
+					}),
+				);
+			});
+
+			it('should use default dataset name for langsmith backend when not specified', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({
+						webhookUrl: 'https://example.com/webhook',
+						backend: 'langsmith',
+						experimentName: undefined,
+						datasetName: undefined,
+						suite: 'llm-judge',
+					}),
+				);
+				mockRunEvaluation.mockResolvedValue(createMockSummary());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).toHaveBeenCalledWith(
+					expect.objectContaining({
+						dataset: 'workflow-builder-canvas-prompts',
+					}),
+				);
+			});
+
+			it('should pass run summary to webhook notification', async () => {
+				const summary = createMockSummary({
+					totalExamples: 25,
+					passed: 20,
+					failed: 5,
+					errors: 0,
+					averageScore: 0.92,
+					totalDurationMs: 12000,
+				});
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ webhookUrl: 'https://example.com/webhook' }),
+				);
+				mockRunEvaluation.mockResolvedValue(summary);
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).toHaveBeenCalledWith(
+					expect.objectContaining({
+						summary,
+					}),
+				);
+			});
+
+			it('should include CI metadata in webhook notification', async () => {
+				mockParseEvaluationArgs.mockReturnValue(
+					createMockArgs({ webhookUrl: 'https://example.com/webhook' }),
+				);
+				mockRunEvaluation.mockResolvedValue(createMockSummary());
+
+				const { runV2Evaluation } = await import('../cli');
+
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
+
+				expect(mockSendWebhookNotification).toHaveBeenCalledWith(
+					expect.objectContaining({
+						metadata: expect.any(Object),
+					}),
+				);
+			});
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/csv-prompt-loader.test.ts

@@ -0,0 +1,174 @@
+/**
+ * Tests for CSV prompt loader.
+ */
+
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import { loadTestCasesFromCsv } from '../cli/csv-prompt-loader';
+
+describe('csv-prompt-loader', () => {
+	const tempDirs: string[] = [];
+
+	function writeTempCsv(filename: string, content: string): string {
+		const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-csv-'));
+		tempDirs.push(dir);
+		const csvPath = path.join(dir, filename);
+		fs.writeFileSync(csvPath, content, 'utf8');
+		return csvPath;
+	}
+
+	afterEach(() => {
+		while (tempDirs.length > 0) {
+			const dir = tempDirs.pop();
+			if (dir && fs.existsSync(dir)) {
+				fs.rmSync(dir, { recursive: true, force: true });
+			}
+		}
+	});
+
+	it('should load dos/donts into context when present', () => {
+		const csvPath = writeTempCsv(
+			'pairwise.csv',
+			'id,prompt,dos,donts\npw-1,"Create a workflow","Must use Notion","No HTTP Request"\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([
+			{
+				id: 'pw-1',
+				prompt: 'Create a workflow',
+				context: { dos: 'Must use Notion', donts: 'No HTTP Request' },
+			},
+		]);
+	});
+
+	it('should load prompts without context when dos/donts are absent', () => {
+		const csvPath = writeTempCsv('llm.csv', 'id,prompt\nllm-1,"Create a workflow"\n');
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]);
+	});
+
+	it('should support header column re-ordering', () => {
+		const csvPath = writeTempCsv(
+			'reorder.csv',
+			'prompt,id,donts,dos\n"Create a workflow","pw-1","No HTTP Request","Must use Notion"\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([
+			{
+				id: 'pw-1',
+				prompt: 'Create a workflow',
+				context: { dos: 'Must use Notion', donts: 'No HTTP Request' },
+			},
+		]);
+	});
+
+	it('should support do/dont header aliases', () => {
+		const csvPath = writeTempCsv(
+			'aliases.csv',
+			'id,prompt,do,dont\npw-1,"Create a workflow","Must use Notion","No HTTP Request"\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([
+			{
+				id: 'pw-1',
+				prompt: 'Create a workflow',
+				context: { dos: 'Must use Notion', donts: 'No HTTP Request' },
+			},
+		]);
+	});
+
+	it('should not create context when dos/donts columns are present but empty', () => {
+		const csvPath = writeTempCsv(
+			'empty-context.csv',
+			'id,prompt,dos,donts\npw-1,"Create a workflow",,\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([{ id: 'pw-1', prompt: 'Create a workflow' }]);
+	});
+
+	it('should allow headerless CSV (treat first column as prompt)', () => {
+		const csvPath = writeTempCsv('no-header.csv', '"Create a workflow"\n"Second prompt"\n');
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([
+			{ id: 'csv-case-1', prompt: 'Create a workflow' },
+			{ id: 'csv-case-2', prompt: 'Second prompt' },
+		]);
+	});
+
+	it('should ignore rows with empty prompts', () => {
+		const csvPath = writeTempCsv('empty-rows.csv', 'id,prompt\nrow-1,\nrow-2,"Valid prompt"\n');
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([{ id: 'row-2', prompt: 'Valid prompt' }]);
+	});
+
+	it('should handle UTF-8 BOM', () => {
+		const csvPath = writeTempCsv('bom.csv', '\ufeffid,prompt\nllm-1,"Create a workflow"\n');
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]);
+	});
+
+	it('should resolve relative paths from process.cwd()', () => {
+		const csvPath = writeTempCsv('relative.csv', 'id,prompt\nllm-1,"Create a workflow"\n');
+		const relativePath = path.relative(process.cwd(), csvPath);
+
+		const testCases = loadTestCasesFromCsv(relativePath);
+		expect(testCases).toEqual([{ id: 'llm-1', prompt: 'Create a workflow' }]);
+	});
+
+	it('should throw when file does not exist', () => {
+		expect(() => loadTestCasesFromCsv('/definitely-not-a-real-path.csv')).toThrow(
+			/CSV file not found/,
+		);
+	});
+
+	it('should throw when CSV is empty', () => {
+		const csvPath = writeTempCsv('empty.csv', '');
+		expect(() => loadTestCasesFromCsv(csvPath)).toThrow('The provided CSV file is empty');
+	});
+
+	it('should throw when no valid prompts exist', () => {
+		const csvPath = writeTempCsv('no-prompts.csv', 'id,prompt\nrow-1,\nrow-2,"  "\n');
+		expect(() => loadTestCasesFromCsv(csvPath)).toThrow(
+			'No valid prompts found in the provided CSV file',
+		);
+	});
+
+	it('should parse annotations column as JSON into context', () => {
+		const csvPath = writeTempCsv(
+			'annotations.csv',
+			'id,prompt,annotations\nann-1,"Create a workflow","{""code_necessary"":true}"\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([
+			{
+				id: 'ann-1',
+				prompt: 'Create a workflow',
+				context: { annotations: { code_necessary: true } },
+			},
+		]);
+	});
+
+	it('should skip invalid JSON in annotations column with warning', () => {
+		const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
+		const csvPath = writeTempCsv(
+			'bad-annotations.csv',
+			'id,prompt,annotations\nbad-1,"Create a workflow","not-json"\n',
+		);
+
+		const testCases = loadTestCasesFromCsv(csvPath);
+		expect(testCases).toEqual([{ id: 'bad-1', prompt: 'Create a workflow', context: {} }]);
+		expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('invalid JSON'));
+		warnSpy.mockRestore();
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluation-helpers.test.ts

@@ -0,0 +1,143 @@
+/**
+ * Tests for evaluation helper utilities.
+ */
+
+import pLimit from 'p-limit';
+
+import { withTimeout, extractSubgraphMetrics } from '../harness/evaluation-helpers';
+
+describe('evaluation-helpers', () => {
+	describe('withTimeout()', () => {
+		it('should allow p-limit slot to be released when timeout triggers (best-effort)', async () => {
+			jest.useFakeTimers();
+			const limit = pLimit(1);
+			const started: string[] = [];
+
+			const never = new Promise<void>(() => {
+				// never resolves
+			});
+
+			const p1 = limit(async () => {
+				started.push('p1');
+				await withTimeout({ promise: never, timeoutMs: 10, label: 'p1' });
+			}).catch(() => {
+				// expected timeout
+			});
+
+			// Give p1 a chance to start.
+			await Promise.resolve();
+
+			const p2 = limit(async () => {
+				started.push('p2');
+			});
+
+			jest.advanceTimersByTime(11);
+			await Promise.resolve();
+			await Promise.resolve();
+
+			await expect(p2).resolves.toBeUndefined();
+			expect(started).toEqual(['p1', 'p2']);
+
+			await p1;
+			jest.useRealTimers();
+		});
+	});
+
+	describe('extractSubgraphMetrics()', () => {
+		it('should return empty object when both inputs are undefined', () => {
+			const result = extractSubgraphMetrics(undefined, undefined);
+			expect(result).toEqual({});
+		});
+
+		it.each([
+			{ nodeCount: 5, expected: { nodeCount: 5 } },
+			{ nodeCount: 0, expected: { nodeCount: 0 } },
+		])('should include nodeCount of $nodeCount when provided', ({ nodeCount, expected }) => {
+			const result = extractSubgraphMetrics(undefined, nodeCount);
+			expect(result).toEqual(expected);
+		});
+
+		it.each([
+			{
+				phase: 'discovery' as const,
+				metricKey: 'discoveryDurationMs',
+				startTs: 1000,
+				endTs: 1500,
+				expectedDuration: 500,
+			},
+			{
+				phase: 'builder' as const,
+				metricKey: 'builderDurationMs',
+				startTs: 2000,
+				endTs: 3500,
+				expectedDuration: 1500,
+			},
+			{
+				phase: 'responder' as const,
+				metricKey: 'responderDurationMs',
+				startTs: 5000,
+				endTs: 5800,
+				expectedDuration: 800,
+			},
+		])(
+			'should calculate $phase duration from in_progress to completed',
+			({ phase, metricKey, startTs, endTs, expectedDuration }) => {
+				const coordinationLog = [
+					{ phase, status: 'in_progress' as const, timestamp: startTs },
+					{ phase, status: 'completed' as const, timestamp: endTs },
+				];
+				const result = extractSubgraphMetrics(coordinationLog, undefined);
+				expect(result).toEqual({ [metricKey]: expectedDuration });
+			},
+		);
+
+		it('should calculate duration from first to last entry when no in_progress status', () => {
+			const coordinationLog = [
+				{ phase: 'discovery' as const, status: 'completed' as const, timestamp: 1000 },
+				{ phase: 'discovery' as const, status: 'completed' as const, timestamp: 1800 },
+			];
+			const result = extractSubgraphMetrics(coordinationLog, undefined);
+			expect(result).toEqual({ discoveryDurationMs: 800 });
+		});
+
+		it('should ignore state_management phase', () => {
+			const coordinationLog = [
+				{ phase: 'state_management' as const, status: 'in_progress' as const, timestamp: 500 },
+				{ phase: 'state_management' as const, status: 'completed' as const, timestamp: 600 },
+			];
+			const result = extractSubgraphMetrics(coordinationLog, undefined);
+			expect(result).toEqual({});
+		});
+
+		it('should return undefined for phase with only one entry', () => {
+			const coordinationLog = [
+				{ phase: 'discovery' as const, status: 'completed' as const, timestamp: 1000 },
+			];
+			const result = extractSubgraphMetrics(coordinationLog, undefined);
+			expect(result).toEqual({});
+		});
+
+		it('should return empty object for empty coordination log', () => {
+			const result = extractSubgraphMetrics([], undefined);
+			expect(result).toEqual({});
+		});
+
+		it('should calculate all phase durations together with nodeCount', () => {
+			const coordinationLog = [
+				{ phase: 'discovery' as const, status: 'in_progress' as const, timestamp: 1000 },
+				{ phase: 'discovery' as const, status: 'completed' as const, timestamp: 1500 },
+				{ phase: 'builder' as const, status: 'in_progress' as const, timestamp: 2000 },
+				{ phase: 'builder' as const, status: 'completed' as const, timestamp: 4000 },
+				{ phase: 'responder' as const, status: 'in_progress' as const, timestamp: 4500 },
+				{ phase: 'responder' as const, status: 'completed' as const, timestamp: 5000 },
+			];
+			const result = extractSubgraphMetrics(coordinationLog, 8);
+			expect(result).toEqual({
+				nodeCount: 8,
+				discoveryDurationMs: 500,
+				builderDurationMs: 2000,
+				responderDurationMs: 500,
+			});
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluators/llm-judge.test.ts

@@ -0,0 +1,192 @@
+/**
+ * Tests for LLM-as-judge evaluator factory.
+ *
+ * These tests mock the underlying evaluateWorkflow function and verify
+ * that the factory correctly wraps it and transforms the results.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+// Store original module
+const mockEvaluateWorkflow = jest.fn();
+
+// Mock the evaluateWorkflow function
+jest.mock('../../evaluators/llm-judge/workflow-evaluator', () => ({
+	evaluateWorkflow: (...args: unknown[]): unknown => mockEvaluateWorkflow(...args),
+}));
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create a mock evaluation result */
+function createMockEvalResult(overrides: Record<string, unknown> = {}) {
+	return {
+		functionality: { score: 0.9, violations: [] },
+		connections: { score: 1.0, violations: [] },
+		expressions: { score: 0.8, violations: [] },
+		nodeConfiguration: { score: 0.85, violations: [] },
+		efficiency: {
+			score: 0.95,
+			violations: [],
+			redundancyScore: 1,
+			pathOptimization: 0.9,
+			nodeCountEfficiency: 0.95,
+		},
+		dataFlow: { score: 0.9, violations: [] },
+		maintainability: {
+			score: 0.88,
+			violations: [],
+			nodeNamingQuality: 0.9,
+			workflowOrganization: 0.85,
+			modularity: 0.9,
+		},
+		bestPractices: { score: 0.82, violations: [] },
+		overallScore: 0.9,
+		summary: 'Good workflow',
+		...overrides,
+	};
+}
+
+describe('LLM-Judge Evaluator', () => {
+	let mockLlm: BaseChatModel;
+	let mockNodeTypes: INodeTypeDescription[];
+
+	beforeEach(() => {
+		jest.clearAllMocks();
+		mockLlm = mock<BaseChatModel>();
+		mockNodeTypes = [];
+	});
+
+	describe('createLLMJudgeEvaluator()', () => {
+		it('should create an evaluator with correct name', async () => {
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			expect(evaluator.name).toBe('llm-judge');
+		});
+
+		it('should call evaluateWorkflow with workflow and prompt', async () => {
+			mockEvaluateWorkflow.mockResolvedValue(createMockEvalResult());
+
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const context = { prompt: 'Create a test workflow' };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockEvaluateWorkflow).toHaveBeenCalledWith(mockLlm, {
+				userPrompt: 'Create a test workflow',
+				generatedWorkflow: workflow,
+			});
+		});
+
+		it('should return feedback array with all category scores', async () => {
+			mockEvaluateWorkflow.mockResolvedValue(createMockEvalResult());
+
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			// Should have feedback for each category
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'functionality' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'connections' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'expressions' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'nodeConfiguration' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'efficiency' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'dataFlow' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'maintainability' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'bestPractices' }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'llm-judge', metric: 'overallScore' }),
+			);
+		});
+
+		it('should include violations in feedback comments', async () => {
+			mockEvaluateWorkflow.mockResolvedValue(
+				createMockEvalResult({
+					functionality: {
+						score: 0.5,
+						violations: [
+							{ type: 'critical', description: 'Missing HTTP node', pointsDeducted: 0.3 },
+							{ type: 'major', description: 'Incorrect branching', pointsDeducted: 0.2 },
+						],
+					},
+				}),
+			);
+
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const funcFeedback = feedback.find(
+				(f) => f.evaluator === 'llm-judge' && f.metric === 'functionality',
+			);
+			expect(funcFeedback?.comment).toContain('Missing HTTP node');
+		});
+
+		it('should include bestPractices violations in comments', async () => {
+			mockEvaluateWorkflow.mockResolvedValue(
+				createMockEvalResult({
+					bestPractices: {
+						score: 0.4,
+						violations: [
+							{ type: 'major', description: 'Missing rate limiting', pointsDeducted: 0.2 },
+						],
+					},
+				}),
+			);
+
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			const feedback = await evaluator.evaluate(createMockWorkflow(), { prompt: 'Test' });
+			const bpFeedback = feedback.find(
+				(f) => f.evaluator === 'llm-judge' && f.metric === 'bestPractices',
+			);
+
+			expect(bpFeedback?.comment).toContain('Missing rate limiting');
+		});
+
+		it('should handle evaluation errors gracefully', async () => {
+			mockEvaluateWorkflow.mockRejectedValue(new Error('LLM API error'));
+
+			const { createLLMJudgeEvaluator } = await import('../../evaluators/llm-judge');
+			const evaluator = createLLMJudgeEvaluator(mockLlm, mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+
+			// Should throw - let the runner handle errors
+			await expect(evaluator.evaluate(workflow, { prompt: 'Test' })).rejects.toThrow(
+				'LLM API error',
+			);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluators/pairwise.test.ts

@@ -0,0 +1,290 @@
+/**
+ * Tests for pairwise evaluator factory.
+ *
+ * These tests mock the underlying runJudgePanel function and verify
+ * that the factory correctly wraps it and transforms the results.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { PAIRWISE_METRICS } from '../../evaluators/pairwise/metrics';
+
+// Store mock for runJudgePanel
+const mockRunJudgePanel = jest.fn();
+
+// Mock the judge panel module
+jest.mock('../../evaluators/pairwise/judge-panel', () => ({
+	runJudgePanel: (...args: unknown[]): unknown => mockRunJudgePanel(...args),
+}));
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create a mock judge panel result */
+function createMockPanelResult(
+	overrides: Partial<{
+		primaryPasses: number;
+		majorityPass: boolean;
+		avgDiagnosticScore: number;
+		judgeResults: Array<{
+			primaryPass: boolean;
+			diagnosticScore: number;
+			violations: Array<{ rule: string; justification: string }>;
+			passes: Array<{ rule: string; justification: string }>;
+		}>;
+	}> = {},
+) {
+	return {
+		primaryPasses: 2,
+		majorityPass: true,
+		avgDiagnosticScore: 0.8,
+		judgeResults: [
+			{
+				primaryPass: true,
+				diagnosticScore: 0.9,
+				violations: [],
+				passes: [{ rule: 'Has trigger', justification: 'Gmail trigger exists' }],
+			},
+			{
+				primaryPass: true,
+				diagnosticScore: 0.8,
+				violations: [],
+				passes: [{ rule: 'Has trigger', justification: 'Trigger present' }],
+			},
+			{
+				primaryPass: false,
+				diagnosticScore: 0.7,
+				violations: [{ rule: 'Missing action', justification: 'No Slack node found' }],
+				passes: [],
+			},
+		],
+		...overrides,
+	};
+}
+
+describe('Pairwise Evaluator', () => {
+	let mockLlm: BaseChatModel;
+	type PairwiseFeedback = { evaluator: string; metric: string; score: number; comment?: string };
+	const findFeedback = (feedback: PairwiseFeedback[], metric: string) =>
+		feedback.find((f) => f.evaluator === 'pairwise' && f.metric === metric);
+
+	beforeEach(() => {
+		jest.clearAllMocks();
+		mockLlm = mock<BaseChatModel>();
+	});
+
+	describe('createPairwiseEvaluator()', () => {
+		it('should create an evaluator with correct name', async () => {
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			expect(evaluator.name).toBe('pairwise');
+		});
+
+		it('should call runJudgePanel with workflow and criteria', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const context = { prompt: 'Test prompt', dos: 'Use Slack', donts: 'No HTTP requests' };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockRunJudgePanel).toHaveBeenCalledWith(
+				mockLlm,
+				workflow,
+				{ dos: 'Use Slack', donts: 'No HTTP requests' },
+				3, // default number of judges
+				expect.any(Object),
+			);
+		});
+
+		it('should use custom number of judges', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 5 });
+
+			const workflow = createMockWorkflow();
+			await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			expect(mockRunJudgePanel).toHaveBeenCalledWith(
+				mockLlm,
+				workflow,
+				expect.any(Object),
+				5,
+				expect.any(Object),
+			);
+		});
+
+		it('should pass through empty criteria when context has no dos/donts', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			expect(mockRunJudgePanel).toHaveBeenCalledWith(
+				mockLlm,
+				workflow,
+				{
+					dos: undefined,
+					donts: undefined,
+				},
+				3,
+				expect.any(Object),
+			);
+		});
+
+		it('should return feedback with majority pass result', async () => {
+			mockRunJudgePanel.mockResolvedValue(
+				createMockPanelResult({ majorityPass: true, primaryPasses: 2 }),
+			);
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			const majorityFeedback = findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_PRIMARY);
+			expect(majorityFeedback).toEqual({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_PRIMARY,
+				score: 1,
+				kind: 'score',
+				comment: '2/3 judges passed',
+			});
+		});
+
+		it('should return feedback with diagnostic score', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult({ avgDiagnosticScore: 0.85 }));
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			const diagnosticFeedback = findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC);
+			expect(diagnosticFeedback).toEqual({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC,
+				score: 0.85,
+				kind: 'metric',
+			});
+		});
+
+		it('should return feedback for each judge', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'pairwise', metric: 'judge1', score: 1 }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'pairwise', metric: 'judge2', score: 1 }),
+			);
+			expect(feedback).toContainEqual(
+				expect.objectContaining({ evaluator: 'pairwise', metric: 'judge3', score: 0 }),
+			);
+		});
+
+		it('should include violations in judge feedback comments', async () => {
+			mockRunJudgePanel.mockResolvedValue(
+				createMockPanelResult({
+					judgeResults: [
+						{
+							primaryPass: false,
+							diagnosticScore: 0.5,
+							violations: [
+								{ rule: 'Has Slack', justification: 'Missing Slack node for notifications' },
+								{ rule: 'Has trigger', justification: 'No trigger node found' },
+							],
+							passes: [],
+						},
+					],
+				}),
+			);
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			const judgeFeedback = findFeedback(feedback, 'judge1');
+			// Full violation output without truncation
+			expect(judgeFeedback?.comment).toContain('[Has Slack] Missing Slack node for notifications');
+			expect(judgeFeedback?.comment).toContain('[Has trigger] No trigger node found');
+		});
+
+		it('should handle evaluation errors gracefully', async () => {
+			mockRunJudgePanel.mockRejectedValue(new Error('Judge panel failed'));
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+
+			// Should throw - let the runner handle errors
+			await expect(evaluator.evaluate(workflow, { prompt: 'Test prompt' })).rejects.toThrow(
+				'Judge panel failed',
+			);
+		});
+
+		it('should accept criteria with only dos (no donts)', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const context = { prompt: 'Test prompt', dos: 'Use Slack node' };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockRunJudgePanel).toHaveBeenCalledWith(
+				mockLlm,
+				workflow,
+				{ dos: 'Use Slack node', donts: undefined },
+				3,
+				expect.any(Object),
+			);
+		});
+
+		it('should accept criteria with only donts (no dos)', async () => {
+			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
+
+			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
+			const evaluator = createPairwiseEvaluator(mockLlm);
+
+			const workflow = createMockWorkflow();
+			const context = { prompt: 'Test prompt', donts: 'Do not use HTTP Request node' };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockRunJudgePanel).toHaveBeenCalledWith(
+				mockLlm,
+				workflow,
+				{ dos: undefined, donts: 'Do not use HTTP Request node' },
+				3,
+				expect.any(Object),
+			);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluators/programmatic.test.ts

@@ -0,0 +1,267 @@
+/**
+ * Tests for programmatic evaluator factory.
+ *
+ * These tests mock the underlying programmaticEvaluation function and verify
+ * that the factory correctly wraps it and transforms the results.
+ */
+
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+// Store mock for programmaticEvaluation
+const mockProgrammaticEvaluation = jest.fn();
+
+// Mock the programmatic evaluation module
+jest.mock('../../programmatic/programmatic-evaluation', () => ({
+	programmaticEvaluation: (...args: unknown[]): unknown => mockProgrammaticEvaluation(...args),
+}));
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create mock evaluation result */
+function createMockEvaluationResult(
+	overrides: Partial<{
+		overallScore: number;
+		connections: { score: number; violations: Array<{ type: string; description: string }> };
+		nodes: { score: number; violations: Array<{ type: string; description: string }> };
+		trigger: { score: number; violations: Array<{ type: string; description: string }> };
+		agentPrompt: { score: number; violations: Array<{ type: string; description: string }> };
+		tools: { score: number; violations: Array<{ type: string; description: string }> };
+		fromAi: { score: number; violations: Array<{ type: string; description: string }> };
+		credentials: { score: number; violations: Array<{ type: string; description: string }> };
+		graphValidation: { score: number; violations: Array<{ type: string; description: string }> };
+		parameters: { score: number; violations: Array<{ type: string; description: string }> };
+		similarity: { score: number; violations: Array<{ type: string; description: string }> } | null;
+	}> = {},
+) {
+	return {
+		overallScore: 0.85,
+		connections: { score: 1.0, violations: [] },
+		nodes: { score: 1.0, violations: [] },
+		trigger: { score: 1.0, violations: [] },
+		agentPrompt: { score: 0.9, violations: [] },
+		tools: { score: 1.0, violations: [] },
+		fromAi: { score: 0.8, violations: [] },
+		credentials: { score: 1.0, violations: [] },
+		graphValidation: { score: 1.0, violations: [] },
+		parameters: { score: 1.0, violations: [] },
+		similarity: null,
+		...overrides,
+	};
+}
+
+describe('Programmatic Evaluator', () => {
+	const mockNodeTypes: INodeTypeDescription[] = [];
+	type ProgrammaticFeedback = {
+		evaluator: string;
+		metric: string;
+		score: number;
+		comment?: string;
+	};
+	const findFeedback = (feedback: ProgrammaticFeedback[], metric: string) =>
+		feedback.find((f) => f.evaluator === 'programmatic' && f.metric === metric);
+
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	describe('createProgrammaticEvaluator()', () => {
+		it('should create an evaluator with correct name', async () => {
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			expect(evaluator.name).toBe('programmatic');
+		});
+
+		it('should call programmaticEvaluation with workflow and context', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(createMockEvaluationResult());
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const context = { prompt: 'Create a test workflow' };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockProgrammaticEvaluation).toHaveBeenCalledWith(
+				{
+					userPrompt: 'Create a test workflow',
+					generatedWorkflow: workflow,
+					referenceWorkflows: undefined,
+				},
+				mockNodeTypes,
+			);
+		});
+
+		it('should pass reference workflows when provided in context', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(createMockEvaluationResult());
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+			const context = { prompt: 'Test prompt', referenceWorkflows: [referenceWorkflow] };
+
+			await evaluator.evaluate(workflow, context);
+
+			expect(mockProgrammaticEvaluation).toHaveBeenCalledWith(
+				{
+					userPrompt: 'Test prompt',
+					generatedWorkflow: workflow,
+					referenceWorkflows: [referenceWorkflow],
+				},
+				mockNodeTypes,
+			);
+		});
+
+		it('should return feedback with overall score', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(
+				createMockEvaluationResult({ overallScore: 0.92 }),
+			);
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const overallFeedback = findFeedback(feedback, 'overall');
+			expect(overallFeedback).toEqual({
+				evaluator: 'programmatic',
+				metric: 'overall',
+				score: 0.92,
+				kind: 'score',
+			});
+		});
+
+		it('should return feedback for all check categories', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(createMockEvaluationResult());
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const metrics = feedback.filter((f) => f.evaluator === 'programmatic').map((f) => f.metric);
+			expect(metrics).toContain('overall');
+			expect(metrics).toContain('connections');
+			expect(metrics).toContain('trigger');
+			expect(metrics).toContain('agentPrompt');
+			expect(metrics).toContain('tools');
+			expect(metrics).toContain('fromAi');
+		});
+
+		it('should include violations in feedback comments', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(
+				createMockEvaluationResult({
+					connections: {
+						score: 0.5,
+						violations: [
+							{ type: 'disconnected-node', description: 'Node A has no connections' },
+							{ type: 'invalid-connection', description: 'Invalid edge between B and C' },
+						],
+					},
+				}),
+			);
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const connectionsFeedback = findFeedback(feedback, 'connections');
+			expect(connectionsFeedback?.comment).toContain(
+				'[disconnected-node] Node A has no connections',
+			);
+			expect(connectionsFeedback?.comment).toContain(
+				'[invalid-connection] Invalid edge between B and C',
+			);
+		});
+
+		it('should not include comment when no violations', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(
+				createMockEvaluationResult({
+					trigger: { score: 1.0, violations: [] },
+				}),
+			);
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const triggerFeedback = findFeedback(feedback, 'trigger');
+			expect(triggerFeedback?.comment).toBeUndefined();
+		});
+
+		it('should include similarity feedback when reference workflows provided', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(
+				createMockEvaluationResult({
+					similarity: {
+						score: 0.75,
+						violations: [{ type: 'node-mismatch', description: 'Missing expected node' }],
+					},
+				}),
+			);
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+			const feedback = await evaluator.evaluate(workflow, {
+				prompt: 'Test',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			const similarityFeedback = findFeedback(feedback, 'similarity');
+			expect(similarityFeedback).toEqual({
+				evaluator: 'programmatic',
+				metric: 'similarity',
+				score: 0.75,
+				kind: 'metric',
+				comment: '[node-mismatch] Missing expected node',
+			});
+		});
+
+		it('should not include similarity feedback when result is null', async () => {
+			mockProgrammaticEvaluation.mockResolvedValue(
+				createMockEvaluationResult({
+					similarity: null,
+				}),
+			);
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test' });
+
+			const similarityFeedback = findFeedback(feedback, 'similarity');
+			expect(similarityFeedback).toBeUndefined();
+		});
+
+		it('should handle evaluation errors gracefully', async () => {
+			mockProgrammaticEvaluation.mockRejectedValue(new Error('Evaluation failed'));
+
+			const { createProgrammaticEvaluator } = await import('../../evaluators/programmatic');
+			const evaluator = createProgrammaticEvaluator(mockNodeTypes);
+
+			const workflow = createMockWorkflow();
+
+			// Should throw - let the runner handle errors
+			await expect(evaluator.evaluate(workflow, { prompt: 'Test' })).rejects.toThrow(
+				'Evaluation failed',
+			);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluators/similarity.test.ts

@@ -0,0 +1,306 @@
+/**
+ * Tests for similarity evaluator factory.
+ *
+ * These tests mock the underlying workflow similarity functions and verify
+ * that the factory correctly wraps them and transforms the results.
+ */
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+// Store mocks for similarity functions
+const mockEvaluateWorkflowSimilarity = jest.fn();
+const mockEvaluateWorkflowSimilarityMultiple = jest.fn();
+
+// Mock the workflow similarity module
+jest.mock('../../programmatic/evaluators/workflow-similarity', () => ({
+	evaluateWorkflowSimilarity: (...args: unknown[]): unknown =>
+		mockEvaluateWorkflowSimilarity(...args),
+	evaluateWorkflowSimilarityMultiple: (...args: unknown[]): unknown =>
+		mockEvaluateWorkflowSimilarityMultiple(...args),
+}));
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create mock similarity result */
+function createMockSimilarityResult(
+	overrides: Partial<{
+		violations: Array<{ name: string; type: string; description: string; pointsDeducted: number }>;
+		score: number;
+	}> = {},
+) {
+	return {
+		violations: [],
+		score: 0.85,
+		...overrides,
+	};
+}
+
+describe('Similarity Evaluator', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+	type SimilarityFeedback = { evaluator: string; metric: string; score: number; comment?: string };
+	const findFeedback = (feedback: SimilarityFeedback[], metric: string) =>
+		feedback.find((f) => f.evaluator === 'similarity' && f.metric === metric);
+
+	describe('createSimilarityEvaluator()', () => {
+		it('should create an evaluator with correct name', async () => {
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			expect(evaluator.name).toBe('similarity');
+		});
+
+		it('should return error feedback when no reference workflow provided', async () => {
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
+
+			expect(mockEvaluateWorkflowSimilarity).not.toHaveBeenCalled();
+			expect(mockEvaluateWorkflowSimilarityMultiple).not.toHaveBeenCalled();
+
+			expect(feedback).toContainEqual({
+				evaluator: 'similarity',
+				metric: 'error',
+				score: 0,
+				kind: 'score',
+				comment: 'No reference workflow provided for comparison',
+			});
+		});
+
+		it('should call evaluateWorkflowSimilarity with single reference workflow', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(createMockSimilarityResult());
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator({ preset: 'strict' });
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			expect(mockEvaluateWorkflowSimilarity).toHaveBeenCalledWith(
+				workflow,
+				referenceWorkflow,
+				'strict',
+				undefined,
+			);
+		});
+
+		it('should call evaluateWorkflowSimilarityMultiple with multiple reference workflows', async () => {
+			mockEvaluateWorkflowSimilarityMultiple.mockResolvedValue(createMockSimilarityResult());
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator({ preset: 'lenient' });
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflows = [
+				createMockWorkflow('Reference 1'),
+				createMockWorkflow('Reference 2'),
+			];
+
+			await evaluator.evaluate(workflow, { prompt: 'Test prompt', referenceWorkflows });
+
+			expect(mockEvaluateWorkflowSimilarityMultiple).toHaveBeenCalledWith(
+				workflow,
+				referenceWorkflows,
+				'lenient',
+				undefined,
+			);
+		});
+
+		it('should use default preset when not specified', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(createMockSimilarityResult());
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			expect(mockEvaluateWorkflowSimilarity).toHaveBeenCalledWith(
+				workflow,
+				referenceWorkflow,
+				'standard',
+				undefined,
+			);
+		});
+
+		it('should return feedback with similarity score', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(createMockSimilarityResult({ score: 0.92 }));
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			const feedback = await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			const scoreFeedback = findFeedback(feedback, 'score');
+			expect(scoreFeedback?.score).toBe(0.92);
+		});
+
+		it('should include violations in score feedback comment', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(
+				createMockSimilarityResult({
+					score: 0.7,
+					violations: [
+						{
+							name: 'workflow-similarity-node-delete',
+							type: 'major',
+							description: 'Missing Slack node',
+							pointsDeducted: 10,
+						},
+					],
+				}),
+			);
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			const feedback = await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			const scoreFeedback = findFeedback(feedback, 'score');
+			expect(scoreFeedback?.comment).toContain('[major] Missing Slack node');
+		});
+
+		it('should return feedback for each violation type', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(
+				createMockSimilarityResult({
+					violations: [
+						{
+							name: 'workflow-similarity-node-delete',
+							type: 'major',
+							description: 'Missing node A',
+							pointsDeducted: 10,
+						},
+						{
+							name: 'workflow-similarity-node-delete',
+							type: 'major',
+							description: 'Missing node B',
+							pointsDeducted: 10,
+						},
+						{
+							name: 'workflow-similarity-edge-insert',
+							type: 'minor',
+							description: 'Extra connection',
+							pointsDeducted: 5,
+						},
+					],
+				}),
+			);
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			const feedback = await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			const nodeDeleteFeedback = findFeedback(feedback, 'node-delete');
+			expect(nodeDeleteFeedback).toBeDefined();
+			expect(nodeDeleteFeedback?.comment).toContain('2 node-delete');
+
+			const edgeInsertFeedback = findFeedback(feedback, 'edge-insert');
+			expect(edgeInsertFeedback).toBeDefined();
+			expect(edgeInsertFeedback?.comment).toContain('1 edge-insert');
+		});
+
+		it('should handle evaluation errors gracefully', async () => {
+			mockEvaluateWorkflowSimilarity.mockRejectedValue(new Error('uvx command not found'));
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			const feedback = await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			const errorFeedback = findFeedback(feedback, 'error');
+			expect(errorFeedback).toEqual({
+				evaluator: 'similarity',
+				metric: 'error',
+				score: 0,
+				kind: 'score',
+				comment: 'uvx command not found',
+			});
+		});
+
+		it('should pass custom config path when provided', async () => {
+			mockEvaluateWorkflowSimilarity.mockResolvedValue(createMockSimilarityResult());
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator({
+				preset: 'standard',
+				customConfigPath: '/path/to/config.json',
+			});
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflow = createMockWorkflow('Reference');
+
+			await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows: [referenceWorkflow],
+			});
+
+			expect(mockEvaluateWorkflowSimilarity).toHaveBeenCalledWith(
+				workflow,
+				referenceWorkflow,
+				'standard',
+				'/path/to/config.json',
+			);
+		});
+
+		it('should use evaluateWorkflowSimilarityMultiple when referenceWorkflows has multiple items', async () => {
+			mockEvaluateWorkflowSimilarityMultiple.mockResolvedValue(createMockSimilarityResult());
+
+			const { createSimilarityEvaluator } = await import('../../evaluators/similarity');
+			const evaluator = createSimilarityEvaluator();
+
+			const workflow = createMockWorkflow();
+			const referenceWorkflows = [
+				createMockWorkflow('Reference 1'),
+				createMockWorkflow('Reference 2'),
+			];
+
+			await evaluator.evaluate(workflow, {
+				prompt: 'Test prompt',
+				referenceWorkflows,
+			});
+
+			expect(mockEvaluateWorkflowSimilarityMultiple).toHaveBeenCalled();
+			expect(mockEvaluateWorkflowSimilarity).not.toHaveBeenCalled();
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/feedback.test.ts

@@ -0,0 +1,152 @@
+import { langsmithMetricKey, toLangsmithEvaluationResult } from '../harness/feedback';
+import type { Feedback } from '../harness/harness-types';
+
+describe('langsmithMetricKey()', () => {
+	it('should keep llm-judge metrics unprefixed (root and sub-metrics)', () => {
+		const root: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'overallScore',
+			score: 1,
+			kind: 'score',
+		};
+		const sub: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'maintainability.workflowOrganization',
+			score: 1,
+			kind: 'detail',
+		};
+
+		expect(langsmithMetricKey(root)).toBe('overallScore');
+		expect(langsmithMetricKey(sub)).toBe('maintainability.workflowOrganization');
+	});
+
+	it('should prefix programmatic metrics with evaluator name', () => {
+		const fb: Feedback = { evaluator: 'programmatic', metric: 'trigger', score: 1, kind: 'metric' };
+		expect(langsmithMetricKey(fb)).toBe('programmatic.trigger');
+	});
+
+	it('should keep pairwise v1 metrics unprefixed and namespace non-v1 details', () => {
+		const v1: Feedback = {
+			evaluator: 'pairwise',
+			metric: 'pairwise_primary',
+			score: 0,
+			kind: 'score',
+		};
+		const detail: Feedback = { evaluator: 'pairwise', metric: 'judge1', score: 0, kind: 'detail' };
+
+		expect(langsmithMetricKey(v1)).toBe('pairwise_primary');
+		expect(langsmithMetricKey(detail)).toBe('pairwise.judge1');
+	});
+
+	it('should prefix metrics evaluator with evaluator name', () => {
+		const discoveryLatency: Feedback = {
+			evaluator: 'metrics',
+			metric: 'discovery_latency_s',
+			score: 0.5,
+			kind: 'metric',
+		};
+		const builderLatency: Feedback = {
+			evaluator: 'metrics',
+			metric: 'builder_latency_s',
+			score: 1.5,
+			kind: 'metric',
+		};
+		const responderLatency: Feedback = {
+			evaluator: 'metrics',
+			metric: 'responder_latency_s',
+			score: 0.2,
+			kind: 'metric',
+		};
+		const nodeCount: Feedback = {
+			evaluator: 'metrics',
+			metric: 'node_count',
+			score: 5,
+			kind: 'metric',
+		};
+
+		expect(langsmithMetricKey(discoveryLatency)).toBe('metrics.discovery_latency_s');
+		expect(langsmithMetricKey(builderLatency)).toBe('metrics.builder_latency_s');
+		expect(langsmithMetricKey(responderLatency)).toBe('metrics.responder_latency_s');
+		expect(langsmithMetricKey(nodeCount)).toBe('metrics.node_count');
+	});
+
+	it('should not produce collisions for the known evaluator contract', () => {
+		const feedback: Feedback[] = [
+			{ evaluator: 'llm-judge', metric: 'connections', score: 1, kind: 'metric' },
+			{ evaluator: 'llm-judge', metric: 'overallScore', score: 1, kind: 'score' },
+			{ evaluator: 'programmatic', metric: 'connections', score: 1, kind: 'metric' },
+			{ evaluator: 'programmatic', metric: 'overall', score: 1, kind: 'score' },
+			{ evaluator: 'pairwise', metric: 'pairwise_primary', score: 1, kind: 'score' },
+			{ evaluator: 'pairwise', metric: 'pairwise_total_violations', score: 1, kind: 'detail' },
+			{ evaluator: 'pairwise', metric: 'judge1', score: 0, kind: 'detail' },
+			{ evaluator: 'metrics', metric: 'discovery_latency_s', score: 0.5, kind: 'metric' },
+			{ evaluator: 'metrics', metric: 'node_count', score: 5, kind: 'metric' },
+		];
+
+		const keys = feedback.map(langsmithMetricKey);
+		expect(new Set(keys).size).toBe(keys.length);
+	});
+});
+
+describe('toLangsmithEvaluationResult()', () => {
+	it('should clamp scores exceeding LangSmith max limit (safety net)', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'some_metric',
+			score: 150000, // Exceeds 99999.9999
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(99999.9999);
+	});
+
+	it('should clamp scores below LangSmith min limit (safety net)', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'some_metric',
+			score: -200000,
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(-99999.9999);
+	});
+
+	it('should preserve scores within valid range', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'discovery_latency_s',
+			score: 161.288, // 161 seconds, well within limits
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(161.288);
+	});
+
+	it('should include comment when present', () => {
+		const fb: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'overallScore',
+			score: 0.85,
+			kind: 'score',
+			comment: 'Good workflow',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.comment).toBe('Good workflow');
+	});
+
+	it('should not include comment when absent', () => {
+		const fb: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'overallScore',
+			score: 0.85,
+			kind: 'score',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.comment).toBeUndefined();
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/lifecycle.test.ts

@@ -0,0 +1,852 @@
+/**
+ * Tests for default console lifecycle implementation.
+ */
+
+import { mock } from 'jest-mock-extended';
+import type { Client } from 'langsmith/client';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type {
+	EvaluationLifecycle,
+	RunConfig,
+	ExampleResult,
+	RunSummary,
+	Feedback,
+} from '../harness/harness-types';
+import { createLogger } from '../harness/logger';
+
+const mockLangsmithClient = () => mock<Client>();
+
+// Mock console methods
+const mockConsole = {
+	log: jest.fn(),
+	warn: jest.fn(),
+	error: jest.fn(),
+};
+
+// Store original console
+const originalConsole = { ...console };
+
+beforeEach(() => {
+	jest.clearAllMocks();
+	console.log = mockConsole.log;
+	console.warn = mockConsole.warn;
+	console.error = mockConsole.error;
+});
+
+afterEach(() => {
+	console.log = originalConsole.log;
+	console.warn = originalConsole.warn;
+	console.error = originalConsole.error;
+});
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+describe('Console Lifecycle', () => {
+	describe('createConsoleLifecycle()', () => {
+		it('should create a lifecycle with all hooks', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			expect(lifecycle.onStart).toBeDefined();
+			expect(lifecycle.onExampleStart).toBeDefined();
+			expect(lifecycle.onWorkflowGenerated).toBeDefined();
+			expect(lifecycle.onEvaluatorComplete).toBeDefined();
+			expect(lifecycle.onEvaluatorError).toBeDefined();
+			expect(lifecycle.onExampleComplete).toBeDefined();
+			expect(lifecycle.onEnd).toBeDefined();
+		});
+
+		it('should log experiment info on start with test cases array', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [],
+				logger: createLogger(false),
+			};
+
+			lifecycle.onStart(config);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('local');
+			expect(logOutput).toContain('Test cases');
+		});
+
+		it('should log dataset name for langsmith mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'my-dataset-name',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [{ name: 'test-eval', evaluate: jest.fn() }],
+				langsmithClient: mockLangsmithClient(),
+				langsmithOptions: {
+					experimentName: 'test-experiment',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: createLogger(false),
+			};
+
+			lifecycle.onStart(config);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('langsmith');
+			expect(logOutput).toContain('my-dataset-name');
+		});
+
+		it('should not log summary in langsmith mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'my-dataset-name',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [{ name: 'test-eval', evaluate: jest.fn() }],
+				langsmithClient: mockLangsmithClient(),
+				langsmithOptions: {
+					experimentName: 'test-experiment',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: createLogger(false),
+			};
+
+			lifecycle.onStart(config);
+			mockConsole.log.mockClear();
+
+			await lifecycle.onEnd({
+				totalExamples: 0,
+				passed: 0,
+				failed: 0,
+				errors: 0,
+				averageScore: 0,
+				totalDurationMs: 0,
+			});
+
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should log example progress in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			lifecycle.onExampleStart(1, 10, 'Test prompt that is quite long and should be truncated');
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('[ex 1/10]');
+		});
+
+		it('should not log example progress in non-verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			lifecycle.onExampleStart(1, 10, 'Test prompt');
+
+			// Should not log in non-verbose mode
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should log workflow generation in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const workflow = createMockWorkflow('My Workflow');
+			lifecycle.onWorkflowGenerated(workflow, 1500);
+
+			// workflow generation is reported as part of the example completion block
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should log evaluator completion in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			lifecycle.onEvaluatorComplete('llm-judge', [
+				{ evaluator: 'llm-judge', metric: 'func', score: 0.8, kind: 'metric' },
+				{ evaluator: 'llm-judge', metric: 'conn', score: 0.9, kind: 'metric' },
+			]);
+
+			// evaluator completion is reported as part of the example completion block
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should not log evaluator completion in non-verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			lifecycle.onEvaluatorComplete('llm-judge', [
+				{ evaluator: 'llm-judge', metric: 'func', score: 0.8, kind: 'metric' },
+			]);
+
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should display critical metrics in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test prompt that is quite long and should be truncated for display in logs',
+				status: 'pass',
+				score: 0.85,
+				feedback: [
+					{ evaluator: 'llm-judge', metric: 'functionality', score: 0.95, kind: 'metric' },
+					{ evaluator: 'llm-judge', metric: 'connections', score: 0.8, kind: 'metric' },
+					{ evaluator: 'llm-judge', metric: 'overallScore', score: 0.85, kind: 'score' },
+					{ evaluator: 'other', metric: 'metric', score: 0.5, kind: 'detail' },
+				],
+				durationMs: 2000,
+				generationDurationMs: 1500,
+				evaluationDurationMs: 500,
+				workflow: createMockWorkflow(),
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('prompt=');
+			expect(logOutput).toContain('functionality');
+			expect(logOutput).toContain('connections');
+			expect(logOutput).toContain('overallScore');
+		});
+
+		it('should display violations in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'fail',
+				score: 0.3,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'functionality',
+						score: 0.5,
+						comment: '[critical] Missing trigger node',
+						kind: 'metric',
+					},
+					{
+						evaluator: 'llm-judge',
+						metric: 'connections',
+						score: 0.3,
+						comment: '[major] Disconnected node found',
+						kind: 'metric',
+					},
+				],
+				durationMs: 1500,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('issues');
+			expect(logOutput).toContain('Missing trigger node');
+			expect(logOutput).toContain('Disconnected node found');
+		});
+
+		it('should display pairwise judge violations in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'fail',
+				score: 0.5,
+				feedback: [
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_primary',
+						score: 0,
+						comment: '1/3 judges passed',
+						kind: 'score',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'judge2',
+						score: 0,
+						comment: '[No HTTP] Contains HTTP Request node',
+						kind: 'detail',
+					},
+				],
+				durationMs: 1500,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('issues');
+			expect(logOutput).toContain('judge2');
+			expect(logOutput).toContain('Contains HTTP Request node');
+		});
+
+		it('should limit violations display to 5 and show count', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'fail',
+				score: 0.5,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'v1',
+						score: 0.5,
+						comment: '[minor] Violation 1',
+						kind: 'detail',
+					},
+					{
+						evaluator: 'llm-judge',
+						metric: 'v2',
+						score: 0.5,
+						comment: '[minor] Violation 2',
+						kind: 'detail',
+					},
+					{
+						evaluator: 'llm-judge',
+						metric: 'v3',
+						score: 0.5,
+						comment: '[minor] Violation 3',
+						kind: 'detail',
+					},
+					{
+						evaluator: 'llm-judge',
+						metric: 'v4',
+						score: 0.5,
+						comment: '[minor] Violation 4',
+						kind: 'detail',
+					},
+				],
+				durationMs: 1000,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('and 1 more');
+		});
+
+		it('should not display violations for error feedback', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'error',
+				score: 0,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'error',
+						score: 0,
+						comment: 'Crashed',
+						kind: 'score',
+					},
+				],
+				durationMs: 500,
+				error: 'Generation failed',
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).not.toContain('issues');
+		});
+
+		it('should handle empty feedback array', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'fail',
+				score: 0,
+				feedback: [],
+				durationMs: 500,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('0%');
+		});
+
+		it('should log evaluator errors in verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			lifecycle.onEvaluatorError('test-evaluator', new Error('Something went wrong'));
+
+			expect(mockConsole.error).toHaveBeenCalled();
+			const errorOutput = mockConsole.error.mock.calls.flat().join(' ');
+			expect(errorOutput).toContain('test-evaluator');
+			expect(errorOutput).toContain('Something went wrong');
+		});
+
+		it('should NOT log evaluator errors in non-verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			lifecycle.onEvaluatorError('test-evaluator', new Error('Something went wrong'));
+
+			expect(mockConsole.error).not.toHaveBeenCalled();
+		});
+
+		it('should log example completion with pass/fail status', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const passResult: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'pass',
+				score: 0.9,
+				feedback: [{ evaluator: 'test-eval', metric: 'test', score: 0.9, kind: 'score' }],
+				durationMs: 2000,
+			};
+
+			const failResult: ExampleResult = {
+				index: 2,
+				prompt: 'Test',
+				status: 'fail',
+				score: 0.3,
+				feedback: [{ evaluator: 'test-eval', metric: 'test', score: 0.3, kind: 'score' }],
+				durationMs: 1500,
+			};
+
+			lifecycle.onExampleComplete(1, passResult);
+			lifecycle.onExampleComplete(2, failResult);
+
+			const allOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(allOutput).toContain('PASS');
+			expect(allOutput).toContain('FAIL');
+		});
+
+		it('should log example completion with error status', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const errorResult: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'error',
+				score: 0,
+				feedback: [],
+				durationMs: 500,
+				error: 'Generation failed',
+			};
+
+			lifecycle.onExampleComplete(1, errorResult);
+
+			const allOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(allOutput).toContain('ERROR');
+		});
+
+		it('should not log example completion in non-verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'pass',
+				score: 0.9,
+				feedback: [{ evaluator: 'test-eval', metric: 'test', score: 0.9, kind: 'score' }],
+				durationMs: 2000,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should not log workflow generation in non-verbose mode', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const workflow = createMockWorkflow('My Workflow');
+			lifecycle.onWorkflowGenerated(workflow, 1500);
+
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+
+		it('should use different colors for different score ranges', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(true) });
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'pass',
+				score: 0.8,
+				feedback: [
+					{ evaluator: 'high', metric: 'test', score: 0.95, kind: 'score' },
+					{ evaluator: 'medium', metric: 'test', score: 0.75, kind: 'score' },
+					{ evaluator: 'low', metric: 'test', score: 0.5, kind: 'score' },
+				],
+				durationMs: 1000,
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			// The coloring is applied, tests verify that the function runs without error
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('high');
+			expect(logOutput).toContain('medium');
+			expect(logOutput).toContain('low');
+		});
+
+		it('should log summary with statistics on end', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: false, logger: createLogger(false) });
+
+			const summary: RunSummary = {
+				totalExamples: 10,
+				passed: 7,
+				failed: 2,
+				errors: 1,
+				averageScore: 0.85,
+				totalDurationMs: 30000,
+			};
+
+			await lifecycle.onEnd(summary);
+
+			expect(mockConsole.log).toHaveBeenCalled();
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).toContain('10');
+			expect(logOutput).toContain('7');
+			expect(logOutput).toContain('85%');
+		});
+
+		it('should not print NaN when feedback contains non-finite scores', async () => {
+			const { createConsoleLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createConsoleLifecycle({ verbose: true, logger: createLogger(false) });
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [{ name: 'programmatic', evaluate: jest.fn() }],
+				logger: createLogger(false),
+			};
+
+			lifecycle.onStart(config);
+
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'pass',
+				score: 1,
+				durationMs: 10,
+				workflow: createMockWorkflow(),
+				feedback: [
+					{ evaluator: 'programmatic', metric: 'connections', score: 1, kind: 'metric' },
+					{ evaluator: 'programmatic', metric: 'trigger', score: Number.NaN, kind: 'metric' },
+				],
+			};
+
+			lifecycle.onExampleComplete(1, result);
+
+			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
+			expect(logOutput).not.toContain('NaN');
+		});
+	});
+
+	describe('createQuietLifecycle()', () => {
+		it('should create lifecycle with empty hooks', async () => {
+			const { createQuietLifecycle } = await import('../harness/lifecycle');
+			const lifecycle = createQuietLifecycle();
+
+			// Should have all hooks
+			expect(lifecycle.onStart).toBeDefined();
+			expect(lifecycle.onEnd).toBeDefined();
+
+			// Hooks should be no-ops
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [],
+				generateWorkflow: jest.fn(),
+				evaluators: [],
+				logger: createLogger(false),
+			};
+
+			lifecycle.onStart(config);
+			lifecycle.onExampleStart(1, 1, 'Test');
+			await lifecycle.onEnd({
+				totalExamples: 1,
+				passed: 1,
+				failed: 0,
+				errors: 0,
+				averageScore: 1,
+				totalDurationMs: 1000,
+			});
+
+			// Should not log anything
+			expect(mockConsole.log).not.toHaveBeenCalled();
+		});
+	});
+
+	describe('mergeLifecycles()', () => {
+		it('should merge multiple lifecycles into one', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = {
+				onStart: hook1,
+			};
+
+			const lifecycle2: Partial<EvaluationLifecycle> = {
+				onStart: hook2,
+			};
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [],
+				generateWorkflow: jest.fn(),
+				evaluators: [],
+				logger: createLogger(false),
+			};
+
+			merged.onStart(config);
+
+			expect(hook1).toHaveBeenCalledWith(config);
+			expect(hook2).toHaveBeenCalledWith(config);
+		});
+
+		it('should handle undefined hooks gracefully', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = {
+				onStart: hook,
+			};
+
+			const lifecycle2: Partial<EvaluationLifecycle> = {
+				// No onStart
+			};
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [],
+				generateWorkflow: jest.fn(),
+				evaluators: [],
+				logger: createLogger(false),
+			};
+
+			merged.onStart(config);
+
+			expect(hook).toHaveBeenCalledWith(config);
+		});
+
+		it('should handle undefined lifecycles in array', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = {
+				onStart: hook,
+			};
+
+			const merged = mergeLifecycles(lifecycle1, undefined, undefined);
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [],
+				generateWorkflow: jest.fn(),
+				evaluators: [],
+				logger: createLogger(false),
+			};
+
+			merged.onStart(config);
+
+			expect(hook).toHaveBeenCalledWith(config);
+		});
+
+		it('should merge onExampleStart hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onExampleStart: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onExampleStart: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			merged.onExampleStart(1, 10, 'Test prompt');
+
+			expect(hook1).toHaveBeenCalledWith(1, 10, 'Test prompt');
+			expect(hook2).toHaveBeenCalledWith(1, 10, 'Test prompt');
+		});
+
+		it('should merge onWorkflowGenerated hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onWorkflowGenerated: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onWorkflowGenerated: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const workflow = createMockWorkflow();
+			merged.onWorkflowGenerated(workflow, 1000);
+
+			expect(hook1).toHaveBeenCalledWith(workflow, 1000);
+			expect(hook2).toHaveBeenCalledWith(workflow, 1000);
+		});
+
+		it('should merge onEvaluatorComplete hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onEvaluatorComplete: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onEvaluatorComplete: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const feedback: Feedback[] = [
+				{ evaluator: 'test-eval', metric: 'test', score: 0.9, kind: 'score' },
+			];
+			merged.onEvaluatorComplete('test-eval', feedback);
+
+			expect(hook1).toHaveBeenCalledWith('test-eval', feedback);
+			expect(hook2).toHaveBeenCalledWith('test-eval', feedback);
+		});
+
+		it('should merge onEvaluatorError hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onEvaluatorError: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onEvaluatorError: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const error = new Error('Test error');
+			merged.onEvaluatorError('test-eval', error);
+
+			expect(hook1).toHaveBeenCalledWith('test-eval', error);
+			expect(hook2).toHaveBeenCalledWith('test-eval', error);
+		});
+
+		it('should merge onExampleComplete hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onExampleComplete: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onExampleComplete: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const result: ExampleResult = {
+				index: 1,
+				prompt: 'Test',
+				status: 'pass',
+				score: 1,
+				feedback: [],
+				durationMs: 1000,
+			};
+			merged.onExampleComplete(1, result);
+
+			expect(hook1).toHaveBeenCalledWith(1, result);
+			expect(hook2).toHaveBeenCalledWith(1, result);
+		});
+
+		it('should merge onEnd hooks', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const hook1 = jest.fn();
+			const hook2 = jest.fn();
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onEnd: hook1 };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onEnd: hook2 };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const summary: RunSummary = {
+				totalExamples: 10,
+				passed: 8,
+				failed: 1,
+				errors: 1,
+				averageScore: 0.85,
+				totalDurationMs: 5000,
+			};
+			await merged.onEnd(summary);
+
+			expect(hook1).toHaveBeenCalledWith(summary);
+			expect(hook2).toHaveBeenCalledWith(summary);
+		});
+
+		it('should properly await async onEnd hooks in mergeLifecycles', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const callOrder: string[] = [];
+
+			const asyncHook = jest.fn(async () => {
+				await new Promise((resolve) => setTimeout(resolve, 50));
+				callOrder.push('async');
+			});
+			const syncHook = jest.fn(() => {
+				callOrder.push('sync');
+			});
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onEnd: asyncHook };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onEnd: syncHook };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const summary: RunSummary = {
+				totalExamples: 1,
+				passed: 1,
+				failed: 0,
+				errors: 0,
+				averageScore: 1,
+				totalDurationMs: 100,
+			};
+
+			await merged.onEnd(summary);
+
+			expect(asyncHook).toHaveBeenCalledWith(summary);
+			expect(syncHook).toHaveBeenCalledWith(summary);
+			// Async hook should complete before sync hook starts (sequential await)
+			expect(callOrder).toEqual(['async', 'sync']);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/output.test.ts

@@ -0,0 +1,403 @@
+/**
+ * Tests for artifact saving functionality.
+ */
+
+import * as fs from 'fs';
+import { jsonParse } from 'n8n-workflow';
+import * as os from 'os';
+import * as path from 'path';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type { ExampleResult, RunSummary } from '../harness/harness-types';
+import { createLogger } from '../harness/logger';
+import { createArtifactSaver } from '../harness/output';
+
+const silentLogger = createLogger(false);
+
+function findExampleDir(baseDir: string, paddedIndex: string): string {
+	const entries = fs.readdirSync(baseDir, { withFileTypes: true });
+	const prefix = `example-${paddedIndex}-`;
+	const match = entries.find((e) => e.isDirectory() && e.name.startsWith(prefix));
+	if (!match) throw new Error(`Expected example dir starting with "${prefix}" in ${baseDir}`);
+	return path.join(baseDir, match.name);
+}
+
+/** Type for parsed workflow JSON */
+interface ParsedWorkflow {
+	name: string;
+	nodes: unknown[];
+	connections: Record<string, unknown>;
+}
+
+/** Type for parsed feedback JSON */
+interface ParsedFeedback {
+	index: number;
+	status: string;
+	score: number;
+	evaluators: Array<{
+		name: string;
+		averageScore: number;
+		feedback: Array<{ key: string; score: number }>;
+	}>;
+	subgraphMetrics?: {
+		nodeCount?: number;
+		discoveryDurationMs?: number;
+		builderDurationMs?: number;
+		responderDurationMs?: number;
+	};
+}
+
+/** Type for parsed summary JSON */
+interface ParsedSummary {
+	totalExamples: number;
+	passed: number;
+	failed: number;
+	passRate: number;
+	timestamp: string;
+	evaluatorAverages: Record<string, number>;
+	results: Array<{
+		prompt: string;
+		nodeCount?: number;
+		discoveryDurationMs?: number;
+		builderDurationMs?: number;
+		responderDurationMs?: number;
+	}>;
+}
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return {
+		name,
+		nodes: [
+			{
+				id: '1',
+				type: 'n8n-nodes-base.start',
+				name: 'Start',
+				position: [0, 0],
+				typeVersion: 1,
+				parameters: {},
+			},
+		],
+		connections: {},
+	};
+}
+
+/** Helper to create a mock example result */
+function createMockResult(overrides: Partial<ExampleResult> = {}): ExampleResult {
+	return {
+		index: 1,
+		prompt: 'Create a test workflow',
+		status: 'pass',
+		score: 0.9,
+		feedback: [
+			{ evaluator: 'llm-judge', metric: 'functionality', score: 0.9, kind: 'metric' },
+			{ evaluator: 'llm-judge', metric: 'connections', score: 0.8, kind: 'metric' },
+			{ evaluator: 'programmatic', metric: 'overall', score: 1.0, kind: 'score' },
+		],
+		durationMs: 1500,
+		workflow: createMockWorkflow(),
+		...overrides,
+	};
+}
+
+/** Helper to create a mock summary */
+function createMockSummary(): RunSummary {
+	return {
+		totalExamples: 3,
+		passed: 2,
+		failed: 1,
+		errors: 0,
+		averageScore: 0.85,
+		totalDurationMs: 5000,
+	};
+}
+
+describe('Artifact Saver', () => {
+	let tempDir: string;
+
+	beforeEach(() => {
+		// Create a unique temp directory for each test
+		tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'v2-eval-test-'));
+	});
+
+	afterEach(() => {
+		// Clean up temp directory
+		if (tempDir && fs.existsSync(tempDir)) {
+			fs.rmSync(tempDir, { recursive: true, force: true });
+		}
+	});
+
+	describe('createArtifactSaver()', () => {
+		it('should create output directory if it does not exist', () => {
+			const outputDir = path.join(tempDir, 'nested', 'output');
+			createArtifactSaver({ outputDir, logger: silentLogger });
+
+			expect(fs.existsSync(outputDir)).toBe(true);
+		});
+
+		it('should return an artifact saver with saveExample and saveSummary methods', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+
+			expect(saver.saveExample).toBeDefined();
+			expect(saver.saveSummary).toBeDefined();
+		});
+	});
+
+	describe('saveExample()', () => {
+		it('should save prompt to prompt.txt', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult({ index: 1 });
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const promptPath = path.join(exampleDir, 'prompt.txt');
+			expect(fs.existsSync(promptPath)).toBe(true);
+			expect(fs.readFileSync(promptPath, 'utf-8')).toBe('Create a test workflow');
+		});
+
+		it('should save workflow to workflow.json in n8n-importable format', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult();
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const workflowPath = path.join(exampleDir, 'workflow.json');
+			expect(fs.existsSync(workflowPath)).toBe(true);
+
+			const workflow = jsonParse<ParsedWorkflow>(fs.readFileSync(workflowPath, 'utf-8'));
+			expect(workflow.name).toBe('Test Workflow');
+			expect(workflow.nodes).toHaveLength(1);
+			expect(workflow.connections).toEqual({});
+		});
+
+		it('should save feedback to feedback.json', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult();
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const feedbackPath = path.join(exampleDir, 'feedback.json');
+			expect(fs.existsSync(feedbackPath)).toBe(true);
+
+			const feedback = jsonParse<ParsedFeedback>(fs.readFileSync(feedbackPath, 'utf-8'));
+			expect(feedback.index).toBe(1);
+			expect(feedback.status).toBe('pass');
+			expect(feedback.evaluators).toHaveLength(2); // llm-judge and programmatic
+		});
+
+		it('should group feedback by evaluator', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult();
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const feedbackPath = path.join(exampleDir, 'feedback.json');
+			const feedback = jsonParse<ParsedFeedback>(fs.readFileSync(feedbackPath, 'utf-8'));
+
+			const llmJudge = feedback.evaluators.find((e) => e.name === 'llm-judge');
+			expect(llmJudge?.feedback).toHaveLength(2);
+
+			const programmatic = feedback.evaluators.find((e) => e.name === 'programmatic');
+			expect(programmatic?.feedback).toHaveLength(1);
+		});
+
+		it('should ignore non-finite scores when computing evaluator averages', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult({
+				feedback: [
+					{ evaluator: 'programmatic', metric: 'connections', score: 1, kind: 'metric' },
+					{ evaluator: 'programmatic', metric: 'trigger', score: Number.NaN, kind: 'metric' },
+				],
+			});
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const feedbackPath = path.join(exampleDir, 'feedback.json');
+			const feedback = jsonParse<ParsedFeedback>(fs.readFileSync(feedbackPath, 'utf-8'));
+
+			const programmatic = feedback.evaluators.find((e) => e.name === 'programmatic');
+			expect(programmatic).toBeDefined();
+			expect(programmatic?.averageScore).toBe(1);
+		});
+
+		it('should save error to error.txt when present', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult({
+				status: 'error',
+				score: 0,
+				error: 'Generation failed: timeout',
+			});
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const errorPath = path.join(exampleDir, 'error.txt');
+			expect(fs.existsSync(errorPath)).toBe(true);
+			expect(fs.readFileSync(errorPath, 'utf-8')).toBe('Generation failed: timeout');
+		});
+
+		it('should not save workflow.json when workflow is undefined', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult({ workflow: undefined });
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const workflowPath = path.join(exampleDir, 'workflow.json');
+			expect(fs.existsSync(workflowPath)).toBe(false);
+		});
+
+		it('should include subgraph metrics in feedback.json when present', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult({
+				subgraphMetrics: {
+					nodeCount: 7,
+					discoveryDurationMs: 350,
+					builderDurationMs: 900,
+					responderDurationMs: 200,
+				},
+			});
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const feedbackPath = path.join(exampleDir, 'feedback.json');
+			const feedback = jsonParse<ParsedFeedback>(fs.readFileSync(feedbackPath, 'utf-8'));
+
+			expect(feedback.subgraphMetrics).toBeDefined();
+			expect(feedback.subgraphMetrics?.nodeCount).toBe(7);
+			expect(feedback.subgraphMetrics?.discoveryDurationMs).toBe(350);
+			expect(feedback.subgraphMetrics?.builderDurationMs).toBe(900);
+			expect(feedback.subgraphMetrics?.responderDurationMs).toBe(200);
+		});
+
+		it('should not include subgraph metrics in feedback.json when not present', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const result = createMockResult(); // No subgraphMetrics
+
+			saver.saveExample(result);
+
+			const exampleDir = findExampleDir(tempDir, '001');
+			const feedbackPath = path.join(exampleDir, 'feedback.json');
+			const feedback = jsonParse<ParsedFeedback>(fs.readFileSync(feedbackPath, 'utf-8'));
+
+			expect(feedback.subgraphMetrics).toBeUndefined();
+		});
+
+		it('should pad example index in directory name', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+
+			saver.saveExample(createMockResult({ index: 1 }));
+			saver.saveExample(createMockResult({ index: 10 }));
+			saver.saveExample(createMockResult({ index: 100 }));
+
+			expect(() => findExampleDir(tempDir, '001')).not.toThrow();
+			expect(() => findExampleDir(tempDir, '010')).not.toThrow();
+			expect(() => findExampleDir(tempDir, '100')).not.toThrow();
+		});
+	});
+
+	describe('saveSummary()', () => {
+		it('should save summary.json with correct structure', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const summary = createMockSummary();
+			const results = [
+				createMockResult({ index: 1, status: 'pass' }),
+				createMockResult({ index: 2, status: 'pass' }),
+				createMockResult({ index: 3, status: 'fail' }),
+			];
+
+			saver.saveSummary(summary, results);
+
+			const summaryPath = path.join(tempDir, 'summary.json');
+			expect(fs.existsSync(summaryPath)).toBe(true);
+
+			const savedSummary = jsonParse<ParsedSummary>(fs.readFileSync(summaryPath, 'utf-8'));
+			expect(savedSummary.totalExamples).toBe(3);
+			expect(savedSummary.passed).toBe(2);
+			expect(savedSummary.failed).toBe(1);
+			expect(savedSummary.passRate).toBeCloseTo(2 / 3);
+		});
+
+		it('should include timestamp', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			saver.saveSummary(createMockSummary(), [createMockResult()]);
+
+			const summaryPath = path.join(tempDir, 'summary.json');
+			const savedSummary = jsonParse<ParsedSummary>(fs.readFileSync(summaryPath, 'utf-8'));
+
+			expect(savedSummary.timestamp).toBeDefined();
+			expect(new Date(savedSummary.timestamp).getTime()).not.toBeNaN();
+		});
+
+		it('should calculate per-evaluator averages', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const results = [createMockResult(), createMockResult()];
+
+			saver.saveSummary(createMockSummary(), results);
+
+			const summaryPath = path.join(tempDir, 'summary.json');
+			const savedSummary = jsonParse<ParsedSummary>(fs.readFileSync(summaryPath, 'utf-8'));
+
+			expect(savedSummary.evaluatorAverages).toBeDefined();
+			expect(savedSummary.evaluatorAverages['llm-judge']).toBeCloseTo(0.85);
+			expect(savedSummary.evaluatorAverages['programmatic']).toBe(1.0);
+		});
+
+		it('should include truncated prompts in results', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const longPrompt = 'A'.repeat(200);
+			const results = [createMockResult({ prompt: longPrompt })];
+
+			saver.saveSummary(createMockSummary(), results);
+
+			const summaryPath = path.join(tempDir, 'summary.json');
+			const savedSummary = jsonParse<ParsedSummary>(fs.readFileSync(summaryPath, 'utf-8'));
+
+			expect(savedSummary.results[0].prompt.length).toBeLessThan(longPrompt.length);
+			expect(savedSummary.results[0].prompt).toContain('...');
+		});
+
+		it('should include subgraph metrics in summary results when present', () => {
+			const saver = createArtifactSaver({ outputDir: tempDir, logger: silentLogger });
+			const results = [
+				createMockResult({
+					index: 1,
+					subgraphMetrics: {
+						nodeCount: 5,
+						discoveryDurationMs: 250,
+						builderDurationMs: 750,
+						responderDurationMs: 150,
+					},
+				}),
+				createMockResult({
+					index: 2,
+					// No subgraphMetrics
+				}),
+			];
+
+			saver.saveSummary(createMockSummary(), results);
+
+			const summaryPath = path.join(tempDir, 'summary.json');
+			const savedSummary = jsonParse<ParsedSummary>(fs.readFileSync(summaryPath, 'utf-8'));
+
+			// First result should have subgraph metrics
+			expect(savedSummary.results[0].nodeCount).toBe(5);
+			expect(savedSummary.results[0].discoveryDurationMs).toBe(250);
+			expect(savedSummary.results[0].builderDurationMs).toBe(750);
+			expect(savedSummary.results[0].responderDurationMs).toBe(150);
+
+			// Second result should not have subgraph metrics
+			expect(savedSummary.results[1].nodeCount).toBeUndefined();
+			expect(savedSummary.results[1].discoveryDurationMs).toBeUndefined();
+			expect(savedSummary.results[1].builderDurationMs).toBeUndefined();
+			expect(savedSummary.results[1].responderDurationMs).toBeUndefined();
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/report-generator.test.ts

@@ -0,0 +1,308 @@
+/**
+ * Tests for markdown report generation.
+ *
+ * These utilities generate human-readable markdown reports
+ * from evaluation results.
+ */
+
+import type { ExampleResult, RunSummary, Feedback } from '../harness/harness-types';
+import {
+	extractViolationSeverity,
+	calculateReportMetrics,
+	generateMarkdownReport,
+} from '../support/report-generator';
+
+/** Helper to create a feedback item */
+function createFeedback(
+	evaluator: string,
+	metric: string,
+	score: number,
+	kind: Feedback['kind'] = 'metric',
+	comment?: string,
+): Feedback {
+	return { evaluator, metric, score, kind, ...(comment ? { comment } : {}) };
+}
+
+/** Helper to create an example result */
+function createExampleResult(overrides: Partial<ExampleResult> = {}): ExampleResult {
+	return {
+		index: 1,
+		prompt: 'Test prompt',
+		status: 'pass',
+		score: 0,
+		feedback: [],
+		durationMs: 1000,
+		...overrides,
+	};
+}
+
+/** Helper to create a run summary */
+function createRunSummary(overrides: Partial<RunSummary> = {}): RunSummary {
+	return {
+		totalExamples: 10,
+		passed: 8,
+		failed: 1,
+		errors: 1,
+		averageScore: 0.75,
+		totalDurationMs: 10000,
+		...overrides,
+	};
+}
+
+describe('Report Generator', () => {
+	describe('extractViolationSeverity()', () => {
+		it('should extract critical severity', () => {
+			expect(extractViolationSeverity('[CRITICAL] Missing trigger')).toBe('critical');
+		});
+
+		it('should extract major severity', () => {
+			expect(extractViolationSeverity('[MAJOR] Bad configuration')).toBe('major');
+		});
+
+		it('should extract minor severity', () => {
+			expect(extractViolationSeverity('[MINOR] Style issue')).toBe('minor');
+		});
+
+		it('should return null for no violation marker', () => {
+			expect(extractViolationSeverity('Just a comment')).toBeNull();
+		});
+
+		it('should return null for undefined comment', () => {
+			expect(extractViolationSeverity(undefined)).toBeNull();
+		});
+
+		it('should be case-insensitive', () => {
+			expect(extractViolationSeverity('[critical] lowercase')).toBe('critical');
+			expect(extractViolationSeverity('[Critical] mixed')).toBe('critical');
+		});
+	});
+
+	describe('calculateReportMetrics()', () => {
+		it('should calculate evaluator averages from feedback keys', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('llm-judge', 'functionality', 0.8),
+						createFeedback('llm-judge', 'connections', 0.6),
+						createFeedback('programmatic', 'trigger', 1.0),
+					],
+				}),
+			];
+
+			const metrics = calculateReportMetrics(results);
+
+			expect(metrics.evaluatorAverages['llm-judge']).toBeCloseTo(0.7);
+			expect(metrics.evaluatorAverages['programmatic']).toBeCloseTo(1.0);
+		});
+
+		it('should ignore non-finite scores when computing evaluator averages', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('programmatic', 'connections', 1),
+						createFeedback('programmatic', 'trigger', Number.NaN),
+					],
+				}),
+			];
+
+			const metrics = calculateReportMetrics(results);
+
+			expect(metrics.evaluatorAverages['programmatic']).toBe(1);
+		});
+
+		it('should count violations by severity from comments', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('a', 'b', 0, 'detail', '[CRITICAL] Missing node'),
+						createFeedback('a', 'c', 0, 'detail', '[MAJOR] Bad config'),
+						createFeedback('a', 'd', 0, 'detail', '[MINOR] Style issue'),
+						createFeedback('a', 'e', 0, 'detail', '[CRITICAL] Another critical'),
+					],
+				}),
+			];
+
+			const metrics = calculateReportMetrics(results);
+
+			expect(metrics.violationCounts.critical).toBe(2);
+			expect(metrics.violationCounts.major).toBe(1);
+			expect(metrics.violationCounts.minor).toBe(1);
+		});
+
+		it('should handle empty results', () => {
+			const metrics = calculateReportMetrics([]);
+
+			expect(metrics.evaluatorAverages).toEqual({});
+			expect(metrics.violationCounts).toEqual({ critical: 0, major: 0, minor: 0 });
+		});
+
+		it('should aggregate across multiple results', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [createFeedback('llm-judge', 'a', 0.8)],
+				}),
+				createExampleResult({
+					feedback: [createFeedback('llm-judge', 'a', 0.6)],
+				}),
+			];
+
+			const metrics = calculateReportMetrics(results);
+
+			expect(metrics.evaluatorAverages['llm-judge']).toBeCloseTo(0.7);
+		});
+
+		it('should handle results with errors', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					status: 'error',
+					feedback: [],
+					error: 'Something went wrong',
+				}),
+				createExampleResult({
+					feedback: [createFeedback('llm-judge', 'a', 0.8)],
+				}),
+			];
+
+			const metrics = calculateReportMetrics(results);
+
+			// Should still calculate from successful results
+			expect(metrics.evaluatorAverages['llm-judge']).toBeCloseTo(0.8);
+		});
+	});
+
+	describe('generateMarkdownReport()', () => {
+		it('should include summary section', () => {
+			const results: ExampleResult[] = [];
+			const summary = createRunSummary({
+				totalExamples: 10,
+				passed: 8,
+				failed: 1,
+				errors: 1,
+				averageScore: 0.75,
+			});
+
+			const report = generateMarkdownReport(results, summary);
+
+			expect(report).toContain('# AI Workflow Builder Evaluation Report');
+			expect(report).toContain('## Summary');
+			expect(report).toContain('Total Tests: 10');
+			expect(report).toContain('Passed: 8');
+			expect(report).toContain('Failed: 1');
+			expect(report).toContain('Errors: 1');
+			expect(report).toContain('75.0%');
+		});
+
+		it('should include evaluator averages', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('llm-judge', 'a', 0.8),
+						createFeedback('programmatic', 'b', 0.6),
+					],
+				}),
+			];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary);
+
+			expect(report).toContain('## Evaluator Averages');
+			expect(report).toContain('llm-judge');
+			expect(report).toContain('programmatic');
+		});
+
+		it('should include violation summary', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('a', 'b', 0, 'detail', '[CRITICAL] Issue 1'),
+						createFeedback('a', 'c', 0, 'detail', '[MAJOR] Issue 2'),
+					],
+				}),
+			];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary);
+
+			expect(report).toContain('## Violations Summary');
+			expect(report).toContain('Critical: 1');
+			expect(report).toContain('Major: 1');
+		});
+
+		it('should include detailed results when option enabled', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					index: 1,
+					prompt: 'Create a workflow that sends emails',
+					status: 'pass',
+					durationMs: 1500,
+					feedback: [createFeedback('llm-judge', 'a', 0.9, 'metric', 'Good job')],
+				}),
+			];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary, { includeDetails: true });
+
+			expect(report).toContain('## Detailed Results');
+			expect(report).toContain('### Test 1');
+			expect(report).toContain('Create a workflow');
+			expect(report).toContain('pass');
+			expect(report).toContain('1500ms');
+		});
+
+		it('should not include details when option disabled', () => {
+			const results: ExampleResult[] = [createExampleResult({ prompt: 'Test prompt here' })];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary, { includeDetails: false });
+
+			expect(report).not.toContain('## Detailed Results');
+			expect(report).not.toContain('Test prompt here');
+		});
+
+		it('should truncate long prompts in details', () => {
+			const longPrompt = 'A'.repeat(200);
+			const results: ExampleResult[] = [createExampleResult({ prompt: longPrompt })];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary, { includeDetails: true });
+
+			expect(report).toContain('...');
+			expect(report).not.toContain(longPrompt);
+		});
+
+		it('should handle empty results gracefully', () => {
+			const summary = createRunSummary({ totalExamples: 0, passed: 0, failed: 0, errors: 0 });
+
+			const report = generateMarkdownReport([], summary);
+
+			expect(report).toContain('# AI Workflow Builder Evaluation Report');
+			expect(report).toContain('Total Tests: 0');
+		});
+
+		it('should include feedback details in test results', () => {
+			const results: ExampleResult[] = [
+				createExampleResult({
+					feedback: [
+						createFeedback('llm-judge', 'functionality', 0.9, 'metric', 'Great functionality'),
+						createFeedback('programmatic', 'trigger', 1.0),
+					],
+				}),
+			];
+			const summary = createRunSummary();
+
+			const report = generateMarkdownReport(results, summary, { includeDetails: true });
+
+			expect(report).toContain('llm-judge.functionality');
+			expect(report).toContain('90.0%');
+			expect(report).toContain('Great functionality');
+		});
+
+		it('should format pass rate as percentage', () => {
+			const summary = createRunSummary({ totalExamples: 10, passed: 8 });
+
+			const report = generateMarkdownReport([], summary);
+
+			expect(report).toContain('80.0%'); // Pass rate
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/runner-langsmith.test.ts

@@ -0,0 +1,736 @@
+/**
+ * Tests for LangSmith mode runner.
+ *
+ * These tests mock the LangSmith evaluate() function to verify:
+ * - Target function does all work (generation + evaluation)
+ * - Evaluator just extracts pre-computed feedback
+ * - Dataset context extraction is respected
+ * - Filters trigger dataset example preloading
+ */
+
+import { mock } from 'jest-mock-extended';
+import type { Client } from 'langsmith/client';
+import { evaluate as langsmithEvaluate } from 'langsmith/evaluation';
+import type { Dataset, Example } from 'langsmith/schemas';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type { Evaluator, Feedback, RunConfig } from '../harness/harness-types';
+import { createLogger } from '../harness/logger';
+
+const silentLogger = createLogger(false);
+
+jest.mock('langsmith/evaluation', () => ({
+	evaluate: jest.fn().mockResolvedValue({ experimentName: 'test-experiment' }),
+}));
+
+jest.mock('langsmith/traceable', () => ({
+	traceable: jest.fn(
+		<T extends (...args: unknown[]) => unknown>(fn: T, _options: unknown): T => fn,
+	),
+}));
+
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+function createMockEvaluator(
+	name: string,
+	feedback: Feedback[] = [{ evaluator: name, metric: 'score', score: 1, kind: 'score' }],
+): Evaluator {
+	return {
+		name,
+		evaluate: jest.fn().mockResolvedValue(feedback),
+	};
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isCallable(value: unknown): value is (...args: unknown[]) => unknown {
+	return typeof value === 'function';
+}
+
+type LangsmithTargetOutput = {
+	workflow: SimpleWorkflow;
+	prompt: string;
+	feedback: Feedback[];
+};
+
+function isSimpleWorkflow(value: unknown): value is SimpleWorkflow {
+	return isRecord(value) && Array.isArray(value.nodes) && isRecord(value.connections);
+}
+
+function isFeedback(value: unknown): value is Feedback {
+	return (
+		isRecord(value) &&
+		typeof value.evaluator === 'string' &&
+		typeof value.metric === 'string' &&
+		typeof value.score === 'number' &&
+		(value.kind === 'score' || value.kind === 'metric' || value.kind === 'detail')
+	);
+}
+
+function isLangsmithTargetOutput(value: unknown): value is LangsmithTargetOutput {
+	return (
+		isRecord(value) &&
+		isSimpleWorkflow(value.workflow) &&
+		typeof value.prompt === 'string' &&
+		Array.isArray(value.feedback) &&
+		value.feedback.every(isFeedback)
+	);
+}
+
+async function callLangsmithTarget(target: unknown, inputs: unknown): Promise<unknown> {
+	if (isCallable(target)) return await target(inputs);
+	if (isRecord(target) && isCallable(target.invoke)) return await target.invoke(inputs);
+	throw new Error('Expected LangSmith target to be callable');
+}
+
+function createMockLangsmithClient() {
+	const lsClient = mock<Client>();
+	lsClient.readDataset.mockResolvedValue(mock<Dataset>({ id: 'test-dataset-id' }));
+	lsClient.listExamples.mockReturnValue((async function* () {})());
+	lsClient.awaitPendingTraceBatches.mockResolvedValue(undefined);
+	return lsClient;
+}
+
+describe('Runner - LangSmith Mode', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	describe('runEvaluation() with LangSmith', () => {
+		it('should call langsmith evaluate() with correct options', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'my-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test-experiment',
+					repetitions: 2,
+					concurrency: 4,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [_target, options] = mockEvaluate.mock.calls[0];
+			expect(options).toEqual(
+				expect.objectContaining({
+					data: 'my-dataset',
+					experimentPrefix: 'test-experiment',
+					numRepetitions: 2,
+					maxConcurrency: 4,
+					client: lsClient,
+				}),
+			);
+		});
+
+		it('should create target function that generates workflow and runs evaluators', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const workflow = createMockWorkflow('Generated');
+			const generateWorkflow = jest.fn().mockResolvedValue(workflow);
+			const evaluator = createMockEvaluator('test', [
+				{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' },
+			]);
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow,
+				evaluators: [evaluator],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [target] = mockEvaluate.mock.calls[0];
+
+			const result = await callLangsmithTarget(target, { prompt: 'Create a workflow' });
+			expect(isLangsmithTargetOutput(result)).toBe(true);
+			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');
+
+			// Collectors are passed explicitly from the traceable wrapper to capture token usage and subgraph metrics
+			expect(generateWorkflow).toHaveBeenCalledWith(
+				'Create a workflow',
+				expect.objectContaining({
+					tokenUsage: expect.any(Function),
+					subgraphMetrics: expect.any(Function),
+				}),
+			);
+			expect(evaluator.evaluate).toHaveBeenCalledWith(
+				workflow,
+				expect.objectContaining({ prompt: 'Create a workflow' }),
+			);
+			expect(result).toEqual({
+				workflow,
+				prompt: 'Create a workflow',
+				feedback: [{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' }],
+			});
+		});
+
+		it('should write artifacts when outputDir is provided', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'v2-evals-langsmith-out-'));
+			try {
+				const config: RunConfig = {
+					mode: 'langsmith',
+					dataset: 'test-dataset',
+					outputDir: tempDir,
+					generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow('Generated')),
+					evaluators: [createMockEvaluator('test')],
+					langsmithClient: lsClient,
+					langsmithOptions: {
+						experimentName: 'test',
+						repetitions: 1,
+						concurrency: 1,
+					},
+					logger: silentLogger,
+				};
+
+				const { runEvaluation } = await import('../harness/runner');
+				await runEvaluation(config);
+
+				expect(mockEvaluate).toHaveBeenCalledTimes(1);
+				const [target] = mockEvaluate.mock.calls[0];
+
+				await callLangsmithTarget(target, { prompt: 'Create a workflow' });
+
+				const entries = fs.readdirSync(tempDir, { withFileTypes: true });
+				const exampleDir = entries.find(
+					(e) => e.isDirectory() && e.name.startsWith('example-001-'),
+				)?.name;
+				expect(exampleDir).toBeDefined();
+
+				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'prompt.txt'))).toBe(true);
+				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'workflow.json'))).toBe(true);
+				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'feedback.json'))).toBe(true);
+			} finally {
+				fs.rmSync(tempDir, { recursive: true, force: true });
+			}
+		});
+
+		it('should aggregate feedback from multiple evaluators in target', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const evaluator1 = createMockEvaluator('e1', [
+				{ evaluator: 'e1', metric: 'score', score: 0.8, kind: 'score' },
+			]);
+			const evaluator2 = createMockEvaluator('e2', [
+				{ evaluator: 'e2', metric: 'a', score: 0.9, kind: 'metric' },
+				{ evaluator: 'e2', metric: 'b', score: 1.0, kind: 'metric' },
+			]);
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator1, evaluator2],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [target] = mockEvaluate.mock.calls[0];
+			const result = await callLangsmithTarget(target, { prompt: 'Test' });
+			expect(isLangsmithTargetOutput(result)).toBe(true);
+			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');
+
+			expect(result.feedback).toHaveLength(3);
+			expect(result.feedback).toContainEqual({
+				evaluator: 'e1',
+				metric: 'score',
+				score: 0.8,
+				kind: 'score',
+			});
+			expect(result.feedback).toContainEqual({
+				evaluator: 'e2',
+				metric: 'a',
+				score: 0.9,
+				kind: 'metric',
+			});
+			expect(result.feedback).toContainEqual({
+				evaluator: 'e2',
+				metric: 'b',
+				score: 1.0,
+				kind: 'metric',
+			});
+		});
+
+		it('should handle evaluator errors gracefully in target', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const goodEvaluator = createMockEvaluator('good', [
+				{ evaluator: 'good', metric: 'score', score: 1, kind: 'score' },
+			]);
+			const badEvaluator: Evaluator = {
+				name: 'bad',
+				evaluate: jest.fn().mockRejectedValue(new Error('Evaluator crashed')),
+			};
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [goodEvaluator, badEvaluator],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [target] = mockEvaluate.mock.calls[0];
+
+			const result = await callLangsmithTarget(target, { prompt: 'Test' });
+			expect(isLangsmithTargetOutput(result)).toBe(true);
+			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');
+
+			expect(result.feedback).toContainEqual({
+				evaluator: 'good',
+				metric: 'score',
+				score: 1,
+				kind: 'score',
+			});
+			expect(result.feedback).toContainEqual({
+				evaluator: 'bad',
+				metric: 'error',
+				score: 0,
+				kind: 'score',
+				comment: 'Evaluator crashed',
+			});
+		});
+
+		it('should create evaluator that extracts pre-computed feedback', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [_target, options] = mockEvaluate.mock.calls[0];
+
+			expect(Array.isArray(options.evaluators)).toBe(true);
+			if (!Array.isArray(options.evaluators))
+				throw new Error('Expected LangSmith evaluators array');
+			expect(options.evaluators).toHaveLength(1);
+
+			const evaluatorFn = options.evaluators[0];
+			expect(isCallable(evaluatorFn)).toBe(true);
+			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');
+
+			const extracted = await evaluatorFn({
+				outputs: {
+					feedback: [
+						{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' },
+						{ evaluator: 'other', metric: 'trigger', score: 0.8, kind: 'metric' },
+					],
+				},
+			});
+
+			expect(extracted).toEqual([
+				{ key: 'test.score', score: 0.9 },
+				{ key: 'other.trigger', score: 0.8 },
+			]);
+		});
+
+		it('should keep programmatic prefixes but not llm-judge metric prefixes', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [_target, options] = mockEvaluate.mock.calls[0];
+			expect(Array.isArray(options.evaluators)).toBe(true);
+			if (!Array.isArray(options.evaluators))
+				throw new Error('Expected LangSmith evaluators array');
+
+			const evaluatorFn = options.evaluators[0];
+			expect(isCallable(evaluatorFn)).toBe(true);
+			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');
+
+			const extracted = await evaluatorFn({
+				outputs: {
+					feedback: [
+						{ evaluator: 'llm-judge', metric: 'functionality', score: 0.9, kind: 'metric' },
+						{ evaluator: 'programmatic', metric: 'trigger', score: 0.8, kind: 'metric' },
+						{
+							evaluator: 'llm-judge',
+							metric: 'maintainability.nodeNamingQuality',
+							score: 0.7,
+							kind: 'detail',
+						},
+					],
+				},
+			});
+
+			expect(extracted).toEqual([
+				{ key: 'functionality', score: 0.9 },
+				{ key: 'programmatic.trigger', score: 0.8 },
+				{ key: 'maintainability.nodeNamingQuality', score: 0.7 },
+			]);
+		});
+
+		it('should handle missing feedback in outputs', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [_target, options] = mockEvaluate.mock.calls[0];
+
+			expect(Array.isArray(options.evaluators)).toBe(true);
+			if (!Array.isArray(options.evaluators))
+				throw new Error('Expected LangSmith evaluators array');
+			const evaluatorFn = options.evaluators[0];
+			expect(isCallable(evaluatorFn)).toBe(true);
+			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');
+
+			const extracted = await evaluatorFn({ outputs: {} });
+			expect(extracted).toEqual([
+				{
+					key: 'evaluationError',
+					score: 0,
+					comment: 'No feedback found in target output',
+				},
+			]);
+		});
+
+		it('should pass dataset-level context to evaluators', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const evaluateContextual: Evaluator['evaluate'] = async (_workflow, ctx) => [
+				{ evaluator: 'contextual', metric: 'score', score: ctx.dos ? 1 : 0, kind: 'score' },
+			];
+
+			const evaluator: Evaluator = {
+				name: 'contextual',
+				evaluate: jest.fn(evaluateContextual),
+			};
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [target] = mockEvaluate.mock.calls[0];
+
+			const result = await callLangsmithTarget(target, {
+				prompt: 'Test',
+				evals: { dos: 'Use Slack', donts: 'No HTTP' },
+			});
+			expect(isLangsmithTargetOutput(result)).toBe(true);
+			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');
+
+			expect(evaluator.evaluate).toHaveBeenCalledWith(
+				expect.anything(),
+				expect.objectContaining({ dos: 'Use Slack', donts: 'No HTTP' }),
+			);
+			expect(result.feedback).toContainEqual({
+				evaluator: 'contextual',
+				metric: 'score',
+				score: 1,
+				kind: 'score',
+			});
+		});
+
+		it('should ignore invalid referenceWorkflow in dataset context', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+			const lsClient = createMockLangsmithClient();
+
+			const evaluate = jest.fn<
+				ReturnType<Evaluator['evaluate']>,
+				Parameters<Evaluator['evaluate']>
+			>(async (_workflow, ctx) => [
+				{
+					evaluator: 'ref-check',
+					metric: 'hasRef',
+					score: ctx.referenceWorkflows && ctx.referenceWorkflows.length > 0 ? 1 : 0,
+					kind: 'score',
+				},
+			]);
+
+			const evaluator: Evaluator = {
+				name: 'ref-check',
+				evaluate,
+			};
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [target] = mockEvaluate.mock.calls[0];
+
+			const result = await callLangsmithTarget(target, {
+				prompt: 'Test',
+				evals: {
+					referenceWorkflow: { nodes: [{}], connections: {} },
+				},
+			});
+			expect(isLangsmithTargetOutput(result)).toBe(true);
+			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');
+
+			const ctx = evaluate.mock.calls[0]?.[1];
+			expect(ctx?.referenceWorkflows).toBeUndefined();
+
+			expect(result.feedback).toContainEqual({
+				evaluator: 'ref-check',
+				metric: 'hasRef',
+				score: 0,
+				kind: 'score',
+			});
+		});
+
+		it('should pre-load and filter examples when filters are provided', async () => {
+			const mockEvaluate = jest.mocked(langsmithEvaluate);
+
+			const examples: Example[] = [
+				mock<Example>({
+					id: 'e1',
+					inputs: { prompt: 'One', evals: { dos: 'Use Slack', donts: 'No HTTP' } },
+					metadata: { notion_id: 'n1', categories: ['data_transformation'] },
+				}),
+				mock<Example>({
+					id: 'e2',
+					inputs: { prompt: 'Two', evals: { dos: 'Use Gmail', donts: 'No Slack' } },
+					metadata: { notion_id: 'n2', categories: ['other'] },
+				}),
+			];
+
+			const lsClient = createMockLangsmithClient();
+			lsClient.listExamples.mockReturnValue(
+				(async function* () {
+					for (const ex of examples) yield ex;
+				})(),
+			);
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+					filters: { notionId: 'n1', technique: 'data_transformation', doSearch: 'slack' },
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(mockEvaluate).toHaveBeenCalledTimes(1);
+			const [_target, options] = mockEvaluate.mock.calls[0];
+			const data: unknown = options.data;
+			expect(Array.isArray(data)).toBe(true);
+			if (!Array.isArray(data)) throw new Error('Expected `evaluate()` to receive example array');
+
+			const ids = data
+				.filter((e): e is { id: string } => isRecord(e) && typeof e.id === 'string')
+				.map((e) => e.id);
+			expect(ids).toEqual(['e1']);
+		});
+
+		it('should throw when filters match no examples', async () => {
+			const lsClient = createMockLangsmithClient();
+			lsClient.listExamples.mockReturnValue(
+				(async function* () {
+					yield mock<Example>({
+						id: 'e1',
+						inputs: { prompt: 'One', evals: { dos: 'Use Slack', donts: 'No HTTP' } },
+						metadata: { notion_id: 'n1', categories: ['data_transformation'] },
+					});
+				})(),
+			);
+
+			const config: RunConfig = {
+				mode: 'langsmith',
+				dataset: 'test-dataset',
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				langsmithClient: lsClient,
+				langsmithOptions: {
+					experimentName: 'test',
+					repetitions: 1,
+					concurrency: 1,
+					filters: { notionId: 'does-not-exist' },
+				},
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await expect(runEvaluation(config)).rejects.toThrow('No examples matched filters');
+		});
+
+		it('should include evaluatorAverages in summary', async () => {
+			const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-test-'));
+			try {
+				const mockEvaluate = jest.mocked(langsmithEvaluate);
+				const lsClient = createMockLangsmithClient();
+
+				const evaluator1 = createMockEvaluator('pairwise', [
+					{ evaluator: 'pairwise', metric: 'score', score: 0.8, kind: 'score' },
+				]);
+				const evaluator2 = createMockEvaluator('programmatic', [
+					{ evaluator: 'programmatic', metric: 'overall', score: 0.9, kind: 'score' },
+				]);
+
+				// Mock evaluate to call the target function with test inputs
+				mockEvaluate.mockImplementationOnce(async (target, _options) => {
+					// Call the target to populate capturedResults
+					await callLangsmithTarget(target, { prompt: 'Test prompt 1' });
+					await callLangsmithTarget(target, { prompt: 'Test prompt 2' });
+					return { experimentName: 'test-experiment' } as Awaited<
+						ReturnType<typeof langsmithEvaluate>
+					>;
+				});
+
+				const config: RunConfig = {
+					mode: 'langsmith',
+					dataset: 'test-dataset',
+					generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+					evaluators: [evaluator1, evaluator2],
+					langsmithClient: lsClient,
+					langsmithOptions: {
+						experimentName: 'test',
+						repetitions: 1,
+						concurrency: 1,
+					},
+					outputDir: tempDir, // Enable artifact saving to capture results
+					logger: silentLogger,
+				};
+
+				const { runEvaluation } = await import('../harness/runner');
+				const summary = await runEvaluation(config);
+
+				// The summary should include evaluatorAverages computed from captured results
+				expect(summary.evaluatorAverages).toBeDefined();
+				expect(summary.evaluatorAverages).toEqual({
+					pairwise: 0.8,
+					programmatic: 0.9,
+				});
+			} finally {
+				fs.rmSync(tempDir, { recursive: true, force: true });
+			}
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/runner.test.ts

@@ -0,0 +1,437 @@
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type {
+	Evaluator,
+	TestCase,
+	Feedback,
+	RunConfig,
+	EvaluationLifecycle,
+	ExampleResult,
+} from '../harness/harness-types';
+import { createLogger } from '../harness/logger';
+
+const silentLogger = createLogger(false);
+
+/** Helper to create a minimal valid workflow for tests */
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+/** Helper to create a simple evaluator */
+function createMockEvaluator(
+	name: string,
+	feedback: Feedback[] = [{ evaluator: name, metric: 'score', score: 1, kind: 'score' }],
+): Evaluator {
+	return {
+		name,
+		evaluate: jest.fn().mockResolvedValue(feedback),
+	};
+}
+
+/** Helper to create a failing evaluator */
+function createFailingEvaluator(name: string, error: Error): Evaluator {
+	return {
+		name,
+		evaluate: jest.fn().mockRejectedValue(error),
+	};
+}
+
+describe('Runner - Local Mode', () => {
+	describe('runEvaluation()', () => {
+		it('should process all test cases sequentially', async () => {
+			const testCases: TestCase[] = [
+				{ prompt: 'Create workflow A' },
+				{ prompt: 'Create workflow B' },
+				{ prompt: 'Create workflow C' },
+			];
+
+			const generateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
+			const evaluator = createMockEvaluator('test');
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: testCases,
+				generateWorkflow,
+				evaluators: [evaluator],
+				logger: silentLogger,
+			};
+
+			// Import dynamically to avoid circular deps in test setup
+			const { runEvaluation } = await import('../harness/runner');
+			const summary = await runEvaluation(config);
+
+			expect(generateWorkflow).toHaveBeenCalledTimes(3);
+			expect(evaluator.evaluate).toHaveBeenCalledTimes(3);
+			expect(summary.totalExamples).toBe(3);
+		});
+
+		it('should run all evaluators in parallel for each example', async () => {
+			const evaluator1 = createMockEvaluator('eval1', [
+				{ evaluator: 'eval1', metric: 'score', score: 0.8, kind: 'score' },
+			]);
+			const evaluator2 = createMockEvaluator('eval2', [
+				{ evaluator: 'eval2', metric: 'score', score: 0.9, kind: 'score' },
+			]);
+			const evaluator3 = createMockEvaluator('eval3', [
+				{ evaluator: 'eval3', metric: 'score', score: 1.0, kind: 'score' },
+			]);
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator1, evaluator2, evaluator3],
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			const summary = await runEvaluation(config);
+
+			// All evaluators should be called
+			expect(evaluator1.evaluate).toHaveBeenCalledTimes(1);
+			expect(evaluator2.evaluate).toHaveBeenCalledTimes(1);
+			expect(evaluator3.evaluate).toHaveBeenCalledTimes(1);
+
+			// Average score should be (0.8 + 0.9 + 1.0) / 3 = 0.9
+			expect(summary.averageScore).toBeCloseTo(0.9, 2);
+		});
+
+		it('should skip and continue when evaluator throws error', async () => {
+			const goodEvaluator = createMockEvaluator('good', [
+				{ evaluator: 'good', metric: 'score', score: 1.0, kind: 'score' },
+			]);
+			const badEvaluator = createFailingEvaluator('bad', new Error('Evaluator crashed'));
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [goodEvaluator, badEvaluator],
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			const summary = await runEvaluation(config);
+
+			// Should complete despite error
+			expect(summary.totalExamples).toBe(1);
+			// Good evaluator should still run
+			expect(goodEvaluator.evaluate).toHaveBeenCalled();
+		});
+
+		it('should skip and continue when workflow generation fails', async () => {
+			const generateWorkflow = jest
+				.fn()
+				.mockResolvedValueOnce(createMockWorkflow())
+				.mockRejectedValueOnce(new Error('Generation failed'))
+				.mockResolvedValueOnce(createMockWorkflow());
+
+			const evaluator = createMockEvaluator('test');
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test 1' }, { prompt: 'Test 2' }, { prompt: 'Test 3' }],
+				generateWorkflow,
+				evaluators: [evaluator],
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			const summary = await runEvaluation(config);
+
+			expect(summary.totalExamples).toBe(3);
+			expect(summary.errors).toBe(1);
+			expect(summary.passed + summary.failed).toBe(2);
+		});
+
+		it('should pass context from test case to evaluators', async () => {
+			const evaluate: Evaluator['evaluate'] = async (_workflow, ctx) => {
+				expect(ctx.dos).toBe('Use Slack');
+				expect(ctx.donts).toBe('No HTTP');
+				return [{ evaluator: 'contextual', metric: 'score', score: 1, kind: 'score' }];
+			};
+
+			const evaluator: Evaluator = {
+				name: 'contextual',
+				evaluate: jest.fn(evaluate),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [
+					{
+						prompt: 'Test',
+						context: { dos: 'Use Slack', donts: 'No HTTP' },
+					},
+				],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator],
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(evaluator.evaluate).toHaveBeenCalled();
+		});
+
+		it('should merge global context with test case context', async () => {
+			const evaluate: Evaluator['evaluate'] = async (_workflow, ctx) => {
+				expect(ctx.dos).toBe('Use Slack');
+				expect(ctx.donts).toBe('No HTTP');
+				return [{ evaluator: 'merged', metric: 'score', score: 1, kind: 'score' }];
+			};
+
+			const evaluator: Evaluator = {
+				name: 'merged',
+				evaluate: jest.fn(evaluate),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test', context: { donts: 'No HTTP' } }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator],
+				context: { dos: 'Use Slack' },
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(evaluator.evaluate).toHaveBeenCalled();
+		});
+
+		it('should calculate pass/fail status based on threshold', async () => {
+			const highScoreEvaluator = createMockEvaluator('high', [
+				{ evaluator: 'high', metric: 'score', score: 0.9, kind: 'score' },
+			]);
+			const lowScoreEvaluator = createMockEvaluator('low', [
+				{ evaluator: 'low', metric: 'score', score: 0.3, kind: 'score' },
+			]);
+
+			// High score should pass (>= 0.7 threshold)
+			const config1: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [highScoreEvaluator],
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			const summary1 = await runEvaluation(config1);
+			expect(summary1.passed).toBe(1);
+
+			// Low score should fail
+			const config2: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [lowScoreEvaluator],
+				logger: silentLogger,
+			};
+
+			const summary2 = await runEvaluation(config2);
+			expect(summary2.failed).toBe(1);
+		});
+
+		it('should aggregate feedback from all evaluators', async () => {
+			const evaluator1 = createMockEvaluator('e1', [
+				{ evaluator: 'e1', metric: 'func', score: 0.8, kind: 'metric' },
+				{ evaluator: 'e1', metric: 'conn', score: 0.9, kind: 'metric' },
+			]);
+			const evaluator2 = createMockEvaluator('e2', [
+				{ evaluator: 'e2', metric: 'overall', score: 0.85, kind: 'score' },
+			]);
+
+			const collected: ExampleResult[] = [];
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onExampleComplete: (_index, result) => collected.push(result),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [evaluator1, evaluator2],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(collected).toHaveLength(1);
+			expect(collected[0].feedback).toHaveLength(3); // 2 from e1 + 1 from e2
+		});
+	});
+
+	describe('Lifecycle Hooks', () => {
+		it('should call onStart at beginning of run', async () => {
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onStart: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onStart).toHaveBeenCalledWith(config);
+		});
+
+		it('should call onExampleStart before each example', async () => {
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onExampleStart: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test 1' }, { prompt: 'Test 2' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onExampleStart).toHaveBeenCalledTimes(2);
+			expect(lifecycle.onExampleStart).toHaveBeenNthCalledWith(1, 1, 2, 'Test 1');
+			expect(lifecycle.onExampleStart).toHaveBeenNthCalledWith(2, 2, 2, 'Test 2');
+		});
+
+		it('should call onWorkflowGenerated after generation', async () => {
+			const workflow = createMockWorkflow('Generated');
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onWorkflowGenerated: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(workflow),
+				evaluators: [],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onWorkflowGenerated).toHaveBeenCalledWith(
+				workflow,
+				expect.any(Number), // durationMs
+			);
+		});
+
+		it('should call onEvaluatorComplete after each evaluator', async () => {
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onEvaluatorComplete: jest.fn(),
+			};
+
+			const feedback1: Feedback[] = [
+				{ evaluator: 'eval1', metric: 'score', score: 0.8, kind: 'score' },
+			];
+			const feedback2: Feedback[] = [
+				{ evaluator: 'eval2', metric: 'score', score: 0.9, kind: 'score' },
+			];
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [
+					createMockEvaluator('eval1', feedback1),
+					createMockEvaluator('eval2', feedback2),
+				],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onEvaluatorComplete).toHaveBeenCalledTimes(2);
+			expect(lifecycle.onEvaluatorComplete).toHaveBeenCalledWith('eval1', feedback1);
+			expect(lifecycle.onEvaluatorComplete).toHaveBeenCalledWith('eval2', feedback2);
+		});
+
+		it('should call onEvaluatorError when evaluator fails', async () => {
+			const error = new Error('Evaluator crashed');
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onEvaluatorError: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createFailingEvaluator('failing', error)],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onEvaluatorError).toHaveBeenCalledWith('failing', error);
+		});
+
+		it('should call onExampleComplete after each example', async () => {
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onExampleComplete: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			await runEvaluation(config);
+
+			expect(lifecycle.onExampleComplete).toHaveBeenCalledWith(
+				1,
+				expect.objectContaining({
+					index: 1,
+					prompt: 'Test',
+					status: 'pass',
+				}),
+			);
+		});
+
+		it('should call onEnd with summary at end of run', async () => {
+			const lifecycle: Partial<EvaluationLifecycle> = {
+				onEnd: jest.fn(),
+			};
+
+			const config: RunConfig = {
+				mode: 'local',
+				dataset: [{ prompt: 'Test' }],
+				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
+				evaluators: [createMockEvaluator('test')],
+				lifecycle,
+				logger: silentLogger,
+			};
+
+			const { runEvaluation } = await import('../harness/runner');
+			const summary = await runEvaluation(config);
+
+			expect(lifecycle.onEnd).toHaveBeenCalledWith(summary);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/score-calculator.test.ts

@@ -0,0 +1,350 @@
+/**
+ * Tests for score calculation utilities.
+ *
+ * These utilities calculate weighted scores and aggregate feedback
+ * from multiple evaluators.
+ */
+
+import type { Feedback } from '../harness/harness-types';
+import {
+	parseFeedbackKey,
+	extractCategory,
+	groupByEvaluator,
+	calculateWeightedScore,
+	aggregateScores,
+	DEFAULT_EVALUATOR_WEIGHTS,
+} from '../harness/score-calculator';
+
+/** Helper to create feedback items */
+function createFeedback(
+	evaluator: string,
+	metric: string,
+	score: number,
+	kind: Feedback['kind'] = 'metric',
+	comment?: string,
+): Feedback {
+	return { evaluator, metric, score, kind, ...(comment ? { comment } : {}) };
+}
+
+describe('Score Calculator', () => {
+	describe('parseFeedbackKey()', () => {
+		it('should parse two-part key', () => {
+			const result = parseFeedbackKey('llm-judge.functionality');
+			expect(result).toEqual({
+				evaluator: 'llm-judge',
+				category: 'functionality',
+				subcategory: undefined,
+			});
+		});
+
+		it('should parse three-part key', () => {
+			const result = parseFeedbackKey('pairwise.gen1.majorityPass');
+			expect(result).toEqual({
+				evaluator: 'pairwise',
+				category: 'gen1',
+				subcategory: 'majorityPass',
+			});
+		});
+
+		it('should handle single-part key', () => {
+			const result = parseFeedbackKey('overall');
+			expect(result).toEqual({
+				evaluator: 'overall',
+				category: '',
+				subcategory: undefined,
+			});
+		});
+
+		it('should handle keys with more than three parts', () => {
+			const result = parseFeedbackKey('a.b.c.d.e');
+			expect(result).toEqual({
+				evaluator: 'a',
+				category: 'b',
+				subcategory: 'c',
+			});
+		});
+	});
+
+	describe('extractCategory()', () => {
+		it('should extract category from llm-judge key', () => {
+			expect(extractCategory('llm-judge.functionality')).toBe('functionality');
+		});
+
+		it('should extract category from programmatic key', () => {
+			expect(extractCategory('programmatic.trigger')).toBe('trigger');
+		});
+
+		it('should extract category from pairwise key', () => {
+			expect(extractCategory('pairwise.majorityPass')).toBe('majorityPass');
+		});
+
+		it('should return empty string for single-part key', () => {
+			expect(extractCategory('overall')).toBe('');
+		});
+
+		it('should extract first category from multi-part key', () => {
+			expect(extractCategory('pairwise.gen1.diagnosticScore')).toBe('gen1');
+		});
+	});
+
+	describe('groupByEvaluator()', () => {
+		it('should group feedback by evaluator prefix', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'functionality', 0.8),
+				createFeedback('llm-judge', 'connections', 0.9),
+				createFeedback('programmatic', 'trigger', 1.0),
+			];
+
+			const grouped = groupByEvaluator(feedback);
+
+			expect(Object.keys(grouped)).toHaveLength(2);
+			expect(grouped['llm-judge']).toHaveLength(2);
+			expect(grouped['programmatic']).toHaveLength(1);
+		});
+
+		it('should handle mixed evaluators', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'a', 0.5),
+				createFeedback('programmatic', 'b', 0.6),
+				createFeedback('pairwise', 'c', 0.7),
+				createFeedback('similarity', 'd', 0.8),
+			];
+
+			const grouped = groupByEvaluator(feedback);
+
+			expect(Object.keys(grouped)).toHaveLength(4);
+			expect(grouped['llm-judge']).toHaveLength(1);
+			expect(grouped['programmatic']).toHaveLength(1);
+			expect(grouped['pairwise']).toHaveLength(1);
+			expect(grouped['similarity']).toHaveLength(1);
+		});
+
+		it('should handle empty array', () => {
+			const grouped = groupByEvaluator([]);
+			expect(grouped).toEqual({});
+		});
+
+		it('should preserve feedback properties', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'test', 0.75, 'metric', 'Test comment'),
+			];
+
+			const grouped = groupByEvaluator(feedback);
+
+			expect(grouped['llm-judge'][0]).toEqual({
+				evaluator: 'llm-judge',
+				metric: 'test',
+				score: 0.75,
+				kind: 'metric',
+				comment: 'Test comment',
+			});
+		});
+	});
+
+	describe('calculateWeightedScore()', () => {
+		it('should use default weights', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'a', 1.0),
+				createFeedback('programmatic', 'b', 0.5),
+				createFeedback('pairwise', 'c', 0.5),
+				createFeedback('similarity', 'd', 0.5),
+			];
+
+			const score = calculateWeightedScore(feedback);
+
+			// llm-judge: 1.0 * 0.35 = 0.35
+			// programmatic: 0.5 * 0.25 = 0.125
+			// pairwise: 0.5 * 0.25 = 0.125
+			// similarity: 0.5 * 0.15 = 0.075
+			// Total: 0.675 / 1.0 = 0.675
+			expect(score).toBeCloseTo(0.675);
+		});
+
+		it('should use custom weights', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'a', 1.0),
+				createFeedback('programmatic', 'b', 0.0),
+			];
+
+			const score = calculateWeightedScore(feedback, {
+				'llm-judge': 0.8,
+				programmatic: 0.2,
+			});
+
+			// llm-judge: 1.0 * 0.8 = 0.8
+			// programmatic: 0.0 * 0.2 = 0.0
+			// Total: 0.8 / 1.0 = 0.8
+			expect(score).toBeCloseTo(0.8);
+		});
+
+		it('should handle missing evaluators with default weight', () => {
+			const feedback: Feedback[] = [createFeedback('unknown-evaluator', 'a', 0.5)];
+
+			const score = calculateWeightedScore(feedback);
+
+			// unknown-evaluator gets default weight of 0.1
+			expect(score).toBeCloseTo(0.5);
+		});
+
+		it('should return 0 for empty feedback', () => {
+			const score = calculateWeightedScore([]);
+			expect(score).toBe(0);
+		});
+
+		it('should average multiple items from same evaluator', () => {
+			const feedback: Feedback[] = [
+				{ ...createFeedback('llm-judge', 'a', 1.0), kind: 'metric' },
+				{ ...createFeedback('llm-judge', 'b', 0.5), kind: 'metric' },
+				{ ...createFeedback('llm-judge', 'c', 0.5), kind: 'metric' },
+			];
+
+			const score = calculateWeightedScore(feedback, { 'llm-judge': 1.0 });
+
+			// avg(1.0, 0.5, 0.5) = 0.666...
+			expect(score).toBeCloseTo(0.666, 2);
+		});
+
+		it('should ignore detail items when score items exist', () => {
+			const feedback: Feedback[] = [
+				{ evaluator: 'llm-judge', metric: 'overallScore', score: 0.8, kind: 'score' },
+				{
+					evaluator: 'llm-judge',
+					metric: 'efficiency.nodeCountEfficiency',
+					score: 0.0,
+					kind: 'detail',
+				},
+				{
+					evaluator: 'llm-judge',
+					metric: 'efficiency.pathOptimization',
+					score: 0.0,
+					kind: 'detail',
+				},
+			];
+
+			expect(calculateWeightedScore(feedback)).toBeCloseTo(0.8, 5);
+		});
+
+		it('should be invariant to extra detail keys', () => {
+			const base: Feedback[] = [
+				{ evaluator: 'pairwise', metric: 'pairwise_primary', score: 1, kind: 'score' },
+			];
+			const withDetails: Feedback[] = [
+				...base,
+				{ evaluator: 'pairwise', metric: 'judge1', score: 0, kind: 'detail' },
+				{ evaluator: 'pairwise', metric: 'judge2', score: 0, kind: 'detail' },
+			];
+
+			expect(calculateWeightedScore(base)).toBeCloseTo(calculateWeightedScore(withDetails), 10);
+		});
+
+		it('should normalize weights', () => {
+			const feedback: Feedback[] = [createFeedback('a', 'x', 1.0), createFeedback('b', 'x', 0.0)];
+
+			// Weights don't sum to 1.0
+			const score = calculateWeightedScore(feedback, {
+				a: 0.5,
+				b: 0.5,
+			});
+
+			// a: 1.0 * 0.5 = 0.5
+			// b: 0.0 * 0.5 = 0.0
+			// Total: 0.5 / 1.0 = 0.5
+			expect(score).toBeCloseTo(0.5);
+		});
+	});
+
+	describe('aggregateScores()', () => {
+		it('should calculate overall score', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'a', 0.8),
+				createFeedback('programmatic', 'b', 0.6),
+			];
+
+			const result = aggregateScores(feedback);
+
+			// llm-judge: 0.8 * 0.4 = 0.32
+			// programmatic: 0.6 * 0.3 = 0.18
+			// Total weight: 0.7, Total: 0.5 / 0.7 = 0.714...
+			expect(result.overall).toBeCloseTo(0.714, 2);
+		});
+
+		it('should calculate by-evaluator averages', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'a', 0.8),
+				createFeedback('llm-judge', 'b', 0.6),
+				createFeedback('programmatic', 'c', 1.0),
+			];
+
+			const result = aggregateScores(feedback);
+
+			expect(result.byEvaluator['llm-judge']).toBeCloseTo(0.7); // (0.8 + 0.6) / 2
+			expect(result.byEvaluator['programmatic']).toBeCloseTo(1.0);
+		});
+
+		it('should calculate by-category averages', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'functionality', 0.8),
+				createFeedback('llm-judge', 'connections', 0.6),
+				createFeedback('programmatic', 'trigger', 1.0),
+			];
+
+			const result = aggregateScores(feedback);
+
+			expect(result.byCategory['functionality']).toBeCloseTo(0.8);
+			expect(result.byCategory['connections']).toBeCloseTo(0.6);
+			expect(result.byCategory['trigger']).toBeCloseTo(1.0);
+		});
+
+		it('should average same categories from different evaluators', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'functionality', 0.8),
+				createFeedback('programmatic', 'functionality', 0.6),
+			];
+
+			const result = aggregateScores(feedback);
+
+			expect(result.byCategory['functionality']).toBeCloseTo(0.7); // (0.8 + 0.6) / 2
+		});
+
+		it('should handle empty feedback', () => {
+			const result = aggregateScores([]);
+
+			expect(result.overall).toBe(0);
+			expect(result.byEvaluator).toEqual({});
+			expect(result.byCategory).toEqual({});
+		});
+	});
+
+	describe('DEFAULT_EVALUATOR_WEIGHTS', () => {
+		it('should have weights for standard evaluators', () => {
+			expect(DEFAULT_EVALUATOR_WEIGHTS['llm-judge']).toBeDefined();
+			expect(DEFAULT_EVALUATOR_WEIGHTS['programmatic']).toBeDefined();
+			expect(DEFAULT_EVALUATOR_WEIGHTS['pairwise']).toBeDefined();
+		});
+
+		it('should have weights that sum to approximately 1', () => {
+			const sum = Object.values(DEFAULT_EVALUATOR_WEIGHTS).reduce((a, b) => a + b, 0);
+			expect(sum).toBeCloseTo(1.0);
+		});
+	});
+
+	describe('binary-checks weight', () => {
+		it('binary-checks evaluator has weight 0 in DEFAULT_EVALUATOR_WEIGHTS', () => {
+			expect(DEFAULT_EVALUATOR_WEIGHTS['binary-checks']).toBe(0);
+		});
+
+		it('binary-checks feedback does not affect weighted score', () => {
+			const feedback: Feedback[] = [
+				createFeedback('llm-judge', 'functionality', 0.8),
+				createFeedback('binary-checks', 'has_nodes', 1),
+				createFeedback('binary-checks', 'has_trigger', 0),
+			];
+			const scoreWithBinary = calculateWeightedScore(feedback);
+
+			const feedbackWithout: Feedback[] = [createFeedback('llm-judge', 'functionality', 0.8)];
+			const scoreWithout = calculateWeightedScore(feedbackWithout);
+
+			expect(scoreWithBinary).toBe(scoreWithout);
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/test-case-generator.test.ts

@@ -0,0 +1,273 @@
+/**
+ * Tests for test case generation.
+ *
+ * These utilities generate test cases for workflow evaluation,
+ * either via LLM or from CSV fixtures.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+
+import { loadDefaultTestCases } from '../cli/csv-prompt-loader';
+import { createTestCaseGenerator, type GeneratedTestCase } from '../support/test-case-generator';
+
+/** Type guard for message objects with content */
+function isMessageWithContent(msg: unknown): msg is { content: unknown } {
+	return msg !== null && typeof msg === 'object' && 'content' in msg;
+}
+
+/** Type guard for objects with _getType method */
+function hasGetTypeMethod(msg: unknown): msg is { _getType: () => string } {
+	if (msg === null || typeof msg !== 'object') return false;
+	if (!('_getType' in msg)) return false;
+	const obj = msg as { _getType: unknown };
+	return typeof obj._getType === 'function';
+}
+
+/** Helper to extract messages from mock invoke calls */
+function getMessagesFromMockCall(mockInvoke: jest.Mock): { system: string; human: string } {
+	const calls = mockInvoke.mock.calls;
+	if (calls.length === 0) throw new Error('No calls recorded');
+
+	const firstCall = calls[0];
+	if (!Array.isArray(firstCall) || firstCall.length === 0) {
+		throw new Error('First call has no arguments');
+	}
+
+	const messages = firstCall[0];
+	if (!Array.isArray(messages) || messages.length < 2) {
+		throw new Error('Messages array invalid');
+	}
+
+	const systemMsg = messages[0];
+	const humanMsg = messages[1];
+
+	// Type-safe content extraction
+	const getContent = (msg: unknown): string => {
+		if (isMessageWithContent(msg)) {
+			const content = msg.content;
+			if (typeof content === 'string') return content;
+		}
+		return '';
+	};
+
+	return {
+		system: getContent(systemMsg),
+		human: getContent(humanMsg),
+	};
+}
+
+describe('Test Case Generator', () => {
+	describe('createTestCaseGenerator()', () => {
+		let mockLlm: BaseChatModel;
+		let mockInvoke: jest.Mock;
+
+		beforeEach(() => {
+			mockInvoke = jest.fn().mockResolvedValue({ testCases: [] });
+			mockLlm = mock<BaseChatModel>();
+			(mockLlm as unknown as { withStructuredOutput: jest.Mock }).withStructuredOutput = jest
+				.fn()
+				.mockReturnValue({ invoke: mockInvoke });
+		});
+
+		it('should return generator with generate method', () => {
+			const generator = createTestCaseGenerator(mockLlm);
+
+			expect(generator).toHaveProperty('generate');
+			expect(typeof generator.generate).toBe('function');
+		});
+
+		it('should call LLM with structured output', async () => {
+			const generator = createTestCaseGenerator(mockLlm);
+			await generator.generate();
+
+			expect(
+				(mockLlm as unknown as { withStructuredOutput: jest.Mock }).withStructuredOutput,
+			).toHaveBeenCalled();
+			expect(mockInvoke).toHaveBeenCalled();
+		});
+
+		it('should include count in generated prompt', async () => {
+			const generator = createTestCaseGenerator(mockLlm, { count: 20 });
+			await generator.generate();
+
+			const { human } = getMessagesFromMockCall(mockInvoke);
+			expect(human).toContain('20');
+		});
+
+		it('should use default count of 10', async () => {
+			const generator = createTestCaseGenerator(mockLlm);
+			await generator.generate();
+
+			const { human } = getMessagesFromMockCall(mockInvoke);
+			expect(human).toContain('10');
+		});
+
+		it('should include custom focus in generated prompt', async () => {
+			const generator = createTestCaseGenerator(mockLlm, {
+				focus: 'API integrations only',
+			});
+			await generator.generate();
+
+			const { human } = getMessagesFromMockCall(mockInvoke);
+			expect(human).toContain('API integrations only');
+		});
+
+		it('should return properly typed test cases', async () => {
+			const mockTestCases: GeneratedTestCase[] = [
+				{
+					id: 'test_001',
+					name: 'Email Automation',
+					summary: 'Sends automated emails',
+					prompt: 'Create a workflow that sends emails',
+				},
+				{
+					id: 'test_002',
+					name: 'Data Processing',
+					summary: 'Processes CSV data',
+					prompt: 'Create a workflow that processes CSV files',
+				},
+			];
+			mockInvoke.mockResolvedValue({ testCases: mockTestCases });
+
+			const generator = createTestCaseGenerator(mockLlm);
+			const result = await generator.generate();
+
+			expect(result).toHaveLength(2);
+			expect(result[0]).toEqual({
+				id: 'test_001',
+				name: 'Email Automation',
+				summary: 'Sends automated emails',
+				prompt: 'Create a workflow that sends emails',
+			});
+		});
+
+		it('should handle LLM errors gracefully', async () => {
+			mockInvoke.mockRejectedValue(new Error('LLM error'));
+
+			const generator = createTestCaseGenerator(mockLlm);
+
+			await expect(generator.generate()).rejects.toThrow('LLM error');
+		});
+
+		it('should use complexity option in focus', async () => {
+			const generator = createTestCaseGenerator(mockLlm, { complexity: 'complex' });
+			await generator.generate();
+
+			const { human } = getMessagesFromMockCall(mockInvoke);
+			expect(human.toLowerCase()).toContain('complex');
+		});
+
+		it('should use simple complexity focus', async () => {
+			const generator = createTestCaseGenerator(mockLlm, { complexity: 'simple' });
+			await generator.generate();
+
+			const { human } = getMessagesFromMockCall(mockInvoke);
+			expect(human.toLowerCase()).toContain('simple');
+		});
+
+		it('should include system prompt in messages', async () => {
+			const generator = createTestCaseGenerator(mockLlm);
+			await generator.generate();
+
+			const calls = mockInvoke.mock.calls;
+			expect(calls.length).toBeGreaterThan(0);
+
+			// Extract messages from mock calls with proper type narrowing
+			const firstCall = calls[0];
+			if (!Array.isArray(firstCall) || firstCall.length === 0) {
+				throw new Error('Expected firstCall to be a non-empty array');
+			}
+			const firstArg = firstCall[0];
+			if (!Array.isArray(firstArg)) {
+				throw new Error('Expected first argument to be an array');
+			}
+			const messages = firstArg;
+			expect(messages).toHaveLength(2);
+
+			// Verify message types using type guard
+			const systemMsg = messages[0];
+			const humanMsg = messages[1];
+			expect(hasGetTypeMethod(systemMsg)).toBe(true);
+			expect(hasGetTypeMethod(humanMsg)).toBe(true);
+			if (hasGetTypeMethod(systemMsg)) {
+				expect(systemMsg._getType()).toBe('system');
+			}
+			if (hasGetTypeMethod(humanMsg)) {
+				expect(humanMsg._getType()).toBe('human');
+			}
+		});
+	});
+
+	describe('loadDefaultTestCases', () => {
+		it('should have at least 5 test cases', () => {
+			const defaultCases = loadDefaultTestCases();
+			expect(defaultCases.length).toBeGreaterThanOrEqual(5);
+		});
+
+		it('should have required properties on each test case', () => {
+			const defaultCases = loadDefaultTestCases();
+			for (const testCase of defaultCases) {
+				expect(testCase).toHaveProperty('id');
+				expect(testCase).toHaveProperty('prompt');
+				expect(typeof testCase.id).toBe('string');
+				expect(typeof testCase.prompt).toBe('string');
+				expect(testCase.id!.length).toBeGreaterThan(0);
+				expect(testCase.prompt.length).toBeGreaterThan(0);
+			}
+		});
+
+		it('should have unique IDs', () => {
+			const defaultCases = loadDefaultTestCases();
+			const ids = defaultCases.map((tc) => tc.id);
+			const uniqueIds = new Set(ids);
+			expect(uniqueIds.size).toBe(ids.length);
+		});
+
+		it('should cover different workflow types', () => {
+			const defaultCases = loadDefaultTestCases();
+			const prompts = defaultCases.map((tc) => tc.prompt.toLowerCase());
+
+			// Check for variety in test cases
+			const hasEmail = prompts.some((p) => p.includes('email'));
+			const hasApi = prompts.some((p) => p.includes('api') || p.includes('webhook'));
+			const hasData = prompts.some((p) => p.includes('data') || p.includes('process'));
+
+			expect(hasEmail || hasApi || hasData).toBe(true);
+		});
+
+		it('should have meaningful prompts', () => {
+			const defaultCases = loadDefaultTestCases();
+			for (const testCase of defaultCases) {
+				// Prompts should be descriptive enough
+				expect(testCase.prompt.length).toBeGreaterThan(20);
+			}
+		});
+	});
+
+	describe('generated test cases', () => {
+		it('should be compatible with v2 TestCase format', async () => {
+			const mockTestCases: GeneratedTestCase[] = [
+				{
+					id: 'gen_001',
+					name: 'Generated Test',
+					summary: 'A generated test case',
+					prompt: 'Create a workflow',
+				},
+			];
+
+			const mockInvoke = jest.fn().mockResolvedValue({ testCases: mockTestCases });
+			const mockLlm = mock<BaseChatModel>();
+			(mockLlm as unknown as { withStructuredOutput: jest.Mock }).withStructuredOutput = jest
+				.fn()
+				.mockReturnValue({ invoke: mockInvoke });
+
+			const generator = createTestCaseGenerator(mockLlm);
+			const generated = await generator.generate();
+
+			// Generated test cases should have id and prompt (compatible with v2 TestCase)
+			expect(generated[0].id).toBe('gen_001');
+			expect(generated[0].prompt).toBe('Create a workflow');
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/trace-filters.test.ts

@@ -0,0 +1,47 @@
+/**
+ * Tests for LangSmith trace filters.
+ */
+
+import type { KVMap } from 'langsmith/schemas';
+
+import type { EvalLogger } from '../harness/logger';
+import { createTraceFilters } from '../langsmith/trace-filters';
+
+describe('trace-filters', () => {
+	it('should not trim messages (keeps array) while still filtering other large state fields', () => {
+		const logs: string[] = [];
+		const logger: EvalLogger = {
+			isVerbose: true,
+			info: (m) => logs.push(m),
+			verbose: (m) => logs.push(m),
+			success: (m) => logs.push(m),
+			warn: (m) => logs.push(m),
+			error: (m) => logs.push(m),
+			dim: (m) => logs.push(m),
+		};
+
+		const { filterInputs } = createTraceFilters(logger);
+
+		const msg = { type: 'ai', content: 'hello' };
+		const input: KVMap = {
+			cachedTemplates: [
+				{
+					templateId: 't1',
+					name: 'Template',
+					// Extra properties that should be filtered out
+					workflow: { nodes: [], connections: {} },
+					description: 'A long description that should be removed',
+				},
+			],
+			messages: [msg],
+		};
+
+		const filtered = filterInputs({ ...input });
+
+		expect(Array.isArray(filtered.messages)).toBe(true);
+		expect(filtered.messages).toEqual([msg]);
+		// Verify that cachedTemplates was summarized - only templateId and name are preserved
+		expect(filtered.cachedTemplates).toEqual([{ templateId: 't1', name: 'Template' }]);
+		expect(logs.join('\n')).toContain('LangSmith trace filtering: ACTIVE');
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/webhook.test.ts

@@ -0,0 +1,709 @@
+/**
+ * Tests for webhook utilities.
+ */
+
+import { jsonParse } from 'n8n-workflow';
+
+import {
+	validateWebhookUrl,
+	sendWebhookNotification,
+	generateWebhookSignature,
+	verifyWebhookSignature,
+	WEBHOOK_SIGNATURE_HEADER,
+	WEBHOOK_TIMESTAMP_HEADER,
+	type WebhookPayload,
+} from '../cli/webhook';
+import type { RunSummary } from '../harness/harness-types';
+
+const mockFetch = jest.fn();
+global.fetch = mockFetch;
+
+jest.mock('node:dns/promises', () => ({
+	resolve: jest.fn().mockResolvedValue(['93.184.216.34']),
+	resolve6: jest.fn().mockRejectedValue(new Error('ENODATA')),
+}));
+
+/** Helper to create a mock logger */
+function createMockLogger() {
+	return {
+		info: jest.fn(),
+		error: jest.fn(),
+		warn: jest.fn(),
+		verbose: jest.fn(),
+		debug: jest.fn(),
+		success: jest.fn(),
+		dim: jest.fn(),
+		isVerbose: false,
+	};
+}
+
+/** Helper to create a mock run summary */
+function createMockSummary(overrides: Partial<RunSummary> = {}): RunSummary {
+	return {
+		totalExamples: 10,
+		passed: 8,
+		failed: 2,
+		errors: 0,
+		averageScore: 0.85,
+		totalDurationMs: 5000,
+		evaluatorAverages: {
+			'llm-judge': 0.85,
+			programmatic: 0.9,
+		},
+		langsmith: {
+			experimentName: 'test-experiment-2024',
+			experimentId: 'exp-uuid-123',
+			datasetId: 'dataset-uuid-456',
+		},
+		...overrides,
+	};
+}
+
+describe('webhook utilities', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	describe('generateWebhookSignature()', () => {
+		it('generates consistent signatures for same payload and secret', () => {
+			const payload = '{"test": "data"}';
+			const secret = 'test-secret-key-1234';
+
+			const sig1 = generateWebhookSignature(payload, secret);
+			const sig2 = generateWebhookSignature(payload, secret);
+
+			expect(sig1).toBe(sig2);
+			expect(sig1).toMatch(/^sha256=[a-f0-9]{64}$/);
+		});
+
+		it('generates different signatures for different payloads', () => {
+			const secret = 'test-secret-key-1234';
+
+			const sig1 = generateWebhookSignature('payload1', secret);
+			const sig2 = generateWebhookSignature('payload2', secret);
+
+			expect(sig1).not.toBe(sig2);
+		});
+
+		it('generates different signatures for different secrets', () => {
+			const payload = '{"test": "data"}';
+
+			const sig1 = generateWebhookSignature(payload, 'secret1');
+			const sig2 = generateWebhookSignature(payload, 'secret2');
+
+			expect(sig1).not.toBe(sig2);
+		});
+
+		it('generates known signature for test vector', () => {
+			// This provides a reference for receiver implementations
+			const payload = '{"suite":"llm-judge","summary":{"totalExamples":10}}';
+			const secret = 'test-webhook-secret-12345678';
+
+			const signature = generateWebhookSignature(payload, secret);
+
+			// Verify format
+			expect(signature).toMatch(/^sha256=[a-f0-9]{64}$/);
+
+			// This is the actual expected signature - useful for testing receiver implementations
+			expect(signature).toBe(
+				'sha256=0905f894294181ac73c6b2d61538c23dfbc5b023f4c2ab83513b1078a7fabe3c',
+			);
+		});
+	});
+
+	describe('verifyWebhookSignature()', () => {
+		it('returns true for valid signature', () => {
+			const payload = '{"test": "data"}';
+			const secret = 'test-secret-key-1234';
+			const signature = generateWebhookSignature(payload, secret);
+
+			expect(verifyWebhookSignature(payload, signature, secret)).toBe(true);
+		});
+
+		it('returns false for invalid signature', () => {
+			const payload = '{"test": "data"}';
+			const secret = 'test-secret-key-1234';
+
+			expect(verifyWebhookSignature(payload, 'sha256=invalid', secret)).toBe(false);
+		});
+
+		it('returns false for wrong secret', () => {
+			const payload = '{"test": "data"}';
+			const signature = generateWebhookSignature(payload, 'correct-secret');
+
+			expect(verifyWebhookSignature(payload, signature, 'wrong-secret')).toBe(false);
+		});
+
+		it('returns false for tampered payload', () => {
+			const secret = 'test-secret-key-1234';
+			const signature = generateWebhookSignature('original payload', secret);
+
+			expect(verifyWebhookSignature('tampered payload', signature, secret)).toBe(false);
+		});
+
+		it('returns false for signature with wrong length', () => {
+			const payload = '{"test": "data"}';
+			const secret = 'test-secret-key-1234';
+
+			expect(verifyWebhookSignature(payload, 'sha256=tooshort', secret)).toBe(false);
+		});
+	});
+
+	describe('validateWebhookUrl()', () => {
+		describe('protocol validation', () => {
+			it('accepts valid HTTPS URLs', () => {
+				expect(() => validateWebhookUrl('https://example.com/webhook')).not.toThrow();
+				expect(() => validateWebhookUrl('https://api.example.com/v1/hook')).not.toThrow();
+				expect(() =>
+					validateWebhookUrl('https://hooks.slack.com/services/T00/B00/xxx'),
+				).not.toThrow();
+			});
+
+			it('rejects HTTP URLs', () => {
+				expect(() => validateWebhookUrl('http://example.com/webhook')).toThrow(
+					'Webhook URL must use HTTPS',
+				);
+			});
+
+			it('rejects other protocols', () => {
+				expect(() => validateWebhookUrl('ftp://example.com/webhook')).toThrow(
+					'Webhook URL must use HTTPS',
+				);
+				expect(() => validateWebhookUrl('file:///etc/passwd')).toThrow(
+					'Webhook URL must use HTTPS',
+				);
+			});
+		});
+
+		describe('localhost blocking', () => {
+			it('rejects localhost', () => {
+				expect(() => validateWebhookUrl('https://localhost/webhook')).toThrow(
+					'Webhook URL cannot target localhost',
+				);
+				expect(() => validateWebhookUrl('https://localhost:3000/webhook')).toThrow(
+					'Webhook URL cannot target localhost',
+				);
+			});
+
+			it('rejects 127.0.0.1', () => {
+				expect(() => validateWebhookUrl('https://127.0.0.1/webhook')).toThrow(
+					'Webhook URL cannot target localhost',
+				);
+				expect(() => validateWebhookUrl('https://127.0.0.1:8080/webhook')).toThrow(
+					'Webhook URL cannot target localhost',
+				);
+			});
+
+			it('rejects IPv6 localhost (::1)', () => {
+				expect(() => validateWebhookUrl('https://[::1]/webhook')).toThrow(
+					'Webhook URL cannot target localhost',
+				);
+			});
+		});
+
+		describe('private IP blocking (SSRF prevention)', () => {
+			it('rejects 10.x.x.x addresses', () => {
+				expect(() => validateWebhookUrl('https://10.0.0.1/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://10.255.255.255/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://10.1.2.3:8080/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+			});
+
+			it('rejects 172.16-31.x.x addresses', () => {
+				expect(() => validateWebhookUrl('https://172.16.0.1/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://172.31.255.255/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://172.20.10.5/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+			});
+
+			it('allows 172.x.x.x addresses outside private range', () => {
+				expect(() => validateWebhookUrl('https://172.15.0.1/webhook')).not.toThrow();
+				expect(() => validateWebhookUrl('https://172.32.0.1/webhook')).not.toThrow();
+			});
+
+			it('rejects 192.168.x.x addresses', () => {
+				expect(() => validateWebhookUrl('https://192.168.0.1/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://192.168.1.100/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://192.168.255.255/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+			});
+
+			it('rejects 169.254.x.x link-local addresses', () => {
+				expect(() => validateWebhookUrl('https://169.254.0.1/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+				expect(() => validateWebhookUrl('https://169.254.169.254/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+			});
+
+			it('rejects 0.0.0.0', () => {
+				expect(() => validateWebhookUrl('https://0.0.0.0/webhook')).toThrow(
+					'Webhook URL cannot target private/internal IP addresses',
+				);
+			});
+		});
+
+		describe('internal hostname blocking', () => {
+			it('rejects "internal" hostname', () => {
+				expect(() => validateWebhookUrl('https://internal/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+				expect(() => validateWebhookUrl('https://api.internal/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+			});
+
+			it('rejects "intranet" hostname', () => {
+				expect(() => validateWebhookUrl('https://intranet/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+				expect(() => validateWebhookUrl('https://app.intranet/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+			});
+
+			it('rejects "corp" hostname', () => {
+				expect(() => validateWebhookUrl('https://corp/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+				expect(() => validateWebhookUrl('https://api.corp/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+			});
+
+			it('rejects "private" hostname', () => {
+				expect(() => validateWebhookUrl('https://private/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+				expect(() => validateWebhookUrl('https://services.private/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+			});
+
+			it('rejects "local" hostname', () => {
+				expect(() => validateWebhookUrl('https://local/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+				expect(() => validateWebhookUrl('https://api.local/webhook')).toThrow(
+					'Webhook URL cannot target internal hostname',
+				);
+			});
+
+			it('allows hostnames containing blocked words but not ending with them', () => {
+				expect(() => validateWebhookUrl('https://internal-api.example.com/webhook')).not.toThrow();
+				expect(() =>
+					validateWebhookUrl('https://my-intranet-app.example.com/webhook'),
+				).not.toThrow();
+			});
+		});
+
+		describe('valid URLs', () => {
+			it('accepts various valid public URLs', () => {
+				expect(() => validateWebhookUrl('https://hooks.slack.com/services/xxx')).not.toThrow();
+				expect(() => validateWebhookUrl('https://api.github.com/webhooks')).not.toThrow();
+				expect(() => validateWebhookUrl('https://webhook.site/unique-id')).not.toThrow();
+				expect(() => validateWebhookUrl('https://n8n.io/webhook/abc123')).not.toThrow();
+			});
+
+			it('accepts URLs with paths, query params, and ports', () => {
+				expect(() =>
+					validateWebhookUrl('https://example.com:8443/api/v1/webhook?token=abc'),
+				).not.toThrow();
+				expect(() =>
+					validateWebhookUrl('https://api.example.com/webhooks/eval/notify'),
+				).not.toThrow();
+			});
+		});
+
+		describe('invalid URL format', () => {
+			it('throws on invalid URL strings', () => {
+				expect(() => validateWebhookUrl('not-a-url')).toThrow();
+				expect(() => validateWebhookUrl('')).toThrow();
+				expect(() => validateWebhookUrl('://missing-protocol.com')).toThrow();
+			});
+		});
+	});
+
+	describe('sendWebhookNotification()', () => {
+		it('sends POST request with correct payload', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				status: 200,
+				statusText: 'OK',
+			});
+
+			const logger = createMockLogger();
+			const summary = createMockSummary();
+
+			await sendWebhookNotification({
+				webhookUrl: 'https://example.com/webhook',
+				summary,
+				dataset: 'test-dataset',
+				suite: 'llm-judge',
+				metadata: { source: 'ci', trigger: 'push' },
+				logger,
+			});
+
+			expect(mockFetch).toHaveBeenCalledTimes(1);
+			expect(mockFetch).toHaveBeenCalledWith('https://example.com/webhook', {
+				method: 'POST',
+				headers: {
+					'Content-Type': 'application/json',
+				},
+				body: expect.any(String),
+			});
+
+			// Verify payload structure
+			const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+			const payload = jsonParse<WebhookPayload>(callArgs[1].body as string);
+
+			expect(payload).toEqual({
+				suite: 'llm-judge',
+				summary: {
+					totalExamples: 10,
+					passed: 8,
+					failed: 2,
+					errors: 0,
+					averageScore: 0.85,
+				},
+				evaluatorAverages: {
+					'llm-judge': 0.85,
+					programmatic: 0.9,
+				},
+				totalDurationMs: 5000,
+				metadata: { source: 'ci', trigger: 'push' },
+				langsmith: {
+					experimentName: 'test-experiment-2024',
+					experimentId: 'exp-uuid-123',
+					datasetId: 'dataset-uuid-456',
+					datasetName: 'test-dataset',
+				},
+			});
+		});
+
+		it('logs info messages on success', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				status: 200,
+				statusText: 'OK',
+			});
+
+			const logger = createMockLogger();
+
+			await sendWebhookNotification({
+				webhookUrl: 'https://example.com/webhook',
+				summary: createMockSummary(),
+				dataset: 'test-dataset',
+				suite: 'llm-judge',
+				metadata: {},
+				logger,
+			});
+
+			expect(logger.info).toHaveBeenCalledWith(
+				'Sending results to webhook: https://example.com/***',
+			);
+			expect(logger.info).toHaveBeenCalledWith(
+				'Webhook notification sent successfully (status: 200)',
+			);
+		});
+
+		it('throws error when webhook URL validation fails', async () => {
+			const logger = createMockLogger();
+
+			await expect(
+				sendWebhookNotification({
+					webhookUrl: 'http://example.com/webhook', // HTTP not allowed
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				}),
+			).rejects.toThrow('Webhook URL must use HTTPS');
+
+			expect(mockFetch).not.toHaveBeenCalled();
+		});
+
+		it('throws error when fetch returns non-ok response', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: false,
+				status: 500,
+				statusText: 'Internal Server Error',
+			});
+
+			const logger = createMockLogger();
+
+			await expect(
+				sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				}),
+			).rejects.toThrow('Webhook request failed: 500 Internal Server Error');
+		});
+
+		it('throws error when fetch returns 4xx response', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: false,
+				status: 401,
+				statusText: 'Unauthorized',
+			});
+
+			const logger = createMockLogger();
+
+			await expect(
+				sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				}),
+			).rejects.toThrow('Webhook request failed: 401 Unauthorized');
+		});
+
+		it('throws error when fetch fails with network error', async () => {
+			mockFetch.mockRejectedValueOnce(new Error('Network error'));
+
+			const logger = createMockLogger();
+
+			await expect(
+				sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				}),
+			).rejects.toThrow('Network error');
+		});
+
+		it('handles summary without evaluatorAverages', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				status: 200,
+				statusText: 'OK',
+			});
+
+			const logger = createMockLogger();
+			const summary = createMockSummary({ evaluatorAverages: undefined });
+
+			await sendWebhookNotification({
+				webhookUrl: 'https://example.com/webhook',
+				summary,
+				dataset: 'test-dataset',
+				suite: 'llm-judge',
+				metadata: {},
+				logger,
+			});
+
+			const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+			const payload = jsonParse<WebhookPayload>(callArgs[1].body as string);
+
+			expect(payload.evaluatorAverages).toBeUndefined();
+		});
+
+		it('handles summary without langsmith data (local mode)', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				status: 200,
+				statusText: 'OK',
+			});
+
+			const logger = createMockLogger();
+			const summary = createMockSummary({ langsmith: undefined });
+
+			await sendWebhookNotification({
+				webhookUrl: 'https://example.com/webhook',
+				summary,
+				dataset: 'test-dataset',
+				suite: 'llm-judge',
+				metadata: {},
+				logger,
+			});
+
+			const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+			const payload = jsonParse<WebhookPayload>(callArgs[1].body as string);
+
+			expect(payload.langsmith).toBeUndefined();
+		});
+
+		it('validates URL before making fetch request (SSRF prevention)', async () => {
+			const logger = createMockLogger();
+
+			await expect(
+				sendWebhookNotification({
+					webhookUrl: 'https://192.168.1.1/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				}),
+			).rejects.toThrow('Webhook URL cannot target private/internal IP addresses');
+
+			expect(mockFetch).not.toHaveBeenCalled();
+		});
+
+		describe('HMAC signature', () => {
+			it('includes signature headers when webhookSecret is provided', async () => {
+				mockFetch.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+					statusText: 'OK',
+				});
+
+				const logger = createMockLogger();
+
+				await sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					webhookSecret: 'test-secret-key-1234',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				});
+
+				expect(mockFetch).toHaveBeenCalledTimes(1);
+				const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+				const headers = callArgs[1].headers as Record<string, string>;
+
+				// Verify signature header is present and valid format
+				expect(headers[WEBHOOK_SIGNATURE_HEADER]).toMatch(/^sha256=[a-f0-9]{64}$/);
+
+				// Verify timestamp header is present and valid
+				expect(headers[WEBHOOK_TIMESTAMP_HEADER]).toMatch(/^\d+$/);
+				const timestamp = parseInt(headers[WEBHOOK_TIMESTAMP_HEADER], 10);
+				expect(timestamp).toBeGreaterThan(Date.now() - 60000); // Within last minute
+				expect(timestamp).toBeLessThanOrEqual(Date.now());
+			});
+
+			it('signature can be verified by receiver', async () => {
+				mockFetch.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+					statusText: 'OK',
+				});
+
+				const logger = createMockLogger();
+				const secret = 'test-secret-key-1234';
+
+				await sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					webhookSecret: secret,
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				});
+
+				const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+				const headers = callArgs[1].headers as Record<string, string>;
+				const body = callArgs[1].body as string;
+
+				const signature = headers[WEBHOOK_SIGNATURE_HEADER];
+				const timestamp = headers[WEBHOOK_TIMESTAMP_HEADER];
+
+				// Verify that the receiver can validate the signature
+				// Signature is computed as: timestamp.body
+				const signaturePayload = `${timestamp}.${body}`;
+				expect(verifyWebhookSignature(signaturePayload, signature, secret)).toBe(true);
+			});
+
+			it('does not include signature headers when webhookSecret is not provided', async () => {
+				mockFetch.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+					statusText: 'OK',
+				});
+
+				const logger = createMockLogger();
+
+				await sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				});
+
+				const callArgs = mockFetch.mock.calls[0] as [string, RequestInit];
+				const headers = callArgs[1].headers as Record<string, string>;
+
+				expect(headers[WEBHOOK_SIGNATURE_HEADER]).toBeUndefined();
+				expect(headers[WEBHOOK_TIMESTAMP_HEADER]).toBeUndefined();
+			});
+
+			it('logs warning when webhook secret is not provided', async () => {
+				mockFetch.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+					statusText: 'OK',
+				});
+
+				const logger = createMockLogger();
+
+				await sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				});
+
+				expect(logger.warn).toHaveBeenCalledWith(
+					expect.stringContaining('No webhook secret provided'),
+				);
+			});
+
+			it('logs info when request is signed', async () => {
+				mockFetch.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+					statusText: 'OK',
+				});
+
+				const logger = createMockLogger();
+
+				await sendWebhookNotification({
+					webhookUrl: 'https://example.com/webhook',
+					webhookSecret: 'test-secret-key-1234',
+					summary: createMockSummary(),
+					dataset: 'test-dataset',
+					suite: 'llm-judge',
+					metadata: {},
+					logger,
+				});
+
+				expect(logger.info).toHaveBeenCalledWith(
+					expect.stringContaining('signed with HMAC-SHA256'),
+				);
+			});
+		});
+	});
+});