first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/__tests__/csv-writer.test.ts

@@ -0,0 +1,436 @@
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { writeResultsCsv } from '../csv-writer';
+import type { ExampleResult } from '../harness-types';
+
+describe('writeResultsCsv', () => {
+	let tempDir: string;
+
+	beforeEach(() => {
+		tempDir = mkdtempSync(join(tmpdir(), 'csv-writer-test-'));
+	});
+
+	afterEach(() => {
+		rmSync(tempDir, { recursive: true, force: true });
+	});
+
+	it('writes sorted results with correct columns', () => {
+		const results: ExampleResult[] = [
+			{
+				index: 2,
+				prompt: 'Zebra workflow',
+				status: 'pass',
+				score: 0.9,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'functionality',
+						score: 0.95,
+						kind: 'metric',
+						comment: '',
+					},
+					{
+						evaluator: 'llm-judge',
+						metric: 'connections',
+						score: 0.85,
+						kind: 'metric',
+						comment: 'Minor issue',
+					},
+				],
+				durationMs: 5000,
+				generationDurationMs: 3000,
+				generationInputTokens: 1000,
+				generationOutputTokens: 500,
+			},
+			{
+				index: 1,
+				prompt: 'Alpha workflow',
+				status: 'fail',
+				score: 0.6,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'functionality',
+						score: 0.5,
+						kind: 'metric',
+						comment: '[CRITICAL] Missing trigger',
+					},
+				],
+				durationMs: 4000,
+				generationDurationMs: 2500,
+				generationInputTokens: 800,
+				generationOutputTokens: 400,
+			},
+		];
+
+		const outputPath = join(tempDir, 'results.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Check header
+		expect(lines[0]).toContain('prompt,overall_score,status,gen_latency_ms');
+		expect(lines[0]).toContain('functionality,functionality_detail');
+
+		// Check sorting (Alpha before Zebra)
+		expect(lines[1]).toContain('Alpha workflow');
+		expect(lines[2]).toContain('Zebra workflow');
+
+		// Check violation text is included
+		expect(lines[1]).toContain('[CRITICAL] Missing trigger');
+	});
+
+	it.each([
+		{
+			description: 'commas and quotes',
+			prompt: 'Workflow with "quotes" and, commas',
+			expected: '"Workflow with ""quotes"" and, commas"',
+		},
+		{
+			description: 'newlines',
+			prompt: 'Workflow with\nnewline',
+			expected: '"Workflow with\nnewline"',
+		},
+	])('escapes $description in values', ({ prompt, expected }) => {
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt,
+				status: 'pass',
+				score: 0.8,
+				feedback: [],
+				durationMs: 1000,
+			},
+		];
+
+		const outputPath = join(tempDir, 'results.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		expect(content).toContain(expected);
+	});
+
+	it('writes pairwise evaluation results with correct columns', () => {
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Test pairwise workflow',
+				status: 'fail',
+				score: 0.5,
+				feedback: [
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_primary',
+						score: 0,
+						kind: 'score',
+						comment: '0/3 judges passed',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_diagnostic',
+						score: 0.67,
+						kind: 'metric',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_judges_passed',
+						score: 0,
+						kind: 'detail',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_total_passes',
+						score: 6,
+						kind: 'detail',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_total_violations',
+						score: 3,
+						kind: 'detail',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'judge1',
+						score: 0,
+						kind: 'detail',
+						comment: '[Spec violation] Missing required field',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'judge2',
+						score: 1,
+						kind: 'detail',
+						comment: '',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'judge3',
+						score: 0,
+						kind: 'detail',
+						comment: '[Spec violation] Wrong parameter value',
+					},
+				],
+				durationMs: 5000,
+				generationDurationMs: 3000,
+				generationInputTokens: 1000,
+				generationOutputTokens: 500,
+			},
+		];
+
+		const outputPath = join(tempDir, 'pairwise-results.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Check header has pairwise columns
+		expect(lines[0]).toContain('prompt,overall_score,status,gen_latency_ms');
+		expect(lines[0]).toContain('pairwise_primary');
+		expect(lines[0]).toContain('pairwise_diagnostic');
+		expect(lines[0]).toContain('pairwise_judges_passed');
+		expect(lines[0]).toContain('pairwise_total_passes');
+		expect(lines[0]).toContain('pairwise_total_violations');
+		expect(lines[0]).toContain('judge1,judge1_detail');
+		expect(lines[0]).toContain('judge2,judge2_detail');
+		expect(lines[0]).toContain('judge3,judge3_detail');
+
+		// Check data row contains judge violation details
+		expect(lines[1]).toContain('[Spec violation] Missing required field');
+		expect(lines[1]).toContain('[Spec violation] Wrong parameter value');
+	});
+
+	it('handles empty results array', () => {
+		const results: ExampleResult[] = [];
+
+		const outputPath = join(tempDir, 'empty-results.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		expect(content).toBe('');
+	});
+
+	it('includes subgraph metrics columns (node_count, discovery_latency_ms, builder_latency_ms)', () => {
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Test workflow with subgraph metrics',
+				status: 'pass',
+				score: 0.9,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'functionality',
+						score: 0.95,
+						kind: 'metric',
+						comment: '',
+					},
+				],
+				durationMs: 5000,
+				generationDurationMs: 3000,
+				generationInputTokens: 1000,
+				generationOutputTokens: 500,
+				subgraphMetrics: {
+					nodeCount: 8,
+					discoveryDurationMs: 450,
+					builderDurationMs: 1200,
+				},
+			},
+			{
+				index: 2,
+				prompt: 'Test workflow without subgraph metrics',
+				status: 'pass',
+				score: 0.8,
+				feedback: [
+					{
+						evaluator: 'llm-judge',
+						metric: 'functionality',
+						score: 0.8,
+						kind: 'metric',
+						comment: '',
+					},
+				],
+				durationMs: 4000,
+				generationDurationMs: 2500,
+				generationInputTokens: 900,
+				generationOutputTokens: 450,
+				// No subgraphMetrics
+			},
+		];
+
+		const outputPath = join(tempDir, 'subgraph-metrics.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Check header includes subgraph metrics columns
+		expect(lines[0]).toContain('node_count');
+		expect(lines[0]).toContain('discovery_latency_ms');
+		expect(lines[0]).toContain('builder_latency_ms');
+
+		// Check data rows (sorted by prompt alphabetically: "with" < "without")
+		// First row: with metrics (should contain the values)
+		expect(lines[1]).toContain('Test workflow with subgraph metrics');
+		expect(lines[1]).toContain('8'); // nodeCount
+		expect(lines[1]).toContain('450'); // discoveryDurationMs
+		expect(lines[1]).toContain('1200'); // builderDurationMs
+
+		// Second row: without metrics (empty values)
+		expect(lines[2]).toContain('Test workflow without subgraph metrics');
+	});
+
+	it('includes subgraph metrics in pairwise format', () => {
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Pairwise with metrics',
+				status: 'pass',
+				score: 0.9,
+				feedback: [
+					{
+						evaluator: 'pairwise',
+						metric: 'pairwise_primary',
+						score: 1,
+						kind: 'score',
+					},
+					{
+						evaluator: 'pairwise',
+						metric: 'judge1',
+						score: 1,
+						kind: 'detail',
+					},
+				],
+				durationMs: 5000,
+				generationDurationMs: 3000,
+				generationInputTokens: 1000,
+				generationOutputTokens: 500,
+				subgraphMetrics: {
+					nodeCount: 5,
+					discoveryDurationMs: 300,
+					builderDurationMs: 800,
+				},
+			},
+		];
+
+		const outputPath = join(tempDir, 'pairwise-with-metrics.csv');
+		writeResultsCsv(results, outputPath);
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Check header includes subgraph metrics columns for pairwise format too
+		expect(lines[0]).toContain('node_count');
+		expect(lines[0]).toContain('discovery_latency_ms');
+		expect(lines[0]).toContain('builder_latency_ms');
+
+		// Check data row contains the metrics
+		expect(lines[1]).toContain('5'); // nodeCount
+		expect(lines[1]).toContain('300'); // discoveryDurationMs
+		expect(lines[1]).toContain('800'); // builderDurationMs
+	});
+
+	it('uses explicit suite option to override auto-detection', () => {
+		// Results with runner errors (no pairwise feedback) should still use pairwise format
+		// when suite is explicitly specified
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Failed during generation',
+				status: 'error',
+				score: 0,
+				feedback: [
+					{
+						evaluator: 'runner', // This would normally trigger unknown/llm-judge format
+						metric: 'error',
+						score: 0,
+						kind: 'score',
+						comment: 'Generation failed',
+					},
+				],
+				durationMs: 1000,
+				generationDurationMs: 500,
+				error: 'Generation failed',
+			},
+		];
+
+		const outputPath = join(tempDir, 'explicit-pairwise-suite.csv');
+		writeResultsCsv(results, outputPath, { suite: 'pairwise' });
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Should use pairwise format headers despite having runner feedback
+		expect(lines[0]).toContain('pairwise_primary');
+		expect(lines[0]).toContain('pairwise_diagnostic');
+		// Should NOT have llm-judge format headers
+		expect(lines[0]).not.toContain('functionality_detail');
+	});
+
+	it('writes binary-checks format with correct columns', () => {
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Test workflow',
+				status: 'pass',
+				score: 0,
+				feedback: [
+					{ evaluator: 'binary-checks', metric: 'has_nodes', score: 1, kind: 'metric' as const },
+					{ evaluator: 'binary-checks', metric: 'has_trigger', score: 0, kind: 'metric' as const },
+				],
+				durationMs: 1000,
+				generationDurationMs: 500,
+			},
+		];
+
+		const outputPath = join(tempDir, 'binary-results.csv');
+		writeResultsCsv(results, outputPath, { suite: 'binary-checks' });
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Check header includes binary check names
+		expect(lines[0]).toContain('prompt,status,gen_latency_ms');
+		expect(lines[0]).toContain('has_nodes');
+		expect(lines[0]).toContain('has_trigger');
+		expect(lines[0]).toContain('descriptive_node_names');
+
+		// Check data row
+		expect(lines[1]).toContain('Test workflow');
+	});
+
+	it('falls back to auto-detection when suite option is not provided', () => {
+		// Results with runner errors should fall back to llm-judge format when no suite specified
+		const results: ExampleResult[] = [
+			{
+				index: 1,
+				prompt: 'Failed during generation',
+				status: 'error',
+				score: 0,
+				feedback: [
+					{
+						evaluator: 'runner',
+						metric: 'error',
+						score: 0,
+						kind: 'score',
+					},
+				],
+				durationMs: 1000,
+				generationDurationMs: 500,
+			},
+		];
+
+		const outputPath = join(tempDir, 'auto-detect-suite.csv');
+		writeResultsCsv(results, outputPath); // No suite option
+
+		const content = readFileSync(outputPath, 'utf-8');
+		const lines = content.trim().split('\n');
+
+		// Should fall back to llm-judge format when no suite detected
+		expect(lines[0]).toContain('functionality');
+		expect(lines[0]).toContain('functionality_detail');
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/__tests__/token-tracking-handler.test.ts

@@ -0,0 +1,139 @@
+import type { LLMResult } from '@langchain/core/outputs';
+
+import { TokenUsageTrackingHandler } from '../token-tracking-handler';
+
+describe('TokenUsageTrackingHandler', () => {
+	let handler: TokenUsageTrackingHandler;
+
+	beforeEach(() => {
+		handler = new TokenUsageTrackingHandler();
+	});
+
+	describe('handleLLMEnd', () => {
+		it.each([
+			{
+				format: 'Anthropic (input_tokens/output_tokens)',
+				usage: { input_tokens: 100, output_tokens: 50 },
+				expected: { inputTokens: 100, outputTokens: 50 },
+			},
+			{
+				format: 'OpenAI (prompt_tokens/completion_tokens)',
+				usage: { prompt_tokens: 150, completion_tokens: 75 },
+				expected: { inputTokens: 150, outputTokens: 75 },
+			},
+		])(
+			'should accumulate tokens from llmOutput.usage in $format format',
+			async ({ usage, expected }) => {
+				const result: LLMResult = {
+					generations: [[]],
+					llmOutput: { usage },
+				};
+
+				await handler.handleLLMEnd(result);
+
+				expect(handler.getUsage()).toEqual(expected);
+			},
+		);
+
+		it('should accumulate tokens from multiple LLM calls', async () => {
+			const result1: LLMResult = {
+				generations: [[]],
+				llmOutput: {
+					usage: { input_tokens: 100, output_tokens: 50 },
+				},
+			};
+
+			const result2: LLMResult = {
+				generations: [[]],
+				llmOutput: {
+					usage: { input_tokens: 200, output_tokens: 100 },
+				},
+			};
+
+			await handler.handleLLMEnd(result1);
+			await handler.handleLLMEnd(result2);
+
+			expect(handler.getUsage()).toEqual({
+				inputTokens: 300,
+				outputTokens: 150,
+			});
+		});
+
+		it.each([
+			{
+				format: 'Anthropic (input_tokens/output_tokens)',
+				usage: { input_tokens: 75, output_tokens: 25 },
+				expected: { inputTokens: 75, outputTokens: 25 },
+			},
+			{
+				format: 'OpenAI (prompt_tokens/completion_tokens)',
+				usage: { prompt_tokens: 80, completion_tokens: 40 },
+				expected: { inputTokens: 80, outputTokens: 40 },
+			},
+		])(
+			'should extract tokens from generationInfo in $format format when llmOutput is empty',
+			async ({ usage, expected }) => {
+				const result: LLMResult = {
+					generations: [
+						[
+							{
+								text: 'response',
+								generationInfo: { usage },
+							},
+						],
+					],
+				};
+
+				await handler.handleLLMEnd(result);
+
+				expect(handler.getUsage()).toEqual(expected);
+			},
+		);
+
+		it('should handle missing usage data gracefully', async () => {
+			const result: LLMResult = {
+				generations: [[{ text: 'response' }]],
+			};
+
+			await handler.handleLLMEnd(result);
+
+			expect(handler.getUsage()).toEqual({
+				inputTokens: 0,
+				outputTokens: 0,
+			});
+		});
+	});
+
+	describe('reset', () => {
+		it('should reset accumulated usage to zero', async () => {
+			const result: LLMResult = {
+				generations: [[]],
+				llmOutput: {
+					usage: {
+						input_tokens: 100,
+						output_tokens: 50,
+					},
+				},
+			};
+
+			await handler.handleLLMEnd(result);
+			expect(handler.getUsage().inputTokens).toBe(100);
+
+			handler.reset();
+
+			expect(handler.getUsage()).toEqual({
+				inputTokens: 0,
+				outputTokens: 0,
+			});
+		});
+	});
+
+	describe('getUsage', () => {
+		it('should return zero for new handler', () => {
+			expect(handler.getUsage()).toEqual({
+				inputTokens: 0,
+				outputTokens: 0,
+			});
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/csv-writer.ts

@@ -0,0 +1,307 @@
+import { writeFileSync } from 'node:fs';
+
+import type { ExampleResult, Feedback } from './harness-types';
+import { DETERMINISTIC_CHECKS } from '../evaluators/binary-checks/checks';
+import { LLM_CHECKS } from '../evaluators/binary-checks/llm-checks';
+
+/**
+ * Fixed columns that appear first in the CSV (in order).
+ */
+const FIXED_COLUMNS = [
+	'prompt',
+	'overall_score',
+	'status',
+	'gen_latency_ms',
+	'gen_input_tokens',
+	'gen_output_tokens',
+	'node_count',
+	'discovery_latency_ms',
+	'builder_latency_ms',
+	'responder_latency_ms',
+] as const;
+
+/**
+ * LLM Judge metrics to include (in order).
+ * Each metric gets a score column and a _detail column.
+ */
+const LLM_JUDGE_METRICS = [
+	'functionality',
+	'connections',
+	'expressions',
+	'nodeConfiguration',
+	'efficiency',
+	'dataFlow',
+	'maintainability',
+	'bestPractices',
+] as const;
+
+/**
+ * Pairwise evaluator metrics.
+ */
+const PAIRWISE_METRICS = [
+	'pairwise_primary',
+	'pairwise_diagnostic',
+	'pairwise_judges_passed',
+	'pairwise_total_passes',
+	'pairwise_total_violations',
+] as const;
+
+/**
+ * Binary check names derived from the check registries.
+ * Order: deterministic checks first, then LLM checks.
+ */
+const BINARY_CHECK_NAMES = [
+	...DETERMINISTIC_CHECKS.map((c) => c.name),
+	...LLM_CHECKS.map((c) => c.name),
+];
+
+type EvaluationSuite = 'llm-judge' | 'pairwise' | 'binary-checks' | 'unknown';
+
+/**
+ * Escape a value for CSV output.
+ * Wraps in quotes if contains comma, quote, or newline.
+ */
+function escapeCsvValue(value: string | number | undefined): string {
+	if (value === undefined || value === null) return '';
+	const str = String(value);
+	if (str.includes(',') || str.includes('"') || str.includes('\n')) {
+		return `"${str.replace(/"/g, '""')}"`;
+	}
+	return str;
+}
+
+/**
+ * Detect the evaluation suite from feedback.
+ */
+function detectSuite(feedback: Feedback[]): EvaluationSuite {
+	if (feedback.some((f) => f.evaluator === 'llm-judge')) {
+		return 'llm-judge';
+	}
+	if (feedback.some((f) => f.evaluator === 'pairwise')) {
+		return 'pairwise';
+	}
+	if (feedback.some((f) => f.evaluator === 'binary-checks')) {
+		return 'binary-checks';
+	}
+	return 'unknown';
+}
+
+/**
+ * Extract the detail text from feedback for a given evaluator and metric.
+ */
+function extractMetricDetail(feedback: Feedback[], evaluator: string, metric: string): string {
+	const item = feedback.find((f) => f.evaluator === evaluator && f.metric === metric && f.comment);
+	return item?.comment ?? '';
+}
+
+/**
+ * Extract the score for a given evaluator and metric.
+ */
+function extractMetricScore(
+	feedback: Feedback[],
+	evaluator: string,
+	metric: string,
+): number | undefined {
+	const item = feedback.find((f) => f.evaluator === evaluator && f.metric === metric);
+	return item?.score;
+}
+
+/**
+ * Get the number of judges used in pairwise evaluation.
+ */
+function getJudgeCount(feedback: Feedback[]): number {
+	const judgeMetrics = feedback.filter(
+		(f) => f.evaluator === 'pairwise' && f.metric.startsWith('judge'),
+	);
+	return judgeMetrics.length;
+}
+
+/**
+ * Build CSV row for LLM Judge suite.
+ */
+function buildLlmJudgeRow(result: ExampleResult): string[] {
+	const row: string[] = [];
+
+	// Fixed columns
+	row.push(escapeCsvValue(result.prompt));
+	row.push(escapeCsvValue(result.score));
+	row.push(escapeCsvValue(result.status));
+	row.push(escapeCsvValue(result.generationDurationMs));
+	row.push(escapeCsvValue(result.generationInputTokens));
+	row.push(escapeCsvValue(result.generationOutputTokens));
+	row.push(escapeCsvValue(result.subgraphMetrics?.nodeCount));
+	row.push(escapeCsvValue(result.subgraphMetrics?.discoveryDurationMs));
+	row.push(escapeCsvValue(result.subgraphMetrics?.builderDurationMs));
+	row.push(escapeCsvValue(result.subgraphMetrics?.responderDurationMs));
+
+	// LLM Judge metric columns (score + detail pairs)
+	for (const metric of LLM_JUDGE_METRICS) {
+		row.push(escapeCsvValue(extractMetricScore(result.feedback, 'llm-judge', metric)));
+		row.push(escapeCsvValue(extractMetricDetail(result.feedback, 'llm-judge', metric)));
+	}
+
+	return row;
+}
+
+/**
+ * Build CSV row for Pairwise suite.
+ */
+function buildPairwiseRow(result: ExampleResult, judgeCount: number): string[] {
+	const row: string[] = [];
+
+	// Fixed columns
+	row.push(escapeCsvValue(result.prompt));
+	row.push(escapeCsvValue(result.score));
+	row.push(escapeCsvValue(result.status));
+	row.push(escapeCsvValue(result.generationDurationMs));
+	row.push(escapeCsvValue(result.generationInputTokens));
+	row.push(escapeCsvValue(result.generationOutputTokens));
+	row.push(escapeCsvValue(result.subgraphMetrics?.nodeCount));
+	row.push(escapeCsvValue(result.subgraphMetrics?.discoveryDurationMs));
+	row.push(escapeCsvValue(result.subgraphMetrics?.builderDurationMs));
+	row.push(escapeCsvValue(result.subgraphMetrics?.responderDurationMs));
+
+	// Pairwise metrics (scores only, no detail)
+	for (const metric of PAIRWISE_METRICS) {
+		row.push(escapeCsvValue(extractMetricScore(result.feedback, 'pairwise', metric)));
+	}
+
+	// Individual judge results (score + violation detail)
+	for (let i = 1; i <= judgeCount; i++) {
+		const judgeMetric = `judge${i}`;
+		row.push(escapeCsvValue(extractMetricScore(result.feedback, 'pairwise', judgeMetric)));
+		row.push(escapeCsvValue(extractMetricDetail(result.feedback, 'pairwise', judgeMetric)));
+	}
+
+	return row;
+}
+
+/**
+ * Build CSV row for binary-checks suite.
+ */
+function buildBinaryChecksRow(result: ExampleResult): string[] {
+	const row: string[] = [];
+
+	row.push(escapeCsvValue(result.prompt));
+	row.push(escapeCsvValue(result.status));
+	row.push(escapeCsvValue(result.generationDurationMs));
+
+	for (const checkName of BINARY_CHECK_NAMES) {
+		row.push(escapeCsvValue(extractMetricScore(result.feedback, 'binary-checks', checkName)));
+	}
+
+	return row;
+}
+
+/**
+ * Build CSV header for LLM Judge suite.
+ */
+function buildLlmJudgeHeader(): string[] {
+	const header: string[] = [...FIXED_COLUMNS];
+
+	for (const metric of LLM_JUDGE_METRICS) {
+		header.push(metric);
+		header.push(`${metric}_detail`);
+	}
+
+	return header;
+}
+
+/**
+ * Build CSV header for Pairwise suite.
+ */
+function buildPairwiseHeader(judgeCount: number): string[] {
+	const header: string[] = [...FIXED_COLUMNS];
+
+	// Pairwise metrics
+	for (const metric of PAIRWISE_METRICS) {
+		header.push(metric);
+	}
+
+	// Individual judge columns
+	for (let i = 1; i <= judgeCount; i++) {
+		header.push(`judge${i}`);
+		header.push(`judge${i}_detail`);
+	}
+
+	return header;
+}
+
+/**
+ * Build CSV header for binary-checks suite.
+ */
+function buildBinaryChecksHeader(): string[] {
+	return ['prompt', 'status', 'gen_latency_ms', ...BINARY_CHECK_NAMES];
+}
+
+export interface WriteResultsCsvOptions {
+	/** Explicitly specify the evaluation suite. If not provided, auto-detects from feedback. */
+	suite?: 'llm-judge' | 'pairwise' | 'binary-checks';
+}
+
+/**
+ * Write evaluation results to a CSV file.
+ * Results are sorted by prompt for consistent ordering across runs.
+ * Automatically detects evaluation suite and formats accordingly,
+ * unless explicitly specified via options.
+ */
+export function writeResultsCsv(
+	results: ExampleResult[],
+	outputPath: string,
+	options?: WriteResultsCsvOptions,
+): void {
+	if (results.length === 0) {
+		writeFileSync(outputPath, '', 'utf-8');
+		return;
+	}
+
+	// Sort by prompt for consistent ordering
+	const sorted = [...results].sort((a, b) => a.prompt.localeCompare(b.prompt));
+
+	// Use explicit suite if provided, otherwise detect from feedback
+	let suite: EvaluationSuite;
+	if (options?.suite) {
+		suite = options.suite;
+	} else {
+		// Detect suite from first result with feedback (excluding runner errors)
+		const firstWithFeedback = sorted.find((r) =>
+			r.feedback.some((f) => f.evaluator !== 'runner' && f.evaluator !== 'programmatic'),
+		);
+		suite = firstWithFeedback ? detectSuite(firstWithFeedback.feedback) : 'unknown';
+	}
+
+	const lines: string[] = [];
+
+	if (suite === 'pairwise') {
+		// Determine max judge count across all results
+		const judgeCount = Math.max(...sorted.map((r) => getJudgeCount(r.feedback)), 0);
+
+		// Header
+		lines.push(buildPairwiseHeader(judgeCount).join(','));
+
+		// Data rows
+		for (const result of sorted) {
+			lines.push(buildPairwiseRow(result, judgeCount).join(','));
+		}
+	} else if (suite === 'binary-checks') {
+		// Header
+		lines.push(buildBinaryChecksHeader().join(','));
+
+		// Data rows
+		for (const result of sorted) {
+			lines.push(buildBinaryChecksRow(result).join(','));
+		}
+	} else {
+		// Default to LLM Judge format (also handles unknown)
+		// Header
+		lines.push(buildLlmJudgeHeader().join(','));
+
+		// Data rows
+		for (const result of sorted) {
+			lines.push(buildLlmJudgeRow(result).join(','));
+		}
+	}
+
+	// Write file (overwrites if exists)
+	writeFileSync(outputPath, lines.join('\n') + '\n', 'utf-8');
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/evaluation-helpers.ts

@@ -0,0 +1,201 @@
+import type { Callbacks } from '@langchain/core/callbacks/manager';
+import { getLangchainCallbacks } from 'langsmith/langchain';
+import { v4 as uuid } from 'uuid';
+
+import type { Evaluator, EvaluationContext, Feedback, LlmCallLimiter } from './harness-types';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+import type { BuilderFeatureFlags, ChatPayload } from '../../src/workflow-builder-agent';
+import { DEFAULTS } from '../support/constants';
+
+/**
+ * Get LangChain callbacks that bridge the current traceable context.
+ * Returns undefined if not in a traceable context.
+ */
+export async function getTracingCallbacks(): Promise<Callbacks | undefined> {
+	try {
+		return await getLangchainCallbacks();
+	} catch {
+		return undefined;
+	}
+}
+
+export async function consumeGenerator<T>(gen: AsyncGenerator<T>) {
+	for await (const _ of gen) {
+		/* consume all */
+	}
+}
+
+export async function runWithOptionalLimiter<T>(
+	fn: () => Promise<T>,
+	limiter?: LlmCallLimiter,
+): Promise<T> {
+	return limiter ? await limiter(fn) : await fn();
+}
+
+export async function withTimeout<T>(args: {
+	promise: Promise<T>;
+	timeoutMs?: number;
+	label: string;
+}): Promise<T> {
+	// NOTE:
+	// - This is a best-effort timeout. It does NOT cancel/abort the underlying work.
+	// - If the underlying work supports cancellation (e.g. AbortSignal), plumb that through instead.
+	// - When combined with `p-limit`, prefer applying the timeout *inside* the limited function so the
+	//   limiter slot is released when the timeout triggers.
+	const { promise, timeoutMs, label } = args;
+	if (typeof timeoutMs !== 'number') return await promise;
+	if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+		throw new Error(`Invalid timeoutMs (${String(timeoutMs)}) for ${label}`);
+	}
+
+	let timer: NodeJS.Timeout | undefined;
+	try {
+		const timeout = new Promise<never>((_resolve, reject) => {
+			timer = setTimeout(
+				() => reject(new Error(`Timed out after ${timeoutMs}ms in ${label}`)),
+				timeoutMs,
+			);
+		});
+		return await Promise.race([promise, timeout]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
+export interface GetChatPayloadOptions {
+	evalType: string;
+	message: string;
+	workflowId: string;
+	featureFlags?: BuilderFeatureFlags;
+}
+
+export function getChatPayload(options: GetChatPayloadOptions): ChatPayload {
+	const { evalType, message, workflowId, featureFlags } = options;
+
+	return {
+		id: `${evalType}-${uuid()}`,
+		featureFlags: featureFlags ?? DEFAULTS.FEATURE_FLAGS,
+		message,
+		workflowContext: {
+			currentWorkflow: { id: workflowId, nodes: [], connections: {} },
+		},
+	};
+}
+
+/**
+ * Coordination log entry for subgraph timing extraction.
+ * Matches the CoordinationLogEntry type from src/types/coordination.ts
+ */
+interface CoordinationLogEntry {
+	phase: 'discovery' | 'builder' | 'assistant' | 'state_management' | 'responder' | 'planner';
+	status: 'completed' | 'in_progress' | 'error';
+	timestamp: number;
+}
+
+/**
+ * Subgraph metrics extracted from coordination log.
+ */
+export interface ExtractedSubgraphMetrics {
+	discoveryDurationMs?: number;
+	builderDurationMs?: number;
+	responderDurationMs?: number;
+	nodeCount?: number;
+}
+
+/**
+ * Calculate duration for a specific phase from coordination log entries.
+ * Looks for the first 'in_progress' and terminal ('completed' or 'error') status for the phase.
+ */
+function calculatePhaseDuration(
+	coordinationLog: CoordinationLogEntry[],
+	phase: 'discovery' | 'builder' | 'responder',
+): number | undefined {
+	const phaseEntries = coordinationLog.filter((entry) => entry.phase === phase);
+	if (phaseEntries.length === 0) return undefined;
+
+	const inProgress = phaseEntries.find((e) => e.status === 'in_progress');
+	// Accept either 'completed' or 'error' as the terminal status
+	const terminal = phaseEntries.find((e) => e.status === 'completed' || e.status === 'error');
+
+	if (inProgress && terminal) {
+		return terminal.timestamp - inProgress.timestamp;
+	}
+
+	// If no in_progress entry, try to calculate from first to last entry
+	if (phaseEntries.length >= 2) {
+		const sorted = [...phaseEntries].sort((a, b) => a.timestamp - b.timestamp);
+		return sorted[sorted.length - 1].timestamp - sorted[0].timestamp;
+	}
+
+	return undefined;
+}
+
+/**
+ * Extract subgraph metrics from coordination log and workflow.
+ */
+export function extractSubgraphMetrics(
+	coordinationLog: CoordinationLogEntry[] | undefined,
+	nodeCount: number | undefined,
+): ExtractedSubgraphMetrics {
+	const metrics: ExtractedSubgraphMetrics = {};
+
+	// Include node count
+	if (nodeCount !== undefined) {
+		metrics.nodeCount = nodeCount;
+	}
+
+	// Extract timing from coordination log
+	if (coordinationLog && coordinationLog.length > 0) {
+		const discoveryDuration = calculatePhaseDuration(coordinationLog, 'discovery');
+		const builderDuration = calculatePhaseDuration(coordinationLog, 'builder');
+		const responderDuration = calculatePhaseDuration(coordinationLog, 'responder');
+
+		if (discoveryDuration !== undefined) {
+			metrics.discoveryDurationMs = discoveryDuration;
+		}
+		if (builderDuration !== undefined) {
+			metrics.builderDurationMs = builderDuration;
+		}
+		if (responderDuration !== undefined) {
+			metrics.responderDurationMs = responderDuration;
+		}
+	}
+
+	return metrics;
+}
+
+/**
+ * Run all evaluators on a workflow + context pair, with per-evaluator timeouts.
+ * Returns flattened feedback; errors are captured as feedback items.
+ */
+export async function runEvaluatorsOnExample(
+	evaluators: Array<Evaluator<EvaluationContext>>,
+	workflow: SimpleWorkflow,
+	context: EvaluationContext,
+	timeoutMs?: number,
+): Promise<Feedback[]> {
+	return (
+		await Promise.all(
+			evaluators.map(async (evaluator): Promise<Feedback[]> => {
+				try {
+					return await withTimeout({
+						promise: evaluator.evaluate(workflow, context),
+						timeoutMs,
+						label: `evaluator:${evaluator.name}`,
+					});
+				} catch (error) {
+					const msg = error instanceof Error ? error.message : String(error);
+					return [
+						{
+							evaluator: evaluator.name,
+							metric: 'error',
+							score: 0,
+							kind: 'score' as const,
+							comment: msg,
+						},
+					];
+				}
+			}),
+		)
+	).flat();
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/feedback.ts

@@ -0,0 +1,72 @@
+import type { Feedback } from './harness-types';
+
+export interface LangsmithEvaluationResultLike {
+	key: string;
+	score: number;
+	comment?: string;
+}
+
+export function feedbackKey(feedback: Feedback): string {
+	return `${feedback.evaluator}.${feedback.metric}`;
+}
+
+function isPairwiseV1Metric(metric: string): boolean {
+	return metric.startsWith('pairwise_');
+}
+
+/**
+ * Metric key mapping for LangSmith.
+ *
+ * Goal: keep keys comparable with historical runs.
+ * - Programmatic: keep evaluator prefix (e.g. `programmatic.trigger`)
+ * - LLM-judge: keep metrics unprefixed (e.g. `overallScore`, `connections`, `maintainability.nodeNamingQuality`)
+ * - Pairwise: keep v1 metrics unprefixed (e.g. `pairwise_primary`), but namespace non-v1 details.
+ * - Metrics: keep evaluator prefix (e.g. `metrics.discovery_latency_s`, `metrics.node_count`)
+ */
+export function langsmithMetricKey(feedback: Feedback): string {
+	if (feedback.evaluator === 'pairwise') {
+		return isPairwiseV1Metric(feedback.metric) ? feedback.metric : feedbackKey(feedback);
+	}
+
+	if (feedback.evaluator === 'programmatic') {
+		return feedbackKey(feedback);
+	}
+
+	if (feedback.evaluator === 'llm-judge') {
+		return feedback.metric;
+	}
+
+	if (feedback.evaluator === 'metrics') {
+		return feedbackKey(feedback);
+	}
+
+	if (feedback.evaluator === 'responder-judge') {
+		// Dimension & overall metrics unprefixed (like llm-judge), judge details prefixed.
+		return feedback.kind === 'detail' ? feedbackKey(feedback) : feedback.metric;
+	}
+
+	// Default: prefix unknown evaluators to avoid collisions with unprefixed `llm-judge` metrics.
+	return feedbackKey(feedback);
+}
+
+/**
+ * LangSmith score limits.
+ */
+const LANGSMITH_SCORE_MIN = -99999.9999;
+const LANGSMITH_SCORE_MAX = 99999.9999;
+
+/**
+ * Clamp a score to LangSmith's valid range.
+ * LangSmith rejects scores outside [-99999.9999, 99999.9999].
+ */
+function clampScoreForLangsmith(score: number): number {
+	return Math.max(LANGSMITH_SCORE_MIN, Math.min(LANGSMITH_SCORE_MAX, score));
+}
+
+export function toLangsmithEvaluationResult(feedback: Feedback): LangsmithEvaluationResultLike {
+	return {
+		key: langsmithMetricKey(feedback),
+		score: clampScoreForLangsmith(feedback.score),
+		...(feedback.comment ? { comment: feedback.comment } : {}),
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/harness-types.ts

@@ -0,0 +1,308 @@
+import type { Client as LangsmithClient } from 'langsmith/client';
+import type { IPinData } from 'n8n-workflow';
+import type pLimit from 'p-limit';
+
+import type { EvalLogger } from './logger';
+import type { GenerationCollectors } from './runner';
+import type { IntrospectionEvent } from '../../src/tools/introspect.tool.js';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+export type LlmCallLimiter = ReturnType<typeof pLimit>;
+
+/**
+ * Shared context passed to all evaluators.
+ *
+ * Keep this as the single "base" context so callers (CLI/runner) never need casts.
+ * Evaluators should validate required fields at runtime when optional fields are needed.
+ */
+export interface EvaluationContext {
+	/** The original user prompt for this example */
+	prompt: string;
+	/** Pairwise criteria: required behaviors */
+	dos?: string;
+	/** Pairwise criteria: forbidden behaviors */
+	donts?: string;
+	/** Optional reference workflows for similarity-based checks (best match wins) */
+	referenceWorkflows?: SimpleWorkflow[];
+	/**
+	 * Optional limiter for LLM-bound work (generation + evaluators).
+	 * When provided, treat it as the global knob for overall parallel LLM calls.
+	 */
+	llmCallLimiter?: LlmCallLimiter;
+	/**
+	 * Optional timeout used for LLM-bound work (generation + evaluators).
+	 * Note: timeouts are best-effort unless underlying calls support cancellation (AbortSignal).
+	 */
+	timeoutMs?: number;
+	/**
+	 * Generated TypeScript SDK code for code-level evaluators.
+	 * Populated from GenerationResult when available.
+	 */
+	generatedCode?: string;
+	/** Pin data for service nodes (used by execution evaluator) */
+	pinData?: IPinData;
+	/** Per-example annotations (e.g., code_necessary) from CSV or LangSmith dataset */
+	annotations?: Record<string, unknown>;
+}
+
+/** Context attached to an individual test case (prompt is provided separately). */
+export type TestCaseContext = Omit<Partial<EvaluationContext>, 'prompt'>;
+
+/** Global context attached to a run (prompt is provided per test case). */
+export type GlobalRunContext = Omit<Partial<EvaluationContext>, 'prompt'>;
+
+/**
+ * A styled line for verbose display output.
+ * Evaluators can provide these in `details.displayLines` for custom formatting.
+ */
+export interface DisplayLine {
+	text: string;
+	color?: 'yellow' | 'red' | 'dim';
+}
+
+/**
+ * What evaluators return - a single piece of feedback.
+ */
+export interface Feedback {
+	/** Evaluator name emitting this feedback (e.g. `llm-judge`, `programmatic`) */
+	evaluator: string;
+	/** Metric name within the evaluator (e.g. `functionality`, `efficiency.nodeCountEfficiency`) */
+	metric: string;
+	score: number;
+	comment?: string;
+	/**
+	 * Classification of this feedback item.
+	 *
+	 * - `score`: the single score used for overall scoring for this evaluator
+	 * - `metric`: stable category-level metrics (useful for dashboards)
+	 * - `detail`: unstable/verbose metrics that should not affect scoring
+	 */
+	kind: 'score' | 'metric' | 'detail';
+	/**
+	 * Optional structured details for rich display.
+	 * Evaluators can provide structured data here for better logging.
+	 * The `comment` field remains the primary text for LangSmith.
+	 */
+	details?: { displayLines?: DisplayLine[] } & Record<string, unknown>;
+}
+
+/**
+ * An evaluator that can assess a generated workflow.
+ * Optionally typed with context for evaluator-specific data.
+ */
+export interface Evaluator<TContext = EvaluationContext> {
+	name: string;
+	evaluate(workflow: SimpleWorkflow, ctx: TContext): Promise<Feedback[]>;
+}
+
+/**
+ * A single test case for evaluation.
+ */
+export interface TestCase {
+	prompt: string;
+	id?: string;
+	/** Context passed to evaluators (e.g., pairwise dos/donts) */
+	context?: TestCaseContext;
+	/** Reference workflows for similarity comparison (best match wins) */
+	referenceWorkflows?: SimpleWorkflow[];
+}
+
+/** Evaluation suite types supported by the harness */
+export type EvaluationSuite =
+	| 'llm-judge'
+	| 'pairwise'
+	| 'programmatic'
+	| 'similarity'
+	| 'introspection'
+	| 'binary-checks';
+
+/**
+ * Configuration for an evaluation run.
+ */
+export interface RunConfigBase {
+	/** Function to generate workflow from prompt. May return GenerationResult with source code. Optional collectors receive metrics. */
+	generateWorkflow: (
+		prompt: string,
+		collectors?: GenerationCollectors,
+	) => Promise<SimpleWorkflow | GenerationResult>;
+	/** Evaluators to run on each generated workflow */
+	evaluators: Array<Evaluator<EvaluationContext>>;
+	/** Global context available to all evaluators */
+	context?: GlobalRunContext;
+	/** Directory for JSON output files */
+	outputDir?: string;
+	/** CSV file path for evaluation results */
+	outputCsv?: string;
+	/** Evaluation suite (used for CSV formatting). If not set, auto-detected from feedback. */
+	suite?: EvaluationSuite;
+	/** Threshold for pass/fail classification of an example score (0-1). */
+	passThreshold?: number;
+	/** Timeout for generation/evaluator operations (ms). */
+	timeoutMs?: number;
+	/** Lifecycle hooks for logging and monitoring */
+	lifecycle?: Partial<EvaluationLifecycle>;
+	/** Logger for all output (use `createQuietLifecycle()` to suppress output in tests) */
+	logger: EvalLogger;
+	/** Optional pin data generator. When provided, generates mock data for service nodes after workflow generation. */
+	pinDataGenerator?: (workflow: SimpleWorkflow) => Promise<IPinData>;
+}
+
+export interface LocalRunConfig extends RunConfigBase {
+	mode: 'local';
+	/** Local mode requires an in-memory dataset */
+	dataset: TestCase[];
+	langsmithOptions?: never;
+	/** Number of examples to run in parallel (default: 1 for sequential) */
+	concurrency?: number;
+}
+
+export interface LangsmithRunConfig extends RunConfigBase {
+	mode: 'langsmith';
+	/** LangSmith mode uses a remote dataset name */
+	dataset: string;
+	langsmithOptions: LangsmithOptions;
+	/** LangSmith client used by both evaluate() and traceable() */
+	langsmithClient: LangsmithClient;
+}
+
+export type RunConfig = LocalRunConfig | LangsmithRunConfig;
+
+/**
+ * LangSmith-specific configuration.
+ */
+export interface LangsmithOptions {
+	experimentName: string;
+	repetitions: number;
+	concurrency: number;
+	/** Maximum number of examples to evaluate from the dataset */
+	maxExamples?: number;
+	/** Optional dataset filtering (requires pre-loading examples). */
+	filters?: LangsmithExampleFilters;
+	/** Enable trace filtering to reduce payload sizes (default: true) */
+	enableTraceFiltering?: boolean;
+	/** Arbitrary metadata passed to LangSmith experiment (e.g., numJudges, scoringMethod) */
+	experimentMetadata?: Record<string, unknown>;
+}
+
+export interface LangsmithExampleFilters {
+	/** Filter by `example.metadata.notion_id`. */
+	notionId?: string;
+	/** Filter by `example.metadata.categories` (contains). */
+	technique?: string;
+	/** Filter by `example.inputs.evals.dos` (substring match, case-insensitive). */
+	doSearch?: string;
+	/** Filter by `example.inputs.evals.donts` (substring match, case-insensitive). */
+	dontSearch?: string;
+}
+
+/**
+ * Subgraph timing metrics extracted from coordination log.
+ */
+export interface SubgraphMetrics {
+	/** Time spent in discovery subgraph (ms) */
+	discoveryDurationMs?: number;
+	/** Time spent in builder subgraph (ms) */
+	builderDurationMs?: number;
+	/** Time spent in responder generating the final response (ms) */
+	responderDurationMs?: number;
+	/** Number of nodes in the final workflow */
+	nodeCount?: number;
+}
+
+/**
+ * Result of evaluating a single example.
+ */
+export interface ExampleResult {
+	index: number;
+	prompt: string;
+	status: 'pass' | 'fail' | 'error';
+	/** Example-level score (0-1). In v2 this should be scoring-strategy aware (not key-count dependent). */
+	score: number;
+	feedback: Feedback[];
+	durationMs: number;
+	/** Time spent generating the workflow, when known. */
+	generationDurationMs?: number;
+	/** Time spent running evaluators, when known. */
+	evaluationDurationMs?: number;
+	/** Input tokens used during workflow generation */
+	generationInputTokens?: number;
+	/** Output tokens used during workflow generation */
+	generationOutputTokens?: number;
+	/** Subgraph timing and workflow metrics */
+	subgraphMetrics?: SubgraphMetrics;
+	/** Introspection events reported by the agent during workflow generation */
+	introspectionEvents?: IntrospectionEvent[];
+	workflow?: SimpleWorkflow;
+	/** Subgraph output (e.g., responder text). Present in subgraph eval mode. */
+	subgraphOutput?: SubgraphExampleOutput;
+	/** Generated source code (e.g., TypeScript SDK code from coding agent) */
+	generatedCode?: string;
+	error?: string;
+}
+
+/**
+ * Output from a subgraph evaluation example.
+ */
+export interface SubgraphExampleOutput {
+	/** The text response from the subgraph (e.g., responder output) */
+	response?: string;
+	/** The workflow produced by the subgraph (for builder/configurator) */
+	workflow?: SimpleWorkflow;
+};
+
+/**
+ * Result from workflow generation that may include source code.
+ * Used by generators that produce code (e.g., coding agent).
+ */
+export interface GenerationResult {
+	workflow: SimpleWorkflow;
+	/** Source code that generated the workflow (e.g., TypeScript SDK code) */
+	generatedCode?: string;
+}
+
+/**
+ * Type guard to check if a generation result is a GenerationResult object.
+ */
+export function isGenerationResult(
+	value: SimpleWorkflow | GenerationResult,
+): value is GenerationResult {
+	return (
+		typeof value === 'object' &&
+		value !== null &&
+		'workflow' in value &&
+		typeof value.workflow === 'object'
+	);
+}
+
+/**
+ * Summary of an entire evaluation run.
+ */
+export interface RunSummary {
+	totalExamples: number;
+	passed: number;
+	failed: number;
+	errors: number;
+	averageScore: number;
+	totalDurationMs: number;
+	evaluatorAverages?: Record<string, number>;
+	/** LangSmith IDs for constructing comparison URLs (only available in langsmith mode) */
+	langsmith?: {
+		experimentName: string;
+		experimentId: string;
+		datasetId: string;
+		datasetName?: string;
+	};
+}
+
+/**
+ * Lifecycle hooks for centralized logging and monitoring.
+ */
+export interface EvaluationLifecycle {
+	onStart(config: RunConfig): void;
+	onExampleStart(index: number, total: number, prompt: string): void;
+	onWorkflowGenerated(workflow: SimpleWorkflow, durationMs: number): void;
+	onEvaluatorComplete(name: string, feedback: Feedback[]): void;
+	onEvaluatorError(name: string, error: Error): void;
+	onExampleComplete(index: number, result: ExampleResult): void;
+	onEnd(summary: RunSummary): void | Promise<void>;
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/langsmith-dataset-writer.ts

@@ -0,0 +1,53 @@
+/**
+ * LangSmith dataset write-back utilities.
+ *
+ * Provides functions to update examples in a LangSmith dataset
+ * with regenerated state.
+ */
+
+import type { Client as LangsmithClient } from 'langsmith/client';
+
+import type { CoordinationLogEntry } from '@/types/coordination';
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type { EvalLogger } from './logger';
+import type { SerializedMessage } from './workflow-regenerator';
+
+/** Entry for LangSmith write-back operations */
+export interface LangSmithWriteBackEntry {
+	exampleId: string;
+	messages: SerializedMessage[];
+	coordinationLog: CoordinationLogEntry[];
+	workflowJSON: SimpleWorkflow;
+}
+
+/**
+ * Write regenerated state back to a LangSmith dataset.
+ * Updates the inputs field of each example while preserving other fields.
+ */
+export async function writeBackToLangSmithDataset(
+	client: LangsmithClient,
+	updates: LangSmithWriteBackEntry[],
+	logger?: EvalLogger,
+): Promise<void> {
+	if (updates.length === 0) {
+		logger?.verbose('No updates to write back to LangSmith');
+		return;
+	}
+
+	logger?.info(`Writing back ${updates.length} examples to LangSmith dataset...`);
+
+	// Use batch update for efficiency
+	const exampleUpdates = updates.map((update) => ({
+		id: update.exampleId,
+		inputs: {
+			messages: update.messages,
+			coordinationLog: update.coordinationLog,
+			workflowJSON: update.workflowJSON,
+		},
+	}));
+
+	await client.updateExamples(exampleUpdates);
+
+	logger?.info(`Successfully updated ${updates.length} examples in LangSmith`);
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/lifecycle.ts

@@ -0,0 +1,507 @@
+import pc from 'picocolors';
+
+import type {
+	DisplayLine,
+	EvaluationLifecycle,
+	RunConfig,
+	Feedback,
+	ExampleResult,
+	RunSummary,
+} from './harness-types';
+import type { EvalLogger } from './logger';
+import { groupByEvaluator, selectScoringItems, calculateFiniteAverage } from './score-calculator';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+/**
+ * Truncate a string for display.
+ */
+function truncate(str: string, maxLen = 50): string {
+	const cleaned = str.replace(/\s+/g, ' ').trim();
+	return cleaned.length > maxLen ? cleaned.slice(0, maxLen) + '...' : cleaned;
+}
+
+function truncateForSingleLine(str: string, maxLen: number): string {
+	return truncate(str.replace(/\n/g, ' '), maxLen);
+}
+
+function exampleLabel(mode: RunConfig['mode'] | undefined): 'call' | 'ex' {
+	return mode === 'langsmith' ? 'call' : 'ex';
+}
+
+/**
+ * Format a score as percentage.
+ */
+function formatScore(score: number): string {
+	if (!Number.isFinite(score)) return 'N/A';
+	return `${(score * 100).toFixed(0)}%`;
+}
+
+/**
+ * Format duration in seconds.
+ */
+function formatDuration(ms: number): string {
+	return `${(ms / 1000).toFixed(1)}s`;
+}
+
+/**
+ * Critical metrics to always show in verbose mode.
+ */
+const CRITICAL_METRICS = [
+	'functionality',
+	'connections',
+	'expressions',
+	'nodeConfiguration',
+	'overallScore',
+	'overall', // programmatic uses 'overall' not 'overallScore'
+	'trigger',
+];
+
+const DISPLAY_METRICS_BY_EVALUATOR: Record<string, string[]> = {
+	'llm-judge': ['functionality', 'connections', 'expressions', 'nodeConfiguration', 'overallScore'],
+	programmatic: ['overall', 'connections', 'trigger'],
+	pairwise: [
+		'pairwise_primary',
+		'pairwise_diagnostic',
+		'pairwise_judges_passed',
+		'pairwise_total_passes',
+		'pairwise_total_violations',
+	],
+	'responder-judge': [
+		'relevance',
+		'accuracy',
+		'completeness',
+		'clarity',
+		'criteriaMatch',
+		'forbiddenPhrases',
+		'overallScore',
+	],
+};
+
+const PAIRWISE_COUNT_METRICS = new Set([
+	'pairwise_judges_passed',
+	'pairwise_total_passes',
+	'pairwise_total_violations',
+]);
+
+const PAIRWISE_DISPLAY_NAMES: Record<string, string> = {
+	pairwise_primary: 'primary',
+	pairwise_diagnostic: 'diagnostic',
+	pairwise_judges_passed: 'judges_passed',
+	pairwise_total_passes: 'total_passes',
+	pairwise_total_violations: 'total_violations',
+};
+
+function getDisplayMetricName(evaluator: string, metric: string): string {
+	if (evaluator === 'pairwise') {
+		return PAIRWISE_DISPLAY_NAMES[metric] ?? metric;
+	}
+	return metric;
+}
+
+function isDisplayLine(item: unknown): item is DisplayLine {
+	if (typeof item !== 'object' || item === null) return false;
+	if (!('text' in item)) return false;
+	return typeof item.text === 'string';
+}
+
+function getDisplayLines(details?: Feedback['details']): DisplayLine[] | undefined {
+	if (!details?.displayLines || !Array.isArray(details.displayLines)) return undefined;
+	// Validate each item matches DisplayLine shape
+	if (!details.displayLines.every(isDisplayLine)) return undefined;
+	return details.displayLines;
+}
+
+function formatMetricValue(evaluator: string, metric: string, score: number): string {
+	if (evaluator === 'pairwise' && PAIRWISE_COUNT_METRICS.has(metric)) {
+		if (!Number.isFinite(score)) return 'N/A';
+		return Number.isInteger(score) ? String(score) : score.toFixed(0);
+	}
+	return formatScore(score);
+}
+
+function hasSeverityMarker(comment: string): boolean {
+	const lower = comment.toLowerCase();
+	return lower.includes('[critical]') || lower.includes('[major]') || lower.includes('[minor]');
+}
+
+function extractIssuesForLogs(evaluator: string, feedback: Feedback[]): Feedback[] {
+	const withComments = feedback.filter(
+		(f) => typeof f.comment === 'string' && f.comment.trim().length > 0 && f.metric !== 'error',
+	);
+
+	if (evaluator === 'llm-judge') {
+		return withComments.filter((f) => (f.comment ? hasSeverityMarker(f.comment) : false));
+	}
+
+	if (evaluator === 'pairwise') {
+		const isJudgeMetric = (metric: string) =>
+			/^judge\d+$/u.test(metric) || /^gen\d+\.judge\d+$/u.test(metric);
+
+		return withComments.filter((f) => {
+			if (isJudgeMetric(f.metric)) return true;
+
+			// Only show high-level status summaries when not fully passing.
+			if (f.metric === 'pairwise_primary' && f.score < 1) return true;
+			if (f.metric === 'pairwise_generation_correctness' && f.score < 1) return true;
+
+			return false;
+		});
+	}
+
+	if (evaluator === 'responder-judge') {
+		return withComments.filter((f) => {
+			// Show per-judge detail summaries
+			if (/^judge\d+$/u.test(f.metric)) return true;
+			// Show dimensions that scored below threshold
+			if (f.kind === 'metric' && f.score < 0.7) return true;
+			return false;
+		});
+	}
+
+	return withComments;
+}
+
+function formatExampleHeaderLines(args: {
+	mode: RunConfig['mode'] | undefined;
+	index: number;
+	status: string;
+	score: number;
+	prompt: string;
+	durationMs: number;
+	generationDurationMs?: number;
+	evaluationDurationMs?: number;
+	nodeCount: number;
+}): string[] {
+	const {
+		mode,
+		index,
+		status,
+		score,
+		prompt,
+		durationMs,
+		generationDurationMs,
+		evaluationDurationMs,
+		nodeCount,
+	} = args;
+
+	const promptSnippet = truncateForSingleLine(prompt, 80);
+	const genStr =
+		typeof generationDurationMs === 'number' ? formatDuration(generationDurationMs) : '?';
+	const evalStr =
+		typeof evaluationDurationMs === 'number' ? formatDuration(evaluationDurationMs) : '?';
+
+	return [
+		`${pc.dim(`[${exampleLabel(mode)} ${index}]`)} ${status} ${formatScore(score)} ${pc.dim(
+			`prompt="${promptSnippet}"`,
+		)}`,
+		pc.dim(
+			`  gen=${genStr} eval=${evalStr} total=${formatDuration(durationMs)} nodes=${nodeCount}`,
+		),
+	];
+}
+
+function splitEvaluatorFeedback(feedback: Feedback[]): {
+	errors: Feedback[];
+	nonErrorFeedback: Feedback[];
+} {
+	return {
+		errors: feedback.filter((f) => f.metric === 'error'),
+		nonErrorFeedback: feedback.filter((f) => f.metric !== 'error'),
+	};
+}
+
+function formatEvaluatorLines(args: {
+	evaluatorName: string;
+	feedback: Feedback[];
+}): string[] {
+	const { evaluatorName, feedback } = args;
+
+	const { errors, nonErrorFeedback } = splitEvaluatorFeedback(feedback);
+
+	const scoringItems = selectScoringItems(feedback);
+	const avgScore = calculateFiniteAverage(scoringItems);
+
+	const colorFn = scoreColor(avgScore);
+
+	const lines: string[] = [];
+	lines.push(
+		pc.dim(`  ${evaluatorName}: `) +
+			colorFn(formatScore(avgScore)) +
+			pc.dim(
+				errors.length > 0
+					? ` (metrics=${nonErrorFeedback.length}, errors=${errors.length})`
+					: ` (metrics=${feedback.length})`,
+			),
+	);
+
+	const displayMetrics = DISPLAY_METRICS_BY_EVALUATOR[evaluatorName] ?? CRITICAL_METRICS;
+	const picked = nonErrorFeedback.filter((f) => displayMetrics.includes(f.metric));
+	if (picked.length > 0) {
+		const metricsLine = picked
+			.map((f) => {
+				const color = scoreColor(f.score);
+				const displayName = getDisplayMetricName(evaluatorName, f.metric);
+				return `${displayName}: ${color(formatMetricValue(evaluatorName, f.metric, f.score))}`;
+			})
+			.join(pc.dim(' | '));
+		lines.push(pc.dim('    ') + metricsLine);
+	}
+
+	if (errors.length > 0) {
+		const topErrors = errors.slice(0, 2);
+		lines.push(pc.dim(`    errors(top=${topErrors.length}):`));
+		for (const errorItem of topErrors) {
+			const comment = truncateForSingleLine(errorItem.comment ?? '', 240);
+			lines.push(pc.dim('      - ') + pc.red(comment));
+		}
+		if (errors.length > topErrors.length) {
+			lines.push(pc.dim(`      ... and ${errors.length - topErrors.length} more`));
+		}
+	}
+
+	const issues = extractIssuesForLogs(evaluatorName, feedback);
+	if (issues.length > 0) {
+		const top = issues.slice(0, 3);
+		lines.push(pc.dim(`    issues(top=${top.length}):`));
+		for (const issue of top) {
+			const displayMetric = getDisplayMetricName(evaluatorName, issue.metric);
+			const displayLines = getDisplayLines(issue.details);
+			if (displayLines && displayLines.length > 0) {
+				// Evaluator provided custom display lines with optional color
+				lines.push(pc.dim(`      - [${displayMetric}]`));
+				for (const dl of displayLines) {
+					const truncated = truncateForSingleLine(dl.text, 300);
+					const colorFn = dl.color === 'yellow' ? pc.yellow : dl.color === 'dim' ? pc.dim : pc.red;
+					lines.push(pc.dim('          ') + colorFn(truncated));
+				}
+			} else {
+				const comment = truncateForSingleLine(issue.comment ?? '', 320);
+				lines.push(pc.dim(`      - [${displayMetric}] `) + pc.red(comment));
+			}
+		}
+		if (issues.length > top.length) {
+			lines.push(pc.dim(`      ... and ${issues.length - top.length} more`));
+		}
+	}
+
+	return lines;
+}
+
+/**
+ * Get color based on score.
+ */
+function scoreColor(score: number): (s: string) => string {
+	if (score >= 0.9) return pc.green;
+	if (score >= 0.7) return pc.yellow;
+	return pc.red;
+}
+
+function formatExampleStatus(status: ExampleResult['status']): string {
+	switch (status) {
+		case 'pass':
+			return pc.green('PASS');
+		case 'fail':
+			return pc.yellow('FAIL');
+		case 'error':
+			return pc.red('ERROR');
+	}
+}
+
+/**
+ * Options for creating a console lifecycle.
+ */
+export interface ConsoleLifecycleOptions {
+	verbose: boolean;
+	logger: EvalLogger;
+}
+
+/**
+ * Create a lifecycle that logs to console.
+ * Verbose mode shows detailed progress, non-verbose shows summary only.
+ */
+export function createConsoleLifecycle(options: ConsoleLifecycleOptions): EvaluationLifecycle {
+	const { verbose, logger } = options;
+	let runMode: RunConfig['mode'] | undefined;
+	let evaluatorOrder: string[] = [];
+
+	return {
+		onStart(config: RunConfig): void {
+			runMode = config.mode;
+			evaluatorOrder = config.evaluators.map((e) => e.name);
+
+			logger.info(`\nStarting evaluation in ${pc.cyan(config.mode)} mode`);
+
+			if (typeof config.dataset === 'string') {
+				logger.info(`Dataset: ${pc.dim(config.dataset)}`);
+			} else {
+				logger.info(`Test cases: ${pc.dim(String(config.dataset.length))}`);
+			}
+
+			logger.info(
+				`Evaluators: ${pc.dim(config.evaluators.map((e) => e.name).join(', ') || 'none')}`,
+			);
+			logger.info('');
+		},
+
+		onExampleStart(index: number, total: number, prompt: string): void {
+			if (!verbose) return;
+
+			const totalStr = total > 0 ? String(total) : '?';
+			const prefix = pc.dim(`[${exampleLabel(runMode)} ${index}/${totalStr}]`);
+			const status = pc.yellow('START');
+			const promptStr = pc.dim(`prompt="${truncateForSingleLine(prompt, 80)}"`);
+			logger.info(`${prefix} ${status} ${promptStr}`);
+		},
+
+		onWorkflowGenerated: () => {},
+
+		onEvaluatorComplete: () => {},
+
+		onEvaluatorError(name: string, error: Error): void {
+			if (!verbose) return;
+			logger.error(`    ERROR in ${name}: ${error.message}`);
+		},
+
+		onExampleComplete(index: number, result: ExampleResult): void {
+			if (!verbose) return;
+
+			const status = formatExampleStatus(result.status);
+
+			const nodeCount = result.workflow?.nodes?.length ?? 0;
+			const lines: string[] = formatExampleHeaderLines({
+				mode: runMode,
+				index,
+				status,
+				score: result.score,
+				prompt: result.prompt,
+				durationMs: result.durationMs,
+				generationDurationMs: result.generationDurationMs,
+				evaluationDurationMs: result.evaluationDurationMs,
+				nodeCount,
+			});
+
+			if (result.error) {
+				lines.push(pc.red(`  error: ${result.error}`));
+				logger.info(lines.join('\n'));
+				return;
+			}
+
+			const grouped = groupByEvaluator(result.feedback);
+			const orderedEvaluators = [
+				...evaluatorOrder.filter((name) => name in grouped),
+				...Object.keys(grouped).filter((name) => !evaluatorOrder.includes(name)),
+			];
+
+			for (const evaluatorName of orderedEvaluators) {
+				const feedback = grouped[evaluatorName] ?? [];
+				lines.push(...formatEvaluatorLines({ evaluatorName, feedback }));
+			}
+
+			logger.info(lines.join('\n'));
+		},
+
+		onEnd(summary: RunSummary): void {
+			if (runMode === 'langsmith') return;
+			logger.info('\n' + pc.bold('═══════════════════ SUMMARY ═══════════════════'));
+			logger.info(
+				`  Total: ${summary.totalExamples} | ` +
+					`Pass: ${pc.green(String(summary.passed))} | ` +
+					`Fail: ${pc.yellow(String(summary.failed))} | ` +
+					`Error: ${pc.red(String(summary.errors))}`,
+			);
+			const passRate = summary.totalExamples > 0 ? summary.passed / summary.totalExamples : 0;
+			logger.info(`  Pass rate: ${formatScore(passRate)}`);
+			logger.info(`  Average score: ${formatScore(summary.averageScore)}`);
+			logger.info(`  Total time: ${formatDuration(summary.totalDurationMs)}`);
+
+			if (summary.evaluatorAverages && Object.keys(summary.evaluatorAverages).length > 0) {
+				logger.info(pc.dim('  Evaluator averages:'));
+				for (const [name, avg] of Object.entries(summary.evaluatorAverages)) {
+					const color = scoreColor(avg);
+					logger.info(`    ${pc.dim(name + ':')} ${color(formatScore(avg))}`);
+				}
+			}
+
+			if (summary.langsmith) {
+				logger.info(pc.dim(`  Experiment: ${summary.langsmith.experimentName}`));
+			}
+
+			logger.info(pc.bold('═══════════════════════════════════════════════\n'));
+		},
+	};
+}
+
+/**
+ * Create a quiet lifecycle that does nothing.
+ * Useful for testing or when no output is desired.
+ */
+export function createQuietLifecycle(): EvaluationLifecycle {
+	return {
+		onStart: () => {},
+		onExampleStart: () => {},
+		onWorkflowGenerated: () => {},
+		onEvaluatorComplete: () => {},
+		onEvaluatorError: () => {},
+		onExampleComplete: () => {},
+		onEnd: () => {},
+	};
+}
+
+/** Type predicate for filtering undefined values */
+function isDefined<T>(value: T | undefined): value is T {
+	return value !== undefined;
+}
+
+/**
+ * Merge multiple partial lifecycles into a single complete lifecycle.
+ * All hooks will be called in order.
+ */
+export function mergeLifecycles(
+	...lifecycles: Array<Partial<EvaluationLifecycle> | undefined>
+): EvaluationLifecycle {
+	const validLifecycles = lifecycles.filter(isDefined);
+
+	return {
+		onStart(config: RunConfig): void {
+			for (const lc of validLifecycles) {
+				lc.onStart?.(config);
+			}
+		},
+
+		onExampleStart(index: number, total: number, prompt: string): void {
+			for (const lc of validLifecycles) {
+				lc.onExampleStart?.(index, total, prompt);
+			}
+		},
+
+		onWorkflowGenerated(workflow: SimpleWorkflow, durationMs: number): void {
+			for (const lc of validLifecycles) {
+				lc.onWorkflowGenerated?.(workflow, durationMs);
+			}
+		},
+
+		onEvaluatorComplete(name: string, feedback: Feedback[]): void {
+			for (const lc of validLifecycles) {
+				lc.onEvaluatorComplete?.(name, feedback);
+			}
+		},
+
+		onEvaluatorError(name: string, error: Error): void {
+			for (const lc of validLifecycles) {
+				lc.onEvaluatorError?.(name, error);
+			}
+		},
+
+		onExampleComplete(index: number, result: ExampleResult): void {
+			for (const lc of validLifecycles) {
+				lc.onExampleComplete?.(index, result);
+			}
+		},
+
+		async onEnd(summary: RunSummary): Promise<void> {
+			for (const lc of validLifecycles) {
+				await lc.onEnd?.(summary);
+			}
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/logger.ts

@@ -0,0 +1,42 @@
+import pc from 'picocolors';
+
+/**
+ * Simple evaluation logger with verbose mode support.
+ *
+ * Usage:
+ *   const log = createLogger(isVerbose);
+ *   log.info('Always shown');
+ *   log.verbose('Only shown in verbose mode');
+ */
+
+export interface EvalLogger {
+	/** Always shown - important info */
+	info: (message: string) => void;
+	/** Only shown in verbose mode - debug details */
+	verbose: (message: string) => void;
+	/** Success messages (green) */
+	success: (message: string) => void;
+	/** Warning messages (yellow) */
+	warn: (message: string) => void;
+	/** Error messages (red) */
+	error: (message: string) => void;
+	/** Dimmed text for secondary info */
+	dim: (message: string) => void;
+	/** Check if verbose mode is enabled */
+	isVerbose: boolean;
+}
+
+export function createLogger(verbose: boolean = false): EvalLogger {
+	return {
+		isVerbose: verbose,
+		// Keep info plain so lifecycle can apply its own formatting without double-coloring.
+		info: (message: string) => console.log(message),
+		verbose: (message: string) => {
+			if (verbose) console.log(pc.dim(message));
+		},
+		success: (message: string) => console.log(pc.green(message)),
+		warn: (message: string) => console.warn(pc.yellow(message)),
+		error: (message: string) => console.error(pc.red(message)),
+		dim: (message: string) => console.log(pc.dim(message)),
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/output.ts

@@ -0,0 +1,241 @@
+/**
+ * Artifact saving for v2 evaluation harness.
+ *
+ * Saves evaluation results to disk in JSON format for later analysis.
+ */
+
+import { createHash } from 'crypto';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { feedbackKey } from './feedback';
+import type { ExampleResult, Feedback, RunSummary } from './harness-types';
+import type { EvalLogger } from './logger';
+import { selectScoringItems, calculateFiniteAverage } from './score-calculator';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+/**
+ * Interface for saving evaluation artifacts to disk.
+ */
+export interface ArtifactSaver {
+	/** Save a single example result */
+	saveExample(result: ExampleResult): void;
+	/** Save the final summary */
+	saveSummary(summary: RunSummary, results: ExampleResult[]): void;
+}
+
+/**
+ * Options for creating an artifact saver.
+ */
+export interface ArtifactSaverOptions {
+	/** Directory to save artifacts to */
+	outputDir: string;
+	/** Logger for optional save logs */
+	logger: EvalLogger;
+}
+
+/**
+ * Create an artifact saver for persisting evaluation results to disk.
+ *
+ * Directory structure:
+ * ```
+ * outputDir/
+ * ├── example-001/
+ * │   ├── prompt.txt
+ * │   ├── workflow.json
+ * │   └── feedback.json
+ * ├── example-002/
+ * │   └── ...
+ * └── summary.json
+ * ```
+ *
+ * @param options - Configuration options
+ * @returns ArtifactSaver instance or null if outputDir is not provided
+ */
+export function createArtifactSaver(options: ArtifactSaverOptions): ArtifactSaver {
+	const { outputDir, logger } = options;
+
+	// Create output directory if it doesn't exist
+	fs.mkdirSync(outputDir, { recursive: true });
+
+	return {
+		saveExample(result: ExampleResult): void {
+			const exampleDir = path.join(outputDir, getExampleDirName(result));
+			fs.mkdirSync(exampleDir, { recursive: true });
+
+			// Save prompt
+			fs.writeFileSync(path.join(exampleDir, 'prompt.txt'), result.prompt, 'utf-8');
+
+			// Save workflow if available
+			if (result.workflow) {
+				const workflowForExport = formatWorkflowForExport(result.workflow);
+				fs.writeFileSync(
+					path.join(exampleDir, 'workflow.json'),
+					JSON.stringify(workflowForExport, null, 2),
+					'utf-8',
+				);
+			}
+
+			// Save generated code if available (e.g., TypeScript SDK code from coding agent)
+			if (result.generatedCode) {
+				fs.writeFileSync(path.join(exampleDir, 'code.ts'), result.generatedCode, 'utf-8');
+			}
+
+			// Save feedback
+			const feedbackOutput = formatFeedbackForExport(result);
+			fs.writeFileSync(
+				path.join(exampleDir, 'feedback.json'),
+				JSON.stringify(feedbackOutput, null, 2),
+				'utf-8',
+			);
+
+			// Save error if present
+			if (result.error) {
+				fs.writeFileSync(path.join(exampleDir, 'error.txt'), result.error, 'utf-8');
+			}
+
+			logger.verbose(`Saved example ${result.index} to ${exampleDir}`);
+		},
+
+		saveSummary(summary: RunSummary, results: ExampleResult[]): void {
+			const summaryOutput = formatSummaryForExport(summary, results);
+			fs.writeFileSync(
+				path.join(outputDir, 'summary.json'),
+				JSON.stringify(summaryOutput, null, 2),
+				'utf-8',
+			);
+
+			logger.verbose(`Saved summary to ${path.join(outputDir, 'summary.json')}`);
+		},
+	};
+}
+
+function getExampleDirName(result: ExampleResult): string {
+	const index = String(result.index).padStart(3, '0');
+	const id = shortId(`${result.prompt}\n${result.index}`);
+	return `example-${index}-${id}`;
+}
+
+function shortId(input: string): string {
+	// Small deterministic id to avoid collisions when example folders are written concurrently
+	// and to keep folder names stable across reruns with the same prompts.
+	return createHash('md5').update(input).digest('hex').slice(0, 8);
+}
+
+/**
+ * Format a workflow for export (n8n-importable format).
+ */
+function formatWorkflowForExport(workflow: SimpleWorkflow): object {
+	return {
+		name: workflow.name ?? 'Generated Workflow',
+		nodes: workflow.nodes ?? [],
+		connections: workflow.connections ?? {},
+	};
+}
+
+/**
+ * Format feedback for export.
+ */
+function formatFeedbackForExport(result: ExampleResult): object {
+	// Group feedback by evaluator
+	const byEvaluator: Record<string, Feedback[]> = {};
+	for (const fb of result.feedback) {
+		const evaluator = fb.evaluator;
+		if (!byEvaluator[evaluator]) {
+			byEvaluator[evaluator] = [];
+		}
+		byEvaluator[evaluator].push(fb);
+	}
+
+	return {
+		index: result.index,
+		status: result.status,
+		durationMs: result.durationMs,
+		generationDurationMs: result.generationDurationMs,
+		evaluationDurationMs: result.evaluationDurationMs,
+		generationInputTokens: result.generationInputTokens,
+		generationOutputTokens: result.generationOutputTokens,
+		score: result.score,
+		// Include subgraph metrics if available
+		...(result.subgraphMetrics && {
+			subgraphMetrics: {
+				nodeCount: result.subgraphMetrics.nodeCount,
+				discoveryDurationMs: result.subgraphMetrics.discoveryDurationMs,
+				builderDurationMs: result.subgraphMetrics.builderDurationMs,
+				responderDurationMs: result.subgraphMetrics.responderDurationMs,
+			},
+		}),
+		evaluators: Object.entries(byEvaluator).map(([name, items]) => ({
+			name,
+			feedback: items.map((f) => ({
+				key: feedbackKey(f),
+				metric: f.metric,
+				score: f.score,
+				kind: f.kind,
+				...(f.comment ? { comment: f.comment } : {}),
+			})),
+			averageScore: calculateFiniteAverage(selectScoringItems(items)),
+		})),
+		allFeedback: result.feedback,
+	};
+}
+
+/**
+ * Format summary for export.
+ */
+function formatSummaryForExport(summary: RunSummary, results: ExampleResult[]): object {
+	const resultsSorted = [...results].sort((a, b) => a.index - b.index);
+
+	// Calculate per-evaluator statistics
+	const evaluatorStats: Record<string, { scores: number[] }> = {};
+	for (const result of resultsSorted) {
+		const byEvaluator: Record<string, Feedback[]> = {};
+		for (const fb of result.feedback) {
+			const evaluator = fb.evaluator;
+			if (!byEvaluator[evaluator]) byEvaluator[evaluator] = [];
+			byEvaluator[evaluator].push(fb);
+		}
+		for (const [evaluator, items] of Object.entries(byEvaluator)) {
+			if (!evaluatorStats[evaluator]) {
+				evaluatorStats[evaluator] = { scores: [] };
+			}
+			const scoringItems = selectScoringItems(items);
+			const avg = calculateFiniteAverage(scoringItems);
+			evaluatorStats[evaluator].scores.push(avg);
+		}
+	}
+
+	const evaluatorAverages: Record<string, number> = {};
+	for (const [name, stats] of Object.entries(evaluatorStats)) {
+		evaluatorAverages[name] = stats.scores.reduce((a, b) => a + b, 0) / stats.scores.length;
+	}
+
+	return {
+		timestamp: new Date().toISOString(),
+		totalExamples: summary.totalExamples,
+		passed: summary.passed,
+		failed: summary.failed,
+		errors: summary.errors,
+		passRate: summary.totalExamples > 0 ? summary.passed / summary.totalExamples : 0,
+		averageScore: summary.averageScore,
+		totalDurationMs: summary.totalDurationMs,
+		evaluatorAverages,
+		results: resultsSorted.map((r) => ({
+			index: r.index,
+			prompt: r.prompt.slice(0, 100) + (r.prompt.length > 100 ? '...' : ''),
+			status: r.status,
+			score: r.score,
+			durationMs: r.durationMs,
+			generationDurationMs: r.generationDurationMs,
+			generationInputTokens: r.generationInputTokens,
+			generationOutputTokens: r.generationOutputTokens,
+			...(r.subgraphMetrics && {
+				nodeCount: r.subgraphMetrics.nodeCount,
+				discoveryDurationMs: r.subgraphMetrics.discoveryDurationMs,
+				builderDurationMs: r.subgraphMetrics.builderDurationMs,
+				responderDurationMs: r.subgraphMetrics.responderDurationMs,
+			}),
+			...(r.error ? { error: r.error } : {}),
+		})),
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/score-calculator.ts

@@ -0,0 +1,256 @@
+/**
+ * Score Calculation Utilities
+ *
+ * Provides functions for calculating weighted scores and aggregating
+ * feedback from multiple evaluators.
+ */
+
+import type { Feedback } from './harness-types';
+
+/**
+ * Weights for each evaluator type.
+ */
+export interface ScoreWeights {
+	[evaluatorPrefix: string]: number;
+}
+
+/**
+ * Result of score aggregation.
+ */
+export interface AggregatedScore {
+	/** Weighted overall score (0-1) */
+	overall: number;
+	/** Average score per evaluator */
+	byEvaluator: Record<string, number>;
+	/** Average score per category */
+	byCategory: Record<string, number>;
+}
+
+/**
+ * Parsed feedback key structure.
+ */
+export interface FeedbackKeyParts {
+	evaluator: string;
+	category: string;
+	subcategory?: string;
+}
+
+/**
+ * Default weights for standard evaluators (cross-evaluator weighting).
+ *
+ * This is the *harness-level* weighting between evaluators like `llm-judge`,
+ * `programmatic`, and `pairwise`. It is independent from any evaluator-internal
+ * weighting (e.g. LLM judge category weights).
+ * Weights should sum to approximately 1.0.
+ */
+export const DEFAULT_EVALUATOR_WEIGHTS: ScoreWeights = {
+	'llm-judge': 0.35,
+	programmatic: 0.25,
+	pairwise: 0.25,
+	similarity: 0.15,
+	'binary-checks': 0,
+};
+
+/**
+ * @deprecated Use `DEFAULT_EVALUATOR_WEIGHTS` (kept for backwards compatibility within the package).
+ */
+export const DEFAULT_WEIGHTS: ScoreWeights = DEFAULT_EVALUATOR_WEIGHTS;
+
+/** Default weight for unknown evaluators */
+const UNKNOWN_EVALUATOR_WEIGHT = 0.1;
+
+/**
+ * Parse a feedback key into its component parts.
+ *
+ * @example
+ * parseFeedbackKey('llm-judge.functionality')
+ * // => { evaluator: 'llm-judge', category: 'functionality' }
+ *
+ * parseFeedbackKey('pairwise.gen1.majorityPass')
+ * // => { evaluator: 'pairwise', category: 'gen1', subcategory: 'majorityPass' }
+ */
+export function parseFeedbackKey(key: string): FeedbackKeyParts {
+	const parts = key.split('.');
+	return {
+		evaluator: parts[0],
+		category: parts[1] ?? '',
+		subcategory: parts[2],
+	};
+}
+
+/**
+ * Extract the category from a feedback key.
+ *
+ * @example
+ * extractCategory('llm-judge.functionality') // => 'functionality'
+ * extractCategory('programmatic.trigger') // => 'trigger'
+ */
+export function extractCategory(key: string): string {
+	return parseFeedbackKey(key).category;
+}
+
+/**
+ * Group feedback items by their evaluator prefix.
+ *
+ * @example
+ * groupByEvaluator([
+ *   { evaluator: 'llm-judge', metric: 'a', score: 0.8 },
+ *   { evaluator: 'programmatic', metric: 'b', score: 0.6 },
+ * ])
+ * // => { 'llm-judge': [...], 'programmatic': [...] }
+ */
+export function groupByEvaluator(feedback: Feedback[]): Record<string, Feedback[]> {
+	const grouped: Record<string, Feedback[]> = {};
+
+	for (const item of feedback) {
+		const evaluator = item.evaluator;
+		if (!grouped[evaluator]) {
+			grouped[evaluator] = [];
+		}
+		grouped[evaluator].push(item);
+	}
+
+	return grouped;
+}
+
+/**
+ * Calculate average score for an array of feedback items.
+ */
+export function calculateFiniteAverage(items: Feedback[]): number {
+	if (items.length === 0) return 0;
+	const finiteScores = items.map((f) => f.score).filter((s) => Number.isFinite(s));
+	if (finiteScores.length === 0) return 0;
+	const total = finiteScores.reduce((sum, s) => sum + s, 0);
+	return total / finiteScores.length;
+}
+
+/**
+ * Pick which feedback items should be used for evaluator-level scoring.
+ *
+ * Order of preference:
+ * - `kind: 'score'` (single authoritative score)
+ * - `kind: 'metric'` (stable category metrics)
+ * - any non-`detail` items
+ * - otherwise, all items
+ */
+export function selectScoringItems(items: Feedback[]): Feedback[] {
+	const scoreItems = items.filter((i) => i.kind === 'score');
+	if (scoreItems.length > 0) return scoreItems;
+
+	const metricItems = items.filter((i) => i.kind === 'metric');
+	if (metricItems.length > 0) return metricItems;
+
+	const nonDetailItems = items.filter((i) => i.kind !== 'detail');
+	if (nonDetailItems.length > 0) return nonDetailItems;
+
+	return items;
+}
+
+/**
+ * Calculate weighted overall score from feedback.
+ *
+ * Each evaluator's average score is weighted according to the weights map.
+ * Unknown evaluators receive the default weight.
+ *
+ * @param feedback - Array of feedback items
+ * @param weights - Weight per evaluator (defaults to DEFAULT_WEIGHTS)
+ * @returns Weighted average score (0-1)
+ */
+export function calculateWeightedScore(
+	feedback: Feedback[],
+	weights: ScoreWeights = DEFAULT_EVALUATOR_WEIGHTS,
+): number {
+	if (feedback.length === 0) return 0;
+
+	const byEvaluator = groupByEvaluator(feedback);
+
+	let totalWeight = 0;
+	let weightedSum = 0;
+
+	for (const [evaluator, items] of Object.entries(byEvaluator)) {
+		const avgScore = calculateFiniteAverage(selectScoringItems(items));
+		const weight = weights[evaluator] ?? UNKNOWN_EVALUATOR_WEIGHT;
+		weightedSum += avgScore * weight;
+		totalWeight += weight;
+	}
+
+	return totalWeight > 0 ? weightedSum / totalWeight : 0;
+}
+
+/**
+ * Compute per-evaluator average scores from a set of example results.
+ *
+ * Groups feedback by evaluator, selects scoring items, and averages
+ * across all examples. Shared between local and LangSmith evaluation runners.
+ */
+export function computeEvaluatorAverages(
+	results: Array<{ feedback: Feedback[] }>,
+): Record<string, number> {
+	const evaluatorStats: Record<string, number[]> = {};
+
+	for (const result of results) {
+		const byEvaluator = groupByEvaluator(result.feedback);
+		for (const [evaluator, items] of Object.entries(byEvaluator)) {
+			if (!evaluatorStats[evaluator]) evaluatorStats[evaluator] = [];
+			const scoringItems = selectScoringItems(items);
+			evaluatorStats[evaluator].push(calculateFiniteAverage(scoringItems));
+		}
+	}
+
+	const averages: Record<string, number> = {};
+	for (const [name, scores] of Object.entries(evaluatorStats)) {
+		averages[name] = scores.reduce((a, b) => a + b, 0) / scores.length;
+	}
+	return averages;
+}
+
+/**
+ * Aggregate scores by evaluator and category.
+ *
+ * @param feedback - Array of feedback items
+ * @returns Aggregated scores with overall, by-evaluator, and by-category breakdowns
+ */
+export function aggregateScores(feedback: Feedback[]): AggregatedScore {
+	if (feedback.length === 0) {
+		return {
+			overall: 0,
+			byEvaluator: {},
+			byCategory: {},
+		};
+	}
+
+	// Calculate overall weighted score
+	const overall = calculateWeightedScore(feedback);
+
+	// Calculate by-evaluator averages
+	const byEvaluator: Record<string, number> = {};
+	const grouped = groupByEvaluator(feedback);
+	for (const [evaluator, items] of Object.entries(grouped)) {
+		byEvaluator[evaluator] = calculateFiniteAverage(selectScoringItems(items));
+	}
+
+	// Calculate by-category averages
+	const byCategory: Record<string, number> = {};
+	const categoryGroups: Record<string, Feedback[]> = {};
+
+	for (const item of feedback) {
+		if (item.kind === 'detail') continue;
+		const category = item.metric.split('.')[0] ?? '';
+		if (category) {
+			if (!categoryGroups[category]) {
+				categoryGroups[category] = [];
+			}
+			categoryGroups[category].push(item);
+		}
+	}
+
+	for (const [category, items] of Object.entries(categoryGroups)) {
+		byCategory[category] = calculateFiniteAverage(items);
+	}
+
+	return {
+		overall,
+		byEvaluator,
+		byCategory,
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/subgraph-evaluation-local.ts

@@ -0,0 +1,326 @@
+/**
+ * Local subgraph evaluation runner.
+ *
+ * Runs subgraph evaluations against a local dataset (JSON file)
+ * without requiring LangSmith.
+ */
+
+import type { INodeTypeDescription } from 'n8n-workflow';
+import pLimit from 'p-limit';
+
+import { runWithOptionalLimiter, withTimeout, runEvaluatorsOnExample } from './evaluation-helpers';
+import type {
+	Evaluator,
+	EvaluationContext,
+	Feedback,
+	RunSummary,
+	ExampleResult,
+	EvaluationLifecycle,
+} from './harness-types';
+import type { EvalLogger } from './logger';
+import { createArtifactSaver } from './output';
+import { calculateWeightedScore, computeEvaluatorAverages } from './score-calculator';
+import {
+	extractPreComputedState,
+	deserializeMessages,
+	type SubgraphRunFn,
+	type PreComputedState,
+	type SubgraphName,
+} from './subgraph-runner';
+import { regenerateWorkflowState } from './workflow-regenerator';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+import { writeBackToDatasetFile, type DatasetWriteBackEntry } from '../cli/dataset-file-loader';
+import type { ResponderEvalCriteria } from '../evaluators/responder/responder-judge.prompt';
+import type { ResolvedStageLLMs } from '../support/environment';
+
+const DEFAULT_PASS_THRESHOLD = 0.7;
+
+interface LocalSubgraphEvaluationConfig {
+	subgraph: SubgraphName;
+	subgraphRunner: SubgraphRunFn;
+	evaluators: Array<Evaluator<EvaluationContext>>;
+	examples: Array<{ inputs: Record<string, unknown> }>;
+	concurrency: number;
+	lifecycle?: Partial<EvaluationLifecycle>;
+	logger: EvalLogger;
+	outputDir?: string;
+	timeoutMs?: number;
+	passThreshold?: number;
+	/** Run full workflow generation from prompt instead of using pre-computed state */
+	regenerate?: boolean;
+	/** Write regenerated state back to dataset file */
+	writeBack?: boolean;
+	/** Path to the dataset file (required for write-back) */
+	datasetFilePath?: string;
+	/** LLMs for regeneration (required if regenerate is true) */
+	llms?: ResolvedStageLLMs;
+	/** Parsed node types for regeneration (required if regenerate is true) */
+	parsedNodeTypes?: INodeTypeDescription[];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function extractPromptFromInputs(inputs: Record<string, unknown>): string {
+	if (typeof inputs.prompt === 'string') return inputs.prompt;
+	if (Array.isArray(inputs.messages) && inputs.messages.length > 0) {
+		const first: unknown = inputs.messages[0];
+		if (isRecord(first) && typeof first.content === 'string') return first.content;
+	}
+	throw new Error('No prompt found in inputs');
+}
+
+function extractResponderEvals(
+	inputs: Record<string, unknown>,
+	logger?: EvalLogger,
+	index?: number,
+): ResponderEvalCriteria | undefined {
+	const raw = inputs.responderEvals;
+	if (!isRecord(raw)) {
+		logger?.verbose(
+			`[${index ?? '?'}] Example missing responderEvals field - evaluator will report an error`,
+		);
+		return undefined;
+	}
+	if (typeof raw.type !== 'string' || typeof raw.criteria !== 'string') {
+		logger?.verbose(
+			`[${index ?? '?'}] Example has invalid responderEvals (missing type or criteria) - evaluator will report an error`,
+		);
+		return undefined;
+	}
+	return { type: raw.type, criteria: raw.criteria } as ResponderEvalCriteria;
+}
+
+/**
+ * Run a subgraph evaluation against a local dataset (no LangSmith required).
+ */
+export async function runLocalSubgraphEvaluation(
+	config: LocalSubgraphEvaluationConfig,
+): Promise<RunSummary> {
+	const {
+		subgraph,
+		subgraphRunner,
+		evaluators,
+		examples,
+		concurrency,
+		lifecycle,
+		logger,
+		outputDir,
+		timeoutMs,
+		passThreshold = DEFAULT_PASS_THRESHOLD,
+		regenerate,
+		writeBack,
+		datasetFilePath,
+		llms,
+		parsedNodeTypes,
+	} = config;
+
+	if (regenerate && (!llms || !parsedNodeTypes)) {
+		throw new Error('`regenerate` mode requires `llms` and `parsedNodeTypes`');
+	}
+
+	if (writeBack && !datasetFilePath) {
+		throw new Error('`writeBack` requires `datasetFilePath`');
+	}
+
+	const llmCallLimiter = pLimit(concurrency);
+	const artifactSaver = outputDir ? createArtifactSaver({ outputDir, logger }) : null;
+	const capturedResults: ExampleResult[] = [];
+	const writeBackEntries: DatasetWriteBackEntry[] = [];
+
+	const stats = {
+		total: 0,
+		passed: 0,
+		failed: 0,
+		errors: 0,
+		scoreSum: 0,
+		durationSumMs: 0,
+	};
+
+	logger.info(
+		`Starting local subgraph "${subgraph}" evaluation with ${examples.length} examples...`,
+	);
+
+	const evalStartTime = Date.now();
+	const limit = pLimit(concurrency);
+
+	await Promise.all(
+		examples.map(
+			async (example, idx) =>
+				await limit(async () => {
+					const index = idx + 1;
+					const { inputs } = example;
+					const prompt = extractPromptFromInputs(inputs);
+					const startTime = Date.now();
+
+					try {
+						let state: PreComputedState;
+
+						if (regenerate && llms && parsedNodeTypes) {
+							// Regenerate state from prompt
+							logger.verbose(`[${index}] Regenerating workflow state from prompt...`);
+							const regenStart = Date.now();
+							const regenerated = await regenerateWorkflowState({
+								prompt,
+								llms,
+								parsedNodeTypes,
+								timeoutMs,
+								logger,
+							});
+							const regenDurationMs = Date.now() - regenStart;
+							logger.verbose(`[${index}] Regeneration completed in ${regenDurationMs}ms`);
+
+							state = {
+								messages: deserializeMessages(regenerated.messages),
+								coordinationLog: regenerated.coordinationLog,
+								workflowJSON: regenerated.workflowJSON,
+								discoveryContext: regenerated.discoveryContext,
+								previousSummary: regenerated.previousSummary,
+							};
+
+							if (writeBack) {
+								writeBackEntries.push({
+									index: idx,
+									messages: regenerated.messages,
+									coordinationLog: regenerated.coordinationLog,
+									workflowJSON: regenerated.workflowJSON,
+								});
+							}
+						} else {
+							// Use pre-computed state from dataset
+							state = extractPreComputedState(inputs);
+						}
+
+						const genStart = Date.now();
+						const subgraphResult = await runWithOptionalLimiter(async () => {
+							return await withTimeout({
+								promise: subgraphRunner(state),
+								timeoutMs,
+								label: `subgraph:${subgraph}`,
+							});
+						}, llmCallLimiter);
+						const genDurationMs = Date.now() - genStart;
+
+						const context: EvaluationContext & Record<string, unknown> = {
+							prompt,
+							llmCallLimiter,
+							timeoutMs,
+						};
+
+						if (subgraph === 'responder' && subgraphResult.response) {
+							context.responderOutput = subgraphResult.response;
+							context.workflowJSON = state.workflowJSON;
+							const evalCriteria = extractResponderEvals(inputs, logger, index);
+							if (evalCriteria) {
+								context.responderEvals = evalCriteria;
+							}
+						}
+
+						const emptyWorkflow: SimpleWorkflow = { name: '', nodes: [], connections: {} };
+
+						const evalStart = Date.now();
+						const feedback = await runEvaluatorsOnExample(
+							evaluators,
+							emptyWorkflow,
+							context,
+							timeoutMs,
+						);
+						const evalDurationMs = Date.now() - evalStart;
+						const totalDurationMs = Date.now() - startTime;
+
+						const score = calculateWeightedScore(feedback);
+						const hasError = feedback.some((f) => f.metric === 'error');
+						const status = hasError ? 'error' : score >= passThreshold ? 'pass' : 'fail';
+
+						stats.total++;
+						stats.scoreSum += score;
+						stats.durationSumMs += totalDurationMs;
+						if (status === 'pass') stats.passed++;
+						else if (status === 'fail') stats.failed++;
+						else stats.errors++;
+
+						const result: ExampleResult = {
+							index,
+							prompt,
+							status,
+							score,
+							feedback,
+							durationMs: totalDurationMs,
+							generationDurationMs: genDurationMs,
+							evaluationDurationMs: evalDurationMs,
+							subgraphOutput: {
+								response: subgraphResult.response,
+								workflow: subgraphResult.workflow,
+							},
+						};
+
+						artifactSaver?.saveExample(result);
+						capturedResults.push(result);
+						lifecycle?.onExampleComplete?.(index, result);
+					} catch (error) {
+						const errorMessage = error instanceof Error ? error.message : String(error);
+						const totalDurationMs = Date.now() - startTime;
+						const feedback: Feedback[] = [
+							{
+								evaluator: 'runner',
+								metric: 'error',
+								score: 0,
+								kind: 'score',
+								comment: errorMessage,
+							},
+						];
+
+						stats.total++;
+						stats.errors++;
+						stats.durationSumMs += totalDurationMs;
+
+						const result: ExampleResult = {
+							index,
+							prompt,
+							status: 'error',
+							score: 0,
+							feedback,
+							durationMs: totalDurationMs,
+							error: errorMessage,
+						};
+
+						artifactSaver?.saveExample(result);
+						capturedResults.push(result);
+						lifecycle?.onExampleComplete?.(index, result);
+					}
+				}),
+		),
+	);
+
+	logger.info(
+		`Local subgraph evaluation completed in ${((Date.now() - evalStartTime) / 1000).toFixed(1)}s`,
+	);
+
+	const evaluatorAverages = computeEvaluatorAverages(capturedResults);
+
+	const summary: RunSummary = {
+		totalExamples: stats.total,
+		passed: stats.passed,
+		failed: stats.failed,
+		errors: stats.errors,
+		averageScore: stats.total > 0 ? stats.scoreSum / stats.total : 0,
+		totalDurationMs: stats.durationSumMs,
+		evaluatorAverages,
+	};
+
+	if (artifactSaver) {
+		artifactSaver.saveSummary(summary, capturedResults);
+	}
+
+	// Write back regenerated state if requested
+	if (writeBack && datasetFilePath && writeBackEntries.length > 0) {
+		logger.info(`Writing back ${writeBackEntries.length} examples to ${datasetFilePath}...`);
+		writeBackToDatasetFile(datasetFilePath, writeBackEntries);
+		logger.info('Write-back complete');
+	}
+
+	await lifecycle?.onEnd?.(summary);
+
+	return summary;
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/subgraph-evaluation.ts

@@ -0,0 +1,501 @@
+/**
+ * Subgraph evaluation runner.
+ *
+ * Orchestrates running evaluations targeting a specific subgraph (e.g., responder)
+ * using pre-computed state from LangSmith dataset examples.
+ */
+
+import type { Client as LangsmithClient } from 'langsmith/client';
+import { evaluate } from 'langsmith/evaluation';
+import type { Run, Example } from 'langsmith/schemas';
+import { traceable } from 'langsmith/traceable';
+import type { INodeTypeDescription } from 'n8n-workflow';
+import pLimit from 'p-limit';
+
+import { runWithOptionalLimiter, withTimeout, runEvaluatorsOnExample } from './evaluation-helpers';
+import { toLangsmithEvaluationResult } from './feedback';
+import type {
+	Evaluator,
+	EvaluationContext,
+	Feedback,
+	RunSummary,
+	EvaluationLifecycle,
+	LangsmithOptions,
+	ExampleResult,
+} from './harness-types';
+import {
+	writeBackToLangSmithDataset,
+	type LangSmithWriteBackEntry,
+} from './langsmith-dataset-writer';
+import type { EvalLogger } from './logger';
+import { createArtifactSaver } from './output';
+import { calculateWeightedScore, computeEvaluatorAverages } from './score-calculator';
+import {
+	extractPreComputedState,
+	deserializeMessages,
+	type SubgraphRunFn,
+	type SubgraphResult,
+	type PreComputedState,
+	type SubgraphName,
+} from './subgraph-runner';
+import { regenerateWorkflowState } from './workflow-regenerator';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+import type { ResponderEvalCriteria } from '../evaluators/responder/responder-judge.prompt';
+import type { ResolvedStageLLMs } from '../support/environment';
+
+const DEFAULT_PASS_THRESHOLD = 0.7;
+
+interface SubgraphEvaluationConfig {
+	subgraph: SubgraphName;
+	subgraphRunner: SubgraphRunFn;
+	evaluators: Array<Evaluator<EvaluationContext>>;
+	datasetName: string;
+	langsmithClient: LangsmithClient;
+	langsmithOptions: LangsmithOptions;
+	lifecycle?: Partial<EvaluationLifecycle>;
+	logger: EvalLogger;
+	outputDir?: string;
+	timeoutMs?: number;
+	passThreshold?: number;
+	/** Run full workflow generation from prompt instead of using pre-computed state */
+	regenerate?: boolean;
+	/** Write regenerated state back to LangSmith dataset */
+	writeBack?: boolean;
+	/** LLMs for regeneration (required if regenerate is true) */
+	llms?: ResolvedStageLLMs;
+	/** Parsed node types for regeneration (required if regenerate is true) */
+	parsedNodeTypes?: INodeTypeDescription[];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isFeedback(value: unknown): value is Feedback {
+	const kinds = new Set(['score', 'metric', 'detail'] as const);
+	return (
+		isRecord(value) &&
+		typeof value.evaluator === 'string' &&
+		typeof value.metric === 'string' &&
+		typeof value.score === 'number' &&
+		typeof value.kind === 'string' &&
+		kinds.has(value.kind as 'score' | 'metric' | 'detail')
+	);
+}
+
+function isUnknownArray(value: unknown): value is unknown[] {
+	return Array.isArray(value);
+}
+
+function extractPromptFromInputs(inputs: Record<string, unknown>): string {
+	if (typeof inputs.prompt === 'string') return inputs.prompt;
+	if (Array.isArray(inputs.messages) && inputs.messages.length > 0) {
+		const first: unknown = inputs.messages[0];
+		if (isRecord(first) && typeof first.content === 'string') return first.content;
+	}
+	throw new Error('No prompt found in inputs');
+}
+
+function extractResponderEvals(
+	inputs: Record<string, unknown>,
+	logger?: EvalLogger,
+	index?: number,
+): ResponderEvalCriteria | undefined {
+	const raw = inputs.responderEvals;
+	if (!isRecord(raw)) {
+		logger?.verbose(
+			`[${index ?? '?'}] Example missing responderEvals field - evaluator will report an error`,
+		);
+		return undefined;
+	}
+	if (typeof raw.type !== 'string' || typeof raw.criteria !== 'string') {
+		logger?.verbose(
+			`[${index ?? '?'}] Example has invalid responderEvals (missing type or criteria) - evaluator will report an error`,
+		);
+		return undefined;
+	}
+	return { type: raw.type, criteria: raw.criteria } as ResponderEvalCriteria;
+}
+
+interface SubgraphTargetOutput {
+	response?: string;
+	workflow?: SimpleWorkflow;
+	prompt: string;
+	feedback: Feedback[];
+	/** Example ID for write-back (only present when regenerate is used) */
+	exampleId?: string;
+}
+
+interface ResolveStateResult {
+	state: PreComputedState;
+	writeBackEntry?: LangSmithWriteBackEntry;
+}
+
+/**
+ * Resolve the pre-computed state for a subgraph evaluation example.
+ * Either regenerates from prompt or uses the pre-computed state from the dataset.
+ */
+async function resolveState(args: {
+	inputs: Record<string, unknown>;
+	regenerate?: boolean;
+	llms?: ResolvedStageLLMs;
+	parsedNodeTypes?: INodeTypeDescription[];
+	timeoutMs?: number;
+	logger: EvalLogger;
+	index: number;
+	prompt: string;
+	exampleId?: string;
+	writeBack?: boolean;
+}): Promise<ResolveStateResult> {
+	if (args.regenerate && args.llms && args.parsedNodeTypes) {
+		args.logger.verbose(`[${args.index}] Regenerating workflow state from prompt...`);
+		const regenStart = Date.now();
+		const regenerated = await regenerateWorkflowState({
+			prompt: args.prompt,
+			llms: args.llms,
+			parsedNodeTypes: args.parsedNodeTypes,
+			timeoutMs: args.timeoutMs,
+			logger: args.logger,
+		});
+		args.logger.verbose(`[${args.index}] Regeneration completed in ${Date.now() - regenStart}ms`);
+
+		const state: PreComputedState = {
+			messages: deserializeMessages(regenerated.messages),
+			coordinationLog: regenerated.coordinationLog,
+			workflowJSON: regenerated.workflowJSON,
+			discoveryContext: regenerated.discoveryContext,
+			previousSummary: regenerated.previousSummary,
+		};
+
+		const writeBackEntry =
+			args.writeBack && args.exampleId
+				? {
+						exampleId: args.exampleId,
+						messages: regenerated.messages,
+						coordinationLog: regenerated.coordinationLog,
+						workflowJSON: regenerated.workflowJSON,
+					}
+				: undefined;
+
+		return { state, writeBackEntry };
+	}
+
+	return { state: extractPreComputedState(args.inputs) };
+}
+
+/**
+ * Pre-load example IDs from a LangSmith dataset for write-back tracking.
+ */
+async function preloadExampleIds(
+	lsClient: LangsmithClient,
+	datasetName: string,
+	logger: EvalLogger,
+): Promise<string[]> {
+	logger.verbose('Pre-loading example IDs for write-back tracking...');
+	const dataset = await lsClient.readDataset({ datasetName });
+	const examples = lsClient.listExamples({ datasetId: dataset.id });
+	const ids: string[] = [];
+	for await (const example of examples) {
+		ids.push(example.id);
+	}
+	logger.verbose(`Loaded ${ids.length} example IDs for write-back`);
+	return ids;
+}
+
+/**
+ * Extract experiment and dataset IDs from LangSmith evaluate() results.
+ */
+async function extractExperimentIds(
+	experimentResults: Awaited<ReturnType<typeof evaluate>>,
+	logger: EvalLogger,
+): Promise<{ experimentId?: string; datasetId?: string }> {
+	try {
+		const manager = (
+			experimentResults as unknown as {
+				manager?: { _getExperiment?: () => { id: string }; datasetId?: Promise<string> };
+			}
+		).manager;
+		return {
+			experimentId: manager?._getExperiment?.()?.id,
+			datasetId: manager?.datasetId ? await manager.datasetId : undefined,
+		};
+	} catch {
+		logger.verbose('Could not extract LangSmith IDs from experiment results');
+		return {};
+	}
+}
+
+/**
+ * Run a subgraph evaluation against a LangSmith dataset.
+ */
+export async function runSubgraphEvaluation(config: SubgraphEvaluationConfig): Promise<RunSummary> {
+	const {
+		subgraph,
+		subgraphRunner,
+		evaluators,
+		datasetName,
+		langsmithClient: lsClient,
+		langsmithOptions,
+		lifecycle,
+		logger,
+		outputDir,
+		timeoutMs,
+		passThreshold = DEFAULT_PASS_THRESHOLD,
+		regenerate,
+		writeBack,
+		llms,
+		parsedNodeTypes,
+	} = config;
+
+	if (regenerate && (!llms || !parsedNodeTypes)) {
+		throw new Error('`regenerate` mode requires `llms` and `parsedNodeTypes`');
+	}
+
+	process.env.LANGSMITH_TRACING = 'true';
+
+	lifecycle?.onStart?.({
+		mode: 'langsmith',
+		dataset: datasetName,
+		generateWorkflow: async () => ({ name: '', nodes: [], connections: {} }),
+		evaluators,
+		langsmithOptions,
+		langsmithClient: lsClient,
+		logger,
+	});
+
+	const llmCallLimiter = pLimit(langsmithOptions.concurrency);
+	const artifactSaver = outputDir ? createArtifactSaver({ outputDir, logger }) : null;
+	const capturedResults: ExampleResult[] = [];
+	const writeBackEntries: LangSmithWriteBackEntry[] = [];
+
+	let targetCallCount = 0;
+	const stats = {
+		total: 0,
+		passed: 0,
+		failed: 0,
+		errors: 0,
+		scoreSum: 0,
+		durationSumMs: 0,
+	};
+
+	const traceableSubgraphRun = traceable(
+		async (args: {
+			state: PreComputedState;
+			runner: SubgraphRunFn;
+			genTimeoutMs?: number;
+		}): Promise<SubgraphResult> => {
+			return await runWithOptionalLimiter(async () => {
+				return await withTimeout({
+					promise: args.runner(args.state),
+					timeoutMs: args.genTimeoutMs,
+					label: `subgraph:${subgraph}`,
+				});
+			}, llmCallLimiter);
+		},
+		{
+			name: `subgraph_${subgraph}`,
+			run_type: 'chain',
+			client: lsClient,
+		},
+	);
+
+	// Pre-load example IDs if write-back is needed
+	const exampleIds = writeBack ? await preloadExampleIds(lsClient, datasetName, logger) : [];
+
+	const target = async (inputs: Record<string, unknown>): Promise<SubgraphTargetOutput> => {
+		targetCallCount++;
+		const index = targetCallCount;
+		const prompt = extractPromptFromInputs(inputs);
+		// Use index-1 since targetCallCount is 1-based
+		const exampleId = exampleIds[index - 1];
+		const startTime = Date.now();
+
+		try {
+			const { state, writeBackEntry } = await resolveState({
+				inputs,
+				regenerate,
+				llms,
+				parsedNodeTypes,
+				timeoutMs,
+				logger,
+				index,
+				prompt,
+				exampleId,
+				writeBack,
+			});
+			if (writeBackEntry) writeBackEntries.push(writeBackEntry);
+
+			const genStart = Date.now();
+			const subgraphResult = await traceableSubgraphRun({
+				state,
+				runner: subgraphRunner,
+				genTimeoutMs: timeoutMs,
+			});
+			const genDurationMs = Date.now() - genStart;
+
+			// Build evaluation context with subgraph-specific fields
+			const context: EvaluationContext & Record<string, unknown> = {
+				prompt,
+				llmCallLimiter,
+				timeoutMs,
+			};
+
+			if (subgraph === 'responder' && subgraphResult.response) {
+				context.responderOutput = subgraphResult.response;
+				context.workflowJSON = state.workflowJSON;
+				const evalCriteria = extractResponderEvals(inputs, logger, index);
+				if (evalCriteria) {
+					context.responderEvals = evalCriteria;
+				}
+			}
+
+			// Use empty workflow for evaluators that expect it
+			const emptyWorkflow: SimpleWorkflow = { name: '', nodes: [], connections: {} };
+
+			// Run evaluators
+			const evalStart = Date.now();
+			const feedback = await runEvaluatorsOnExample(evaluators, emptyWorkflow, context, timeoutMs);
+			const evalDurationMs = Date.now() - evalStart;
+			const totalDurationMs = Date.now() - startTime;
+
+			const score = calculateWeightedScore(feedback);
+			const hasError = feedback.some((f) => f.metric === 'error');
+			const status = hasError ? 'error' : score >= passThreshold ? 'pass' : 'fail';
+
+			stats.total++;
+			stats.scoreSum += score;
+			stats.durationSumMs += totalDurationMs;
+			if (status === 'pass') stats.passed++;
+			else if (status === 'fail') stats.failed++;
+			else stats.errors++;
+
+			const result: ExampleResult = {
+				index,
+				prompt,
+				status,
+				score,
+				feedback,
+				durationMs: totalDurationMs,
+				generationDurationMs: genDurationMs,
+				evaluationDurationMs: evalDurationMs,
+				subgraphOutput: {
+					response: subgraphResult.response,
+					workflow: subgraphResult.workflow,
+				},
+			};
+
+			artifactSaver?.saveExample(result);
+			capturedResults.push(result);
+			lifecycle?.onExampleComplete?.(index, result);
+
+			return {
+				response: subgraphResult.response,
+				workflow: subgraphResult.workflow,
+				prompt,
+				feedback,
+				exampleId,
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			const totalDurationMs = Date.now() - startTime;
+			const feedback: Feedback[] = [
+				{ evaluator: 'runner', metric: 'error', score: 0, kind: 'score', comment: errorMessage },
+			];
+
+			stats.total++;
+			stats.errors++;
+			stats.durationSumMs += totalDurationMs;
+
+			const result: ExampleResult = {
+				index,
+				prompt,
+				status: 'error',
+				score: 0,
+				feedback,
+				durationMs: totalDurationMs,
+				error: errorMessage,
+			};
+
+			artifactSaver?.saveExample(result);
+			capturedResults.push(result);
+			lifecycle?.onExampleComplete?.(index, result);
+
+			return { prompt, feedback, exampleId };
+		}
+	};
+
+	const feedbackExtractor = async (rootRun: Run, _example?: Example) => {
+		const outputs = rootRun.outputs;
+		const feedback =
+			isRecord(outputs) && isUnknownArray(outputs.feedback) && outputs.feedback.every(isFeedback)
+				? outputs.feedback
+				: undefined;
+
+		if (!feedback) {
+			return [{ key: 'evaluationError', score: 0, comment: 'No feedback found' }];
+		}
+		return feedback.map((fb) => toLangsmithEvaluationResult(fb));
+	};
+
+	logger.info(`Starting subgraph "${subgraph}" evaluation with dataset "${datasetName}"...`);
+
+	const evalStartTime = Date.now();
+	const experimentResults = await evaluate(target, {
+		data: datasetName,
+		evaluators: [feedbackExtractor],
+		experimentPrefix: langsmithOptions.experimentName,
+		maxConcurrency: langsmithOptions.concurrency,
+		client: lsClient,
+		...(langsmithOptions.repetitions > 1 && {
+			numRepetitions: langsmithOptions.repetitions,
+		}),
+		metadata: {
+			subgraph,
+			repetitions: langsmithOptions.repetitions,
+			concurrency: langsmithOptions.concurrency,
+			...langsmithOptions.experimentMetadata,
+		},
+	});
+
+	logger.info(
+		`Subgraph evaluation completed in ${((Date.now() - evalStartTime) / 1000).toFixed(1)}s (target called ${targetCallCount} times)`,
+	);
+
+	logger.verbose('Flushing pending trace batches...');
+	await lsClient.awaitPendingTraceBatches();
+
+	const experimentName = experimentResults.experimentName;
+	logger.info(`Experiment completed: ${experimentName}`);
+
+	const { experimentId, datasetId } = await extractExperimentIds(experimentResults, logger);
+
+	const evaluatorAverages = computeEvaluatorAverages(capturedResults);
+
+	const summary: RunSummary = {
+		totalExamples: stats.total,
+		passed: stats.passed,
+		failed: stats.failed,
+		errors: stats.errors,
+		averageScore: stats.total > 0 ? stats.scoreSum / stats.total : 0,
+		totalDurationMs: stats.durationSumMs,
+		evaluatorAverages,
+		...(experimentName &&
+			experimentId &&
+			datasetId && {
+				langsmith: { experimentName, experimentId, datasetId },
+			}),
+	};
+
+	if (artifactSaver) {
+		artifactSaver.saveSummary(summary, capturedResults);
+	}
+
+	// Write back regenerated state if requested
+	if (writeBack && writeBackEntries.length > 0) {
+		await writeBackToLangSmithDataset(lsClient, writeBackEntries, logger);
+	}
+
+	await lifecycle?.onEnd?.(summary);
+
+	return summary;
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/subgraph-runner.ts

@@ -0,0 +1,131 @@
+import type { BaseMessage } from '@langchain/core/messages';
+import { HumanMessage, AIMessage } from '@langchain/core/messages';
+
+import { createResponderAgent, invokeResponderAgent } from '@/agents/responder.agent';
+import type { ResponderContext } from '@/agents/responder.agent';
+import type { CoordinationLogEntry } from '@/types/coordination';
+import type { DiscoveryContext } from '@/types/discovery-types';
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type { ResolvedStageLLMs } from '../support/environment';
+
+export type SubgraphName = 'responder' | 'discovery' | 'builder' | 'configurator';
+
+/**
+ * Pre-computed state extracted from a dataset example.
+ * Contains all fields needed to run a subgraph without re-running upstream phases.
+ */
+export interface PreComputedState {
+	messages: BaseMessage[];
+	coordinationLog: CoordinationLogEntry[];
+	workflowJSON: SimpleWorkflow;
+	discoveryContext?: DiscoveryContext | null;
+	previousSummary?: string;
+}
+
+export interface SubgraphResult {
+	/** The text response (for responder subgraph) */
+	response?: string;
+	/** The workflow output (for builder/configurator subgraphs) */
+	workflow?: SimpleWorkflow;
+}
+
+export type SubgraphRunFn = (state: PreComputedState) => Promise<SubgraphResult>;
+
+interface SubgraphRunnerConfig {
+	subgraph: SubgraphName;
+	llms: ResolvedStageLLMs;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Deserialize messages from dataset JSON into BaseMessage instances.
+ * Dataset messages are stored as plain objects with `type` and `content` fields.
+ */
+export function deserializeMessages(raw: unknown[]): BaseMessage[] {
+	return raw.map((m) => {
+		if (!isRecord(m)) throw new Error('Invalid message format: expected object');
+		const type = m.type as string;
+		const content = (m.content as string) ?? '';
+		switch (type) {
+			case 'human':
+				return new HumanMessage(content);
+			case 'ai':
+				return new AIMessage(content);
+			default:
+				return new HumanMessage(content);
+		}
+	});
+}
+
+/**
+ * Extract pre-computed state from a dataset example's inputs.
+ */
+export function extractPreComputedState(inputs: Record<string, unknown>): PreComputedState {
+	const rawMessages = inputs.messages;
+	if (!Array.isArray(rawMessages) || rawMessages.length === 0) {
+		throw new Error('Dataset example missing required "messages" field');
+	}
+
+	const messages = deserializeMessages(rawMessages);
+
+	const rawCoordinationLog = inputs.coordinationLog;
+	if (!Array.isArray(rawCoordinationLog)) {
+		throw new Error('Dataset example missing required "coordinationLog" field');
+	}
+
+	const rawWorkflow = inputs.workflowJSON;
+	if (!isRecord(rawWorkflow)) {
+		throw new Error('Dataset example missing required "workflowJSON" field');
+	}
+
+	return {
+		messages,
+		coordinationLog: rawCoordinationLog as CoordinationLogEntry[],
+		workflowJSON: rawWorkflow as SimpleWorkflow,
+		discoveryContext: isRecord(inputs.discoveryContext)
+			? (inputs.discoveryContext as unknown as DiscoveryContext)
+			: null,
+		previousSummary:
+			typeof inputs.previousSummary === 'string' ? inputs.previousSummary : undefined,
+	};
+}
+
+/**
+ * Create a function that runs only the targeted subgraph with pre-computed state.
+ *
+ * For the responder subgraph: creates the agent via createResponderAgent, invokes it
+ * with the pre-computed state, and returns the response text.
+ */
+export function createSubgraphRunner(config: SubgraphRunnerConfig): SubgraphRunFn {
+	const { subgraph, llms } = config;
+
+	switch (subgraph) {
+		case 'responder':
+			return async (state: PreComputedState): Promise<SubgraphResult> => {
+				const agent = createResponderAgent({ llm: llms.responder });
+
+				const context: ResponderContext = {
+					messages: state.messages,
+					coordinationLog: state.coordinationLog,
+					workflowJSON: state.workflowJSON,
+					discoveryContext: state.discoveryContext,
+					previousSummary: state.previousSummary,
+				};
+
+				const result = await invokeResponderAgent(agent, context);
+				const { content } = result.response;
+				const response = typeof content === 'string' ? content : JSON.stringify(content);
+
+				return { response };
+			};
+
+		default:
+			throw new Error(
+				`Subgraph "${subgraph}" is not yet supported. Currently supported: responder`,
+			);
+	}
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/token-tracking-handler.ts

@@ -0,0 +1,4 @@
+export {
+	TokenUsageTrackingHandler,
+	type AccumulatedTokenUsage,
+} from '../../src/utils/token-usage-tracking-handler';

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/workflow-regenerator.ts

@@ -0,0 +1,110 @@
+/**
+ * Workflow regeneration runner.
+ *
+ * Runs the full multi-agent workflow from a prompt and extracts final state
+ * for dataset regeneration purposes.
+ */
+
+import type { BaseMessage } from '@langchain/core/messages';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { CoordinationLogEntry } from '@/types/coordination';
+import type { DiscoveryContext } from '@/types/discovery-types';
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { consumeGenerator, getChatPayload } from './evaluation-helpers';
+import type { EvalLogger } from './logger';
+import { generateRunId, isWorkflowStateValues } from '../langsmith/types';
+import { EVAL_TYPES, EVAL_USERS } from '../support/constants';
+import { createAgent, type ResolvedStageLLMs } from '../support/environment';
+
+/** Serialized message format for JSON storage */
+export interface SerializedMessage {
+	type: 'human' | 'ai';
+	content: string;
+}
+
+/** State extracted from a completed workflow generation */
+export interface RegeneratedState {
+	messages: SerializedMessage[];
+	coordinationLog: CoordinationLogEntry[];
+	workflowJSON: SimpleWorkflow;
+	discoveryContext?: DiscoveryContext | null;
+	previousSummary?: string;
+}
+
+export interface RegenerateOptions {
+	prompt: string;
+	llms: ResolvedStageLLMs;
+	parsedNodeTypes: INodeTypeDescription[];
+	timeoutMs?: number;
+	abortSignal?: AbortSignal;
+	logger?: EvalLogger;
+}
+
+function serializeMessage(msg: BaseMessage): SerializedMessage {
+	const msgType = msg._getType();
+	return {
+		type: msgType === 'human' ? 'human' : 'ai',
+		content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content),
+	};
+}
+
+/**
+ * Run the full multi-agent workflow to generate state from a prompt.
+ * Returns the final messages, coordinationLog, and workflowJSON.
+ */
+export async function regenerateWorkflowState(
+	options: RegenerateOptions,
+): Promise<RegeneratedState> {
+	const { prompt, llms, parsedNodeTypes, abortSignal, logger } = options;
+
+	const runId = generateRunId();
+
+	logger?.verbose(`Regenerating workflow state for prompt: ${prompt.slice(0, 50)}...`);
+
+	const agent = createAgent({
+		parsedNodeTypes,
+		llms,
+	});
+
+	const payload = getChatPayload({
+		evalType: EVAL_TYPES.LANGSMITH,
+		message: prompt,
+		workflowId: runId,
+	});
+
+	await consumeGenerator(agent.chat(payload, EVAL_USERS.LANGSMITH, abortSignal));
+
+	const state = await agent.getState(runId, EVAL_USERS.LANGSMITH);
+
+	if (!state.values || !isWorkflowStateValues(state.values)) {
+		throw new Error('Invalid workflow state: workflow or messages missing');
+	}
+
+	const values = state.values as {
+		messages: BaseMessage[];
+		workflowJSON: SimpleWorkflow;
+		coordinationLog?: CoordinationLogEntry[];
+		discoveryContext?: DiscoveryContext | null;
+		previousSummary?: string;
+	};
+
+	const messages = values.messages.map(serializeMessage);
+	const coordinationLog = values.coordinationLog ?? [];
+	const workflowJSON = values.workflowJSON;
+	const discoveryContext = values.discoveryContext;
+	const previousSummary = values.previousSummary;
+
+	logger?.verbose(
+		`Regeneration complete: ${messages.length} messages, ${coordinationLog.length} log entries`,
+	);
+
+	return {
+		messages,
+		coordinationLog,
+		workflowJSON,
+		discoveryContext,
+		previousSummary,
+	};
+}