first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/argument-parser.ts

@@ -0,0 +1,714 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { z } from 'zod';
+
+import { AVAILABLE_MODELS, DEFAULT_MODEL, type ModelId } from '@/llm-config';
+import type { BuilderFeatureFlags } from '@/workflow-builder-agent';
+
+import type { LangsmithExampleFilters } from '../harness/harness-types';
+import { DEFAULTS } from '../support/constants';
+import type { StageModels } from '../support/environment';
+
+export type EvaluationSuite =
+	| 'llm-judge'
+	| 'pairwise'
+	| 'programmatic'
+	| 'similarity'
+	| 'introspection'
+	| 'binary-checks';
+export type EvaluationBackend = 'local' | 'langsmith';
+export type AgentType = 'multi-agent' | 'code-builder';
+
+export type SubgraphName = 'responder' | 'discovery' | 'builder' | 'configurator';
+
+export interface EvaluationArgs {
+	suite: EvaluationSuite;
+	backend: EvaluationBackend;
+	agent: AgentType;
+
+	verbose: boolean;
+	repetitions: number;
+	concurrency: number;
+	timeoutMs: number;
+	experimentName?: string;
+	outputDir?: string;
+	datasetName?: string;
+	maxExamples?: number;
+	filters?: LangsmithExampleFilters;
+
+	testCase?: string;
+	promptsCsv?: string;
+
+	prompt?: string;
+	dos?: string;
+	donts?: string;
+
+	numJudges: number;
+
+	featureFlags?: BuilderFeatureFlags;
+
+	/** Target a specific subgraph for evaluation (requires --dataset or --dataset-file) */
+	subgraph?: SubgraphName;
+
+	/** Path to a local JSON dataset file (alternative to --dataset for subgraph evals) */
+	datasetFile?: string;
+
+	/** Run full workflow generation from prompt instead of using pre-computed state */
+	regenerate?: boolean;
+
+	/** Write regenerated state back to the dataset source */
+	writeBack?: boolean;
+
+	/** URL to POST evaluation results to when complete */
+	webhookUrl?: string;
+	/** Secret for HMAC-SHA256 signature of webhook payload */
+	webhookSecret?: string;
+
+	/** CSV file path for evaluation results */
+	outputCsv?: string;
+
+	/** Comma-separated list of binary check names to run */
+	checks?: string[];
+
+	// Model configuration
+	/** Default model for all stages */
+	model: ModelId;
+	/** Model for LLM judge evaluation */
+	judgeModel?: ModelId;
+	/** Model for supervisor stage */
+	supervisorModel?: ModelId;
+	/** Model for responder stage */
+	responderModel?: ModelId;
+	/** Model for discovery stage */
+	discoveryModel?: ModelId;
+	/** Model for builder stage (structure and configuration) */
+	builderModel?: ModelId;
+	/** Model for parameter updater (within builder) */
+	parameterUpdaterModel?: ModelId;
+}
+
+type CliValueKind = 'boolean' | 'string';
+type FlagGroup =
+	| 'input'
+	| 'eval'
+	| 'pairwise'
+	| 'langsmith'
+	| 'output'
+	| 'feature'
+	| 'model'
+	| 'advanced';
+
+// Model ID validation schema
+const modelIdSchema = z.enum(AVAILABLE_MODELS as [ModelId, ...ModelId[]]);
+
+const cliSchema = z
+	.object({
+		suite: z
+			.enum([
+				'llm-judge',
+				'pairwise',
+				'programmatic',
+				'similarity',
+				'introspection',
+				'binary-checks',
+			])
+			.default('llm-judge'),
+		backend: z.enum(['local', 'langsmith']).default('local'),
+		agent: z.enum(['code-builder', 'multi-agent']).default('code-builder'),
+
+		verbose: z.boolean().default(false),
+		repetitions: z.coerce.number().int().positive().default(DEFAULTS.REPETITIONS),
+		concurrency: z.coerce.number().int().positive().default(DEFAULTS.CONCURRENCY),
+		timeoutMs: z.coerce.number().int().positive().default(DEFAULTS.TIMEOUT_MS),
+		experimentName: z.string().min(1).optional(),
+		outputDir: z.string().min(1).optional(),
+		outputCsv: z.string().min(1).optional(),
+		datasetName: z.string().min(1).optional(),
+		maxExamples: z.coerce.number().int().positive().optional(),
+		filter: z.array(z.string().min(1)).default([]),
+		notionId: z.string().min(1).optional(),
+		technique: z.string().min(1).optional(),
+
+		subgraph: z.enum(['responder', 'discovery', 'builder', 'configurator']).optional(),
+		datasetFile: z.string().min(1).optional(),
+		regenerate: z.boolean().default(false),
+		writeBack: z.boolean().default(false),
+
+		testCase: z.string().min(1).optional(),
+		promptsCsv: z.string().min(1).optional(),
+
+		prompt: z.string().min(1).optional(),
+		dos: z.string().min(1).optional(),
+		donts: z.string().min(1).optional(),
+
+		numJudges: z.coerce.number().int().positive().default(DEFAULTS.NUM_JUDGES),
+
+		checks: z.string().min(1).optional(),
+		langsmith: z.boolean().optional(),
+		templateExamples: z.boolean().default(false),
+		webhookUrl: z.string().url().optional(),
+		webhookSecret: z.string().min(16).optional(),
+
+		// Model configuration
+		model: modelIdSchema.default(DEFAULT_MODEL),
+		judgeModel: modelIdSchema.optional(),
+		supervisorModel: modelIdSchema.optional(),
+		responderModel: modelIdSchema.optional(),
+		discoveryModel: modelIdSchema.optional(),
+		builderModel: modelIdSchema.optional(),
+		parameterUpdaterModel: modelIdSchema.optional(),
+	})
+	.strict();
+
+type CliKey = keyof z.infer<typeof cliSchema>;
+
+type FlagDef = { key: CliKey; kind: CliValueKind; desc: string; group: FlagGroup };
+
+const FLAG_DEFS: Record<string, FlagDef> = {
+	// Input sources
+	'--prompt': { key: 'prompt', kind: 'string', group: 'input', desc: 'Single prompt to evaluate' },
+	'--prompts-csv': {
+		key: 'promptsCsv',
+		kind: 'string',
+		group: 'input',
+		desc: 'CSV file with prompts',
+	},
+	'--test-case': {
+		key: 'testCase',
+		kind: 'string',
+		group: 'input',
+		desc: 'Run specific default test case by ID',
+	},
+	'--dataset': {
+		key: 'datasetName',
+		kind: 'string',
+		group: 'input',
+		desc: 'LangSmith dataset name',
+	},
+
+	'--subgraph': {
+		key: 'subgraph',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Target subgraph (responder|discovery|builder|configurator). Requires --dataset or --dataset-file',
+	},
+	'--dataset-file': {
+		key: 'datasetFile',
+		kind: 'string',
+		group: 'input',
+		desc: 'Path to local JSON dataset file (for subgraph evals)',
+	},
+	'--regenerate': {
+		key: 'regenerate',
+		kind: 'boolean',
+		group: 'eval',
+		desc: 'Run full workflow generation from prompt instead of using pre-computed state',
+	},
+	'--write-back': {
+		key: 'writeBack',
+		kind: 'boolean',
+		group: 'eval',
+		desc: 'Write regenerated state back to dataset source (requires --regenerate)',
+	},
+
+	// Evaluation options
+	'--suite': {
+		key: 'suite',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Evaluation suite (llm-judge|pairwise|programmatic|similarity|introspection|binary-checks)',
+	},
+	'--checks': {
+		key: 'checks',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Comma-separated binary check names to run (binary-checks suite only)',
+	},
+	'--backend': { key: 'backend', kind: 'string', group: 'eval', desc: 'Backend (local|langsmith)' },
+	'--agent': {
+		key: 'agent',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Agent type (code-builder|multi-agent)',
+	},
+	'--max-examples': {
+		key: 'maxExamples',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Limit number of examples',
+	},
+	'--repetitions': {
+		key: 'repetitions',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Repeat each example N times',
+	},
+	'--concurrency': {
+		key: 'concurrency',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Max parallel evaluations',
+	},
+	'--timeout-ms': {
+		key: 'timeoutMs',
+		kind: 'string',
+		group: 'eval',
+		desc: 'Timeout per evaluation (ms)',
+	},
+
+	// Pairwise options
+	'--dos': {
+		key: 'dos',
+		kind: 'string',
+		group: 'pairwise',
+		desc: 'Requirements the workflow must satisfy',
+	},
+	'--donts': {
+		key: 'donts',
+		kind: 'string',
+		group: 'pairwise',
+		desc: 'Things the workflow must avoid',
+	},
+
+	// LangSmith options
+	'--langsmith': {
+		key: 'langsmith',
+		kind: 'boolean',
+		group: 'langsmith',
+		desc: 'Shorthand for --backend langsmith',
+	},
+	'--name': { key: 'experimentName', kind: 'string', group: 'langsmith', desc: 'Experiment name' },
+	'--filter': {
+		key: 'filter',
+		kind: 'string',
+		group: 'langsmith',
+		desc: 'Filter examples (key:value, repeatable)',
+	},
+	'--notion-id': {
+		key: 'notionId',
+		kind: 'string',
+		group: 'langsmith',
+		desc: 'Filter by Notion ID',
+	},
+	'--technique': {
+		key: 'technique',
+		kind: 'string',
+		group: 'langsmith',
+		desc: 'Filter by technique',
+	},
+
+	// Output
+	'--output-dir': {
+		key: 'outputDir',
+		kind: 'string',
+		group: 'output',
+		desc: 'Directory for artifacts',
+	},
+	'--output-csv': {
+		key: 'outputCsv',
+		kind: 'string',
+		group: 'output',
+		desc: 'CSV file for evaluation results - if pre-existing file found it will be overwritten',
+	},
+	'--verbose': { key: 'verbose', kind: 'boolean', group: 'output', desc: 'Verbose logging' },
+	'--webhook-url': {
+		key: 'webhookUrl',
+		kind: 'string',
+		group: 'output',
+		desc: 'URL to POST results to when complete',
+	},
+	'--webhook-secret': {
+		key: 'webhookSecret',
+		kind: 'string',
+		group: 'output',
+		desc: 'Secret for HMAC-SHA256 signature (min 16 chars)',
+	},
+
+	// Feature flags
+	'--template-examples': {
+		key: 'templateExamples',
+		kind: 'boolean',
+		group: 'feature',
+		desc: 'Enable template examples phase',
+	},
+
+	// Model configuration
+	'--model': {
+		key: 'model',
+		kind: 'string',
+		group: 'model',
+		desc: `Default model for all stages (default: ${DEFAULT_MODEL})`,
+	},
+	'--judge-model': {
+		key: 'judgeModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for LLM judge evaluation',
+	},
+	'--supervisor-model': {
+		key: 'supervisorModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for supervisor stage',
+	},
+	'--responder-model': {
+		key: 'responderModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for responder stage',
+	},
+	'--discovery-model': {
+		key: 'discoveryModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for discovery stage',
+	},
+	'--builder-model': {
+		key: 'builderModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for builder stage (structure and configuration)',
+	},
+	'--parameter-updater-model': {
+		key: 'parameterUpdaterModel',
+		kind: 'string',
+		group: 'model',
+		desc: 'Model for parameter updater',
+	},
+
+	// Advanced
+	'--judges': { key: 'numJudges', kind: 'string', group: 'advanced', desc: 'Number of LLM judges' },
+};
+
+// Aliases (not shown in help)
+const FLAG_ALIASES: Record<string, string> = {
+	'--mode': '--suite',
+	'-v': '--verbose',
+};
+
+// Combined lookup for parsing
+const FLAG_TO_KEY: Record<string, FlagDef> = {
+	...FLAG_DEFS,
+	...Object.fromEntries(
+		Object.entries(FLAG_ALIASES).map(([alias, target]) => [alias, FLAG_DEFS[target]]),
+	),
+};
+
+function formatValidFlags(): string {
+	return Object.keys(FLAG_TO_KEY)
+		.filter((f) => f.startsWith('--'))
+		.sort()
+		.join('\n  ');
+}
+
+const GROUP_TITLES: Record<FlagGroup, string> = {
+	input: 'Input Sources',
+	eval: 'Evaluation Options',
+	pairwise: 'Pairwise Options',
+	langsmith: 'LangSmith Options',
+	output: 'Output',
+	feature: 'Feature Flags',
+	model: 'Model Configuration',
+	advanced: 'Advanced',
+};
+
+function formatHelp(): string {
+	const lines: string[] = [
+		'Usage: pnpm eval [options]',
+		'',
+		'Evaluation harness for AI Workflow Builder.',
+		'',
+	];
+
+	const groups: FlagGroup[] = [
+		'input',
+		'eval',
+		'pairwise',
+		'langsmith',
+		'output',
+		'feature',
+		'model',
+		'advanced',
+	];
+
+	for (const group of groups) {
+		const flags = Object.entries(FLAG_DEFS).filter(([, def]) => def.group === group);
+		if (flags.length === 0) continue;
+
+		lines.push(`${GROUP_TITLES[group]}:`);
+		for (const [flag, def] of flags) {
+			const valueHint = def.kind === 'string' ? ' <value>' : '';
+			const padded = `  ${flag}${valueHint}`.padEnd(28);
+			lines.push(`${padded}${def.desc}`);
+		}
+		lines.push('');
+	}
+
+	lines.push('Examples:');
+	lines.push('  pnpm eval --verbose');
+	lines.push('  pnpm eval --prompt "Create a Slack notification workflow"');
+	lines.push('  pnpm eval --prompts-csv my-prompts.csv --max-examples 5');
+	lines.push('  pnpm eval:langsmith --dataset "workflow-builder-canvas-prompts" --name "test-run"');
+	lines.push(
+		'  pnpm eval:langsmith --subgraph responder --dataset "responder-eval-dataset" --verbose',
+	);
+
+	return lines.join('\n');
+}
+
+export function printHelp(): void {
+	console.log(formatHelp());
+}
+
+function ensureValue(argv: string[], i: number, flag: string): string {
+	const value = argv[i + 1];
+	if (value === undefined) throw new Error(`Flag ${flag} requires a value`);
+	return value;
+}
+
+function splitFlagToken(token: string): { flag: string; inlineValue?: string } {
+	if (!token.startsWith('--')) return { flag: token };
+	const equalsIndex = token.indexOf('=');
+	if (equalsIndex === -1) return { flag: token };
+	return { flag: token.slice(0, equalsIndex), inlineValue: token.slice(equalsIndex + 1) };
+}
+
+function isStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.every((v): v is string => typeof v === 'string');
+}
+
+function parseCli(argv: string[]): {
+	values: Partial<Record<CliKey, unknown>>;
+	seenKeys: Set<CliKey>;
+} {
+	const values: Partial<Record<CliKey, unknown>> = {};
+	const seenKeys = new Set<CliKey>();
+
+	for (let i = 0; i < argv.length; i++) {
+		const token = argv[i];
+		if (!token.startsWith('-')) continue;
+
+		const { flag, inlineValue } = splitFlagToken(token);
+		const def = FLAG_TO_KEY[flag];
+
+		if (!def) {
+			throw new Error(`Unknown flag: ${flag}\n\nValid flags:\n  ${formatValidFlags()}`);
+		}
+
+		seenKeys.add(def.key);
+
+		if (def.kind === 'boolean') {
+			values[def.key] = true;
+			continue;
+		}
+
+		const value = inlineValue ?? ensureValue(argv, i, flag);
+		if (inlineValue === undefined) i++;
+
+		if (def.key === 'filter') {
+			const existing = values.filter;
+			values.filter = isStringArray(existing) ? [...existing, value] : [value];
+			continue;
+		}
+
+		values[def.key] = value;
+	}
+
+	return { values, seenKeys };
+}
+
+function parseFeatureFlags(args: {
+	templateExamples: boolean;
+	suite: EvaluationSuite;
+}): BuilderFeatureFlags | undefined {
+	const templateExamplesFromEnv = process.env.EVAL_FEATURE_TEMPLATE_EXAMPLES === 'true';
+	const templateExamples = templateExamplesFromEnv || args.templateExamples;
+
+	// Auto-enable introspection for introspection suite
+	const enableIntrospection = args.suite === 'introspection';
+
+	if (!templateExamples && !enableIntrospection) return undefined;
+
+	return {
+		templateExamples: templateExamples || undefined,
+		enableIntrospection: enableIntrospection || undefined,
+	};
+}
+
+function parseFilters(args: {
+	filter: string[];
+	notionId?: string;
+	technique?: string;
+}): LangsmithExampleFilters | undefined {
+	const filters: LangsmithExampleFilters = {};
+
+	for (const raw of args.filter) {
+		const match = raw.match(/^(\w+):(.+)$/);
+		if (!match) {
+			throw new Error('Invalid `--filter` format. Expected: --filter "key:value"');
+		}
+
+		const [, key, valueRaw] = match;
+		const value = valueRaw.trim();
+		if (value.length === 0) {
+			throw new Error(`Invalid \`--filter\` value for "${key}": value cannot be empty`);
+		}
+		switch (key) {
+			case 'do':
+				filters.doSearch = value;
+				break;
+			case 'dont':
+				filters.dontSearch = value;
+				break;
+			case 'technique':
+				filters.technique = value;
+				break;
+			case 'id':
+				filters.notionId = value;
+				break;
+			default:
+				throw new Error(`Unknown filter key "${key}". Expected one of: do, dont, technique, id`);
+		}
+	}
+
+	if (args.notionId && !filters.notionId) filters.notionId = args.notionId;
+	if (args.technique && !filters.technique) filters.technique = args.technique;
+
+	const hasAny = Object.values(filters).some((v) => typeof v === 'string' && v.length > 0);
+	return hasAny ? filters : undefined;
+}
+
+function validateSubgraphArgs(parsed: {
+	subgraph?: string;
+	datasetName?: string;
+	datasetFile?: string;
+	prompt?: string;
+	promptsCsv?: string;
+	testCase?: string;
+	writeBack: boolean;
+	regenerate: boolean;
+}): void {
+	if (parsed.subgraph && !parsed.datasetName && !parsed.datasetFile) {
+		throw new Error(
+			'`--subgraph` requires `--dataset` or `--dataset-file`. Subgraph evaluation needs pre-computed state from a dataset.',
+		);
+	}
+
+	if (parsed.subgraph && (parsed.prompt || parsed.promptsCsv || parsed.testCase)) {
+		throw new Error(
+			'`--subgraph` cannot be combined with `--prompt`, `--prompts-csv`, or `--test-case`. Use `--dataset` instead.',
+		);
+	}
+
+	if (parsed.writeBack && !parsed.regenerate) {
+		throw new Error('`--write-back` requires `--regenerate`');
+	}
+
+	if (parsed.regenerate && !parsed.subgraph) {
+		throw new Error('`--regenerate` requires `--subgraph`');
+	}
+}
+
+export function parseEvaluationArgs(argv: string[] = process.argv.slice(2)): EvaluationArgs {
+	// Check for help flag before parsing
+	if (argv.includes('--help') || argv.includes('-h')) {
+		printHelp();
+		process.exit(0);
+	}
+
+	const { values, seenKeys } = parseCli(argv);
+
+	if (values.langsmith === true) {
+		const backendWasExplicit = seenKeys.has('backend');
+		if (backendWasExplicit && values.backend !== 'langsmith') {
+			throw new Error('Cannot combine `--langsmith` with `--backend local`');
+		}
+		values.backend = 'langsmith';
+	}
+
+	const parsed = cliSchema.parse(values);
+
+	const featureFlags = parseFeatureFlags({
+		templateExamples: parsed.templateExamples,
+		suite: parsed.suite,
+	});
+
+	const filters = parseFilters({
+		filter: parsed.filter,
+		notionId: parsed.notionId,
+		technique: parsed.technique,
+	});
+
+	validateSubgraphArgs(parsed);
+
+	if (parsed.suite !== 'pairwise' && (filters?.doSearch || filters?.dontSearch)) {
+		throw new Error(
+			'`--filter do:` and `--filter dont:` are only supported for `--suite pairwise`',
+		);
+	}
+
+	if (parsed.checks && parsed.suite !== 'binary-checks') {
+		throw new Error('`--checks` is only supported for `--suite binary-checks`');
+	}
+
+	return {
+		suite: parsed.suite,
+		backend: parsed.backend,
+		agent: parsed.agent,
+		verbose: parsed.verbose,
+		repetitions: parsed.repetitions,
+		concurrency: parsed.concurrency,
+		timeoutMs: parsed.timeoutMs,
+		experimentName: parsed.experimentName,
+		outputDir: parsed.outputDir,
+		outputCsv: parsed.outputCsv,
+		datasetName: parsed.datasetName,
+		maxExamples: parsed.maxExamples,
+		filters,
+		testCase: parsed.testCase,
+		promptsCsv: parsed.promptsCsv,
+		prompt: parsed.prompt,
+		dos: parsed.dos,
+		donts: parsed.donts,
+		numJudges: parsed.numJudges,
+		featureFlags,
+		subgraph: parsed.subgraph,
+		datasetFile: parsed.datasetFile,
+		regenerate: parsed.regenerate,
+		writeBack: parsed.writeBack,
+		webhookUrl: parsed.webhookUrl,
+		webhookSecret: parsed.webhookSecret,
+		checks: parsed.checks?.split(',').map((s) => s.trim()),
+		// Model configuration
+		model: parsed.model,
+		judgeModel: parsed.judgeModel,
+		supervisorModel: parsed.supervisorModel,
+		responderModel: parsed.responderModel,
+		discoveryModel: parsed.discoveryModel,
+		builderModel: parsed.builderModel,
+		parameterUpdaterModel: parsed.parameterUpdaterModel,
+	};
+}
+
+/**
+ * Converts EvaluationArgs to StageModels for use with environment setup.
+ */
+export function argsToStageModels(args: EvaluationArgs): StageModels {
+	return {
+		default: args.model,
+		supervisor: args.supervisorModel,
+		responder: args.responderModel,
+		discovery: args.discoveryModel,
+		builder: args.builderModel,
+		parameterUpdater: args.parameterUpdaterModel,
+		judge: args.judgeModel,
+	};
+}
+
+export function getDefaultExperimentName(suite: EvaluationSuite): string {
+	return suite === 'pairwise' ? DEFAULTS.EXPERIMENT_NAME : DEFAULTS.LLM_JUDGE_EXPERIMENT_NAME;
+}
+
+export function getDefaultDatasetName(suite: EvaluationSuite): string {
+	if (suite === 'pairwise') return DEFAULTS.DATASET_NAME;
+	return process.env.LANGSMITH_DATASET_NAME ?? 'workflow-builder-canvas-prompts';
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/ci-metadata.ts

@@ -0,0 +1,38 @@
+/**
+ * CI Metadata for LangSmith experiments.
+ *
+ * Provides metadata to distinguish CI runs from local development runs
+ * and to track provenance of automated evaluation results.
+ *
+ * Note: Git info (commit SHA, branch) is not included here as LangSmith
+ * automatically tracks it via LANGSMITH_REVISION_ID and LANGSMITH_BRANCH.
+ */
+
+export interface CIMetadata {
+	/** Whether this run is from CI or local development */
+	source: 'ci' | 'local';
+	/** The GitHub Actions event that triggered this run (e.g., 'push', 'release', 'workflow_dispatch') */
+	trigger?: string;
+	/** The GitHub Actions run ID for linking back to the workflow run */
+	runId?: string;
+}
+
+/**
+ * Build CI metadata from environment variables.
+ *
+ * When running in GitHub Actions, populates metadata from standard GH environment variables.
+ * When running locally, only sets source to 'local'.
+ */
+export function buildCIMetadata(): CIMetadata {
+	const isCI = process.env.GITHUB_ACTIONS === 'true';
+
+	if (!isCI) {
+		return { source: 'local' };
+	}
+
+	return {
+		source: 'ci',
+		trigger: process.env.GITHUB_EVENT_NAME,
+		runId: process.env.GITHUB_RUN_ID,
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/csv-prompt-loader.ts

@@ -0,0 +1,155 @@
+import { parse } from 'csv-parse/sync';
+import { existsSync, readFileSync } from 'node:fs';
+import { join, isAbsolute, resolve } from 'node:path';
+
+import type { TestCase } from '../harness/harness-types';
+
+/** Path to the default prompts CSV fixture */
+const DEFAULT_PROMPTS_PATH = join(__dirname, '..', 'fixtures', 'default-prompts.csv');
+
+type ParsedCsvRow = string[];
+
+function isHeaderRow(row: ParsedCsvRow) {
+	return row.some((cell) => cell.trim().toLowerCase() === 'prompt');
+}
+
+function detectColumnIndex(header: ParsedCsvRow, name: string) {
+	const normalized = name.toLowerCase();
+	const index = header.findIndex((cell) => cell.trim().toLowerCase() === normalized);
+	return index >= 0 ? index : undefined;
+}
+
+function sanitizeValue(value: string | undefined) {
+	return value?.trim() ?? '';
+}
+
+function parseCsv(content: string): ParsedCsvRow[] {
+	try {
+		const rows = parse(content.replace(/^\ufeff/, ''), {
+			columns: false,
+			skip_empty_lines: true,
+			trim: true,
+			relax_column_count: true,
+		}) as ParsedCsvRow[];
+
+		return rows.map((row) => row.map((cell) => cell ?? ''));
+	} catch (error) {
+		const message = error instanceof Error ? error.message : 'Unknown parsing error';
+		throw new Error(`Failed to parse CSV file: ${message}`);
+	}
+}
+
+export function loadTestCasesFromCsv(csvPath: string): TestCase[] {
+	const resolvedPath = isAbsolute(csvPath) ? csvPath : resolve(process.cwd(), csvPath);
+
+	if (!existsSync(resolvedPath)) {
+		throw new Error(`CSV file not found at ${resolvedPath}`);
+	}
+
+	const rows = parseCsv(readFileSync(resolvedPath, 'utf8'));
+
+	if (rows.length === 0) {
+		throw new Error('The provided CSV file is empty');
+	}
+
+	const hasHeader = isHeaderRow(rows[0]);
+	const header = hasHeader ? rows[0] : undefined;
+	const dataRows = hasHeader ? rows.slice(1) : rows;
+
+	if (dataRows.length === 0) {
+		throw new Error('No prompt rows found in the provided CSV file');
+	}
+
+	// Find column index by name(s), returns undefined if no header
+	const findColumn = (...names: string[]): number | undefined => {
+		if (!header) return undefined;
+		for (const name of names) {
+			const idx = detectColumnIndex(header, name);
+			if (idx !== undefined) return idx;
+		}
+		return undefined;
+	};
+
+	const promptIdx = findColumn('prompt') ?? 0;
+	const idIdx = findColumn('id');
+	const dosIdx = findColumn('dos', 'do');
+	const dontsIdx = findColumn('donts', 'dont');
+	const annotationsIdx = findColumn('annotations');
+
+	const getCell = (row: ParsedCsvRow, idx: number | undefined): string =>
+		idx !== undefined ? sanitizeValue(row[idx]) : '';
+
+	const parseAnnotations = (raw: string, rowIndex: number): Record<string, unknown> | undefined => {
+		try {
+			const parsed: unknown = JSON.parse(raw);
+			if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
+				return parsed as Record<string, unknown>;
+			}
+		} catch {
+			console.warn(
+				`Warning: invalid JSON in annotations column for row ${rowIndex + (hasHeader ? 1 : 0)}, skipping`,
+			);
+		}
+		return undefined;
+	};
+
+	const testCases: TestCase[] = [];
+
+	for (let i = 0; i < dataRows.length; i++) {
+		const row = dataRows[i];
+		const prompt = getCell(row, promptIdx);
+
+		if (!prompt) continue;
+
+		const dos = getCell(row, dosIdx);
+		const donts = getCell(row, dontsIdx);
+
+		const testCase: TestCase = {
+			id: getCell(row, idIdx) || `csv-case-${i + 1}`,
+			prompt,
+		};
+
+		const annotationsRaw = getCell(row, annotationsIdx);
+
+		if (dos || donts || annotationsRaw) {
+			testCase.context = {};
+			if (dos) testCase.context.dos = dos;
+			if (donts) testCase.context.donts = donts;
+			if (annotationsRaw) {
+				const annotations = parseAnnotations(annotationsRaw, i + 1);
+				if (annotations) {
+					testCase.context.annotations = annotations;
+				}
+			}
+		}
+
+		testCases.push(testCase);
+	}
+
+	if (testCases.length === 0) {
+		throw new Error('No valid prompts found in the provided CSV file');
+	}
+
+	return testCases;
+}
+
+/** Cached default test cases */
+let cachedDefaultTestCases: TestCase[] | null = null;
+
+/**
+ * Load the default test cases from the bundled CSV fixture.
+ * Results are cached after first load.
+ */
+export function loadDefaultTestCases(): TestCase[] {
+	cachedDefaultTestCases ??= loadTestCasesFromCsv(DEFAULT_PROMPTS_PATH);
+	return cachedDefaultTestCases;
+}
+
+/**
+ * Get available test case IDs from the default fixture.
+ */
+export function getDefaultTestCaseIds(): string[] {
+	return loadDefaultTestCases()
+		.map((tc) => tc.id)
+		.filter((id): id is string => id !== undefined);
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/dataset-file-loader.ts

@@ -0,0 +1,116 @@
+import * as fs from 'fs';
+import { jsonParse } from 'n8n-workflow';
+import * as path from 'path';
+
+import type { CoordinationLogEntry } from '@/types/coordination';
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import type { SerializedMessage } from '../harness/workflow-regenerator';
+
+interface DatasetExample {
+	prompt: string;
+	messages: unknown[];
+	coordinationLog: unknown[];
+	workflowJSON: Record<string, unknown>;
+	responderEvals?: { type: string; criteria: string };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function validateExample(raw: unknown, index: number): DatasetExample {
+	if (!isRecord(raw)) {
+		throw new Error(`Dataset example at index ${index} is not an object`);
+	}
+
+	if (typeof raw.prompt !== 'string' || raw.prompt.length === 0) {
+		throw new Error(`Dataset example at index ${index} missing required "prompt" string`);
+	}
+
+	if (!Array.isArray(raw.messages)) {
+		throw new Error(`Dataset example at index ${index} missing required "messages" array`);
+	}
+
+	if (!Array.isArray(raw.coordinationLog)) {
+		throw new Error(`Dataset example at index ${index} missing required "coordinationLog" array`);
+	}
+
+	if (!isRecord(raw.workflowJSON)) {
+		throw new Error(`Dataset example at index ${index} missing required "workflowJSON" object`);
+	}
+
+	return raw as unknown as DatasetExample;
+}
+
+/**
+ * Load and validate a local JSON dataset file for subgraph evaluation.
+ * Returns parsed examples with their inputs structured for the subgraph runner.
+ */
+export function loadSubgraphDatasetFile(
+	filePath: string,
+): Array<{ inputs: Record<string, unknown> }> {
+	const resolved = path.resolve(filePath);
+
+	if (!fs.existsSync(resolved)) {
+		throw new Error(`Dataset file not found: ${resolved}`);
+	}
+
+	const content = fs.readFileSync(resolved, 'utf-8');
+	let parsed: unknown;
+
+	try {
+		parsed = JSON.parse(content);
+	} catch {
+		throw new Error(`Failed to parse dataset file as JSON: ${resolved}`);
+	}
+
+	if (!Array.isArray(parsed) || parsed.length === 0) {
+		throw new Error(`Dataset file must contain a non-empty JSON array: ${resolved}`);
+	}
+
+	return parsed.map((raw, index) => {
+		const example = validateExample(raw, index);
+		return {
+			inputs: {
+				prompt: example.prompt,
+				messages: example.messages,
+				coordinationLog: example.coordinationLog,
+				workflowJSON: example.workflowJSON,
+				...(example.responderEvals ? { responderEvals: example.responderEvals } : {}),
+			},
+		};
+	});
+}
+
+/** Entry for write-back operations */
+export interface DatasetWriteBackEntry {
+	index: number;
+	messages: SerializedMessage[];
+	coordinationLog: CoordinationLogEntry[];
+	workflowJSON: SimpleWorkflow;
+}
+
+/**
+ * Write regenerated state back to the dataset file.
+ * Preserves existing fields (prompt, responderEvals) and updates state fields.
+ */
+export function writeBackToDatasetFile(filePath: string, updates: DatasetWriteBackEntry[]): void {
+	const resolved = path.resolve(filePath);
+	const content = fs.readFileSync(resolved, 'utf-8');
+	const dataset = jsonParse<unknown[]>(content);
+
+	for (const update of updates) {
+		if (update.index < 0 || update.index >= dataset.length) {
+			throw new Error(
+				`Write-back index ${update.index} is out of bounds (dataset has ${dataset.length} examples)`,
+			);
+		}
+		const example = dataset[update.index] as Record<string, unknown>;
+		example.messages = update.messages;
+		example.coordinationLog = update.coordinationLog;
+		example.workflowJSON = update.workflowJSON;
+	}
+
+	fs.writeFileSync(resolved, JSON.stringify(dataset, null, '\t') + '\n', 'utf-8');
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/index.ts

@@ -0,0 +1,621 @@
+/**
+ * V2 CLI Entry Point
+ *
+ * Demonstrates how to use the v2 evaluation harness.
+ * Can be run directly or used as a reference for custom setups.
+ */
+
+import type { INodeTypeDescription } from 'n8n-workflow';
+import pLimit from 'p-limit';
+
+import { CodeWorkflowBuilder } from '@/code-builder';
+import type { CoordinationLogEntry } from '@/types/coordination';
+import type { StreamChunk, WorkflowUpdateChunk } from '@/types/streaming';
+import type { SimpleWorkflow } from '@/types/workflow';
+import type { BuilderFeatureFlags } from '@/workflow-builder-agent';
+
+/** Type guard for SimpleWorkflow */
+function isSimpleWorkflow(value: unknown): value is SimpleWorkflow {
+	return (
+		typeof value === 'object' &&
+		value !== null &&
+		'name' in value &&
+		'nodes' in value &&
+		'connections' in value
+	);
+}
+
+import {
+	argsToStageModels,
+	getDefaultDatasetName,
+	getDefaultExperimentName,
+	parseEvaluationArgs,
+	type EvaluationArgs,
+} from './argument-parser';
+import { buildCIMetadata } from './ci-metadata';
+import {
+	loadTestCasesFromCsv,
+	loadDefaultTestCases,
+	getDefaultTestCaseIds,
+} from './csv-prompt-loader';
+import { loadSubgraphDatasetFile } from './dataset-file-loader';
+import { sendWebhookNotification } from './webhook';
+import { WorkflowGenerationError } from '../errors';
+import {
+	consumeGenerator,
+	extractSubgraphMetrics,
+	getChatPayload,
+} from '../harness/evaluation-helpers';
+import { createLogger } from '../harness/logger';
+import type { GenerationCollectors, SubgraphMetricsCollector } from '../harness/runner';
+import { TokenUsageTrackingHandler } from '../harness/token-tracking-handler';
+import {
+	runEvaluation,
+	createConsoleLifecycle,
+	mergeLifecycles,
+	createLLMJudgeEvaluator,
+	createProgrammaticEvaluator,
+	createPairwiseEvaluator,
+	createSimilarityEvaluator,
+	createExecutionEvaluator,
+	createBinaryChecksEvaluator,
+	type RunConfig,
+	type TestCase,
+	type Evaluator,
+	type EvaluationContext,
+	type GenerationResult,
+	createSubgraphRunner,
+	createResponderEvaluator,
+	type EvaluationLifecycle,
+	runLocalSubgraphEvaluation,
+	runSubgraphEvaluation,
+} from '../index';
+import { generateRunId, isWorkflowStateValues } from '../langsmith/types';
+import { createIntrospectionAnalysisLifecycle } from '../lifecycles/introspection-analysis';
+import { AGENT_TYPES, EVAL_TYPES, EVAL_USERS } from '../support/constants';
+import {
+	setupTestEnvironment,
+	createAgent,
+	resolveNodesBasePath,
+	type ResolvedStageLLMs,
+	type TestEnvironment,
+} from '../support/environment';
+import { generateEvalPinData } from '../support/pin-data-generator';
+
+/**
+ * Type guard for workflow update chunks from streaming output.
+ */
+function isWorkflowUpdateChunk(chunk: StreamChunk): chunk is WorkflowUpdateChunk {
+	return chunk.type === 'workflow-updated';
+}
+
+/**
+ * Type guard to check if state values contain a coordination log.
+ */
+function hasCoordinationLog(
+	values: unknown,
+): values is { coordinationLog: CoordinationLogEntry[] } {
+	if (!values || typeof values !== 'object') return false;
+	const obj = values as Record<string, unknown>;
+	return Array.isArray(obj.coordinationLog);
+}
+
+/**
+ * Report subgraph metrics from coordination log and workflow.
+ */
+function reportSubgraphMetrics(
+	collector: SubgraphMetricsCollector,
+	stateValues: unknown,
+	workflow: SimpleWorkflow,
+): void {
+	const coordinationLog = hasCoordinationLog(stateValues) ? stateValues.coordinationLog : undefined;
+	const nodeCount = workflow.nodes?.length;
+	const metrics = extractSubgraphMetrics(coordinationLog, nodeCount);
+
+	if (
+		metrics.discoveryDurationMs !== undefined ||
+		metrics.builderDurationMs !== undefined ||
+		metrics.responderDurationMs !== undefined ||
+		metrics.nodeCount !== undefined
+	) {
+		collector(metrics);
+	}
+}
+
+/**
+ * Create a workflow generator function for the multi-agent system.
+ * LangSmith tracing is handled via traceable() in the runner.
+ * Callbacks are passed explicitly from the runner to ensure correct trace context
+ * under high concurrency (avoids AsyncLocalStorage race conditions).
+ */
+function createWorkflowGenerator(
+	parsedNodeTypes: INodeTypeDescription[],
+	llms: ResolvedStageLLMs,
+	featureFlags?: BuilderFeatureFlags,
+): (prompt: string, collectors?: GenerationCollectors) => Promise<SimpleWorkflow> {
+	return async (prompt: string, collectors?: GenerationCollectors): Promise<SimpleWorkflow> => {
+		const runId = generateRunId();
+
+		const agent = createAgent({
+			parsedNodeTypes,
+			llms,
+			featureFlags,
+		});
+
+		// Create token tracking handler to capture usage from all LLM calls
+		// (supervisor, discovery, builder, responder agents)
+		const tokenTracker = collectors?.tokenUsage ? new TokenUsageTrackingHandler() : undefined;
+
+		await consumeGenerator(
+			agent.chat(
+				getChatPayload({
+					evalType: EVAL_TYPES.LANGSMITH,
+					message: prompt,
+					workflowId: runId,
+					featureFlags,
+				}),
+				EVAL_USERS.LANGSMITH,
+				undefined, // abortSignal
+				tokenTracker ? [tokenTracker] : undefined, // externalCallbacks
+			),
+		);
+
+		const state = await agent.getState(runId, EVAL_USERS.LANGSMITH);
+
+		if (!state.values || !isWorkflowStateValues(state.values)) {
+			throw new Error('Invalid workflow state: workflow or messages missing');
+		}
+
+		const workflow = state.values.workflowJSON;
+
+		// Report accumulated token usage from all agents
+		if (collectors?.tokenUsage && tokenTracker) {
+			const usage = tokenTracker.getUsage();
+			if (usage.inputTokens > 0 || usage.outputTokens > 0) {
+				collectors.tokenUsage(usage);
+			}
+		}
+
+		// Extract and report subgraph metrics from coordination log
+		if (collectors?.subgraphMetrics) {
+			reportSubgraphMetrics(collectors.subgraphMetrics, state.values, workflow);
+		}
+
+		// Report introspection events
+		collectors?.introspectionEvents?.(state.values.introspectionEvents ?? []);
+
+		return workflow;
+	};
+}
+
+/**
+ * Create evaluators based on suite type.
+ */
+function createEvaluators(params: {
+	suite: string;
+	judgeLlm: ResolvedStageLLMs['judge'];
+	parsedNodeTypes: Parameters<typeof createProgrammaticEvaluator>[0];
+	numJudges: number;
+	checks?: string[];
+}): Array<Evaluator<EvaluationContext>> {
+	const { suite, judgeLlm, parsedNodeTypes, numJudges, checks } = params;
+	const evaluators: Array<Evaluator<EvaluationContext>> = [];
+
+	switch (suite) {
+		case 'llm-judge':
+			evaluators.push(createLLMJudgeEvaluator(judgeLlm, parsedNodeTypes));
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'pairwise':
+			evaluators.push(createPairwiseEvaluator(judgeLlm, { numJudges }));
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'programmatic':
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'similarity':
+			evaluators.push(createSimilarityEvaluator());
+			break;
+		case 'binary-checks':
+			evaluators.push(
+				createBinaryChecksEvaluator({
+					nodeTypes: parsedNodeTypes,
+					llm: judgeLlm,
+					checks,
+				}),
+			);
+			break;
+	}
+
+	return evaluators;
+}
+
+/**
+ * Create a CodeWorkflowBuilder generator function.
+ * Uses the CodeWorkflowBuilder which coordinates planning and coding agents to generate
+ * workflows via TypeScript SDK code and emits workflow JSON directly in the stream.
+ * Returns GenerationResult including the source code for artifact saving.
+ *
+ * @param timeoutMs - Optional timeout in milliseconds. When provided, the agent will be
+ *                    aborted if it exceeds this duration. This ensures the generator
+ *                    actually stops instead of continuing to run after timeout rejection.
+ */
+function createCodeWorkflowBuilderGenerator(
+	parsedNodeTypes: INodeTypeDescription[],
+	llms: ResolvedStageLLMs,
+	timeoutMs?: number,
+	nodeDefinitionDirs?: string[],
+): (prompt: string, collectors?: GenerationCollectors) => Promise<GenerationResult> {
+	// Subgraph metrics are not applicable since CodeWorkflowBuilder doesn't use coordination logs.
+	return async (prompt: string, collectors?: GenerationCollectors): Promise<GenerationResult> => {
+		const runId = generateRunId();
+
+		// Accumulate token usage across all LLM calls
+		let totalInputTokens = 0;
+		let totalOutputTokens = 0;
+
+		const builder = new CodeWorkflowBuilder({
+			llm: llms.builder,
+			nodeTypes: parsedNodeTypes,
+			nodeDefinitionDirs,
+			onTokenUsage: collectors?.tokenUsage
+				? (usage) => {
+						totalInputTokens += usage.inputTokens;
+						totalOutputTokens += usage.outputTokens;
+					}
+				: undefined,
+		});
+
+		const payload = getChatPayload({
+			evalType: EVAL_TYPES.LANGSMITH,
+			message: prompt,
+			workflowId: runId,
+			featureFlags: { codeBuilder: true },
+		});
+
+		let workflow: SimpleWorkflow | null = null;
+		let generatedCode: string | undefined;
+
+		// Create an AbortController to properly cancel the agent on timeout or error.
+		// Without this, the agent continues running even after Promise.race rejects,
+		// causing the full timeout duration to elapse before the error surfaces.
+		const abortController = new AbortController();
+		let timeoutId: NodeJS.Timeout | undefined;
+
+		if (timeoutMs !== undefined && timeoutMs > 0) {
+			timeoutId = setTimeout(() => {
+				abortController.abort(new Error(`CodeWorkflowBuilder timed out after ${timeoutMs}ms`));
+			}, timeoutMs);
+		}
+
+		try {
+			for await (const output of builder.chat(
+				payload,
+				EVAL_USERS.LANGSMITH,
+				abortController.signal,
+			)) {
+				for (const message of output.messages) {
+					if (isWorkflowUpdateChunk(message)) {
+						const parsed: unknown = JSON.parse(message.codeSnippet);
+						if (isSimpleWorkflow(parsed)) {
+							workflow = parsed;
+							generatedCode = message.sourceCode;
+						}
+					}
+				}
+			}
+		} finally {
+			if (timeoutId !== undefined) {
+				clearTimeout(timeoutId);
+			}
+		}
+
+		if (!workflow) {
+			throw new WorkflowGenerationError('CodeWorkflowBuilder did not produce a workflow');
+		}
+
+		// Report accumulated token usage
+		if (collectors?.tokenUsage && (totalInputTokens > 0 || totalOutputTokens > 0)) {
+			collectors.tokenUsage({ inputTokens: totalInputTokens, outputTokens: totalOutputTokens });
+		}
+
+		return { workflow, generatedCode };
+	};
+}
+
+/**
+ * Load test cases from various sources.
+ */
+function loadTestCases(args: ReturnType<typeof parseEvaluationArgs>): TestCase[] {
+	// From CSV file
+	if (args.promptsCsv) {
+		const testCases = loadTestCasesFromCsv(args.promptsCsv);
+		return args.maxExamples ? testCases.slice(0, args.maxExamples) : testCases;
+	}
+
+	// Predefined test case by id
+	if (args.testCase) {
+		const defaultCases = loadDefaultTestCases();
+		const match = defaultCases.find((tc) => tc.id === args.testCase);
+		if (!match) {
+			const options = getDefaultTestCaseIds().join(', ');
+			throw new Error(`Unknown --test-case "${args.testCase}". Available: ${options}`);
+		}
+
+		const testCases: TestCase[] = [
+			{
+				prompt: match.prompt,
+				id: match.id,
+				context: { dos: args.dos, donts: args.donts },
+			},
+		];
+
+		return args.maxExamples ? testCases.slice(0, args.maxExamples) : testCases;
+	}
+
+	// Single prompt from CLI
+	if (args.prompt) {
+		const testCases: TestCase[] = [
+			{
+				prompt: args.prompt,
+				context: {
+					dos: args.dos,
+					donts: args.donts,
+				},
+			},
+		];
+		return args.maxExamples ? testCases.slice(0, args.maxExamples) : testCases;
+	}
+
+	// Default: use bundled test cases
+	const defaultCases = loadDefaultTestCases();
+	return args.maxExamples ? defaultCases.slice(0, args.maxExamples) : defaultCases;
+}
+
+/**
+ * Handle subgraph evaluation mode (--subgraph flag).
+ * Supports both local dataset files and LangSmith datasets.
+ */
+async function handleSubgraphMode(
+	args: EvaluationArgs,
+	env: TestEnvironment,
+	lifecycle: EvaluationLifecycle,
+	logger: ReturnType<typeof createLogger>,
+): Promise<void> {
+	const { subgraph } = args;
+	if (!subgraph) throw new Error('subgraph is required');
+
+	const subgraphRunner = createSubgraphRunner({
+		subgraph,
+		llms: env.llms,
+	});
+
+	const evaluators: Array<Evaluator<EvaluationContext>> = [];
+	if (subgraph === 'responder') {
+		evaluators.push(createResponderEvaluator(env.llms.judge, { numJudges: args.numJudges }));
+	} else {
+		logger.warn(`Subgraph evaluation not supported for ${subgraph}`);
+	}
+
+	let summary: Awaited<ReturnType<typeof runSubgraphEvaluation>>;
+
+	if (args.datasetFile) {
+		const examples = loadSubgraphDatasetFile(args.datasetFile);
+		const slicedExamples = args.maxExamples ? examples.slice(0, args.maxExamples) : examples;
+
+		summary = await runLocalSubgraphEvaluation({
+			subgraph,
+			subgraphRunner,
+			evaluators,
+			examples: slicedExamples,
+			concurrency: args.concurrency,
+			lifecycle,
+			logger,
+			outputDir: args.outputDir,
+			timeoutMs: args.timeoutMs,
+			regenerate: args.regenerate,
+			writeBack: args.writeBack,
+			datasetFilePath: args.datasetFile,
+			llms: args.regenerate ? env.llms : undefined,
+			parsedNodeTypes: args.regenerate ? env.parsedNodeTypes : undefined,
+		});
+	} else {
+		if (!args.datasetName) {
+			throw new Error('`--subgraph` requires `--dataset` or `--dataset-file`');
+		}
+		if (!env.lsClient) {
+			throw new Error('LangSmith client not initialized - check LANGSMITH_API_KEY');
+		}
+
+		summary = await runSubgraphEvaluation({
+			subgraph,
+			subgraphRunner,
+			evaluators,
+			datasetName: args.datasetName,
+			langsmithClient: env.lsClient,
+			langsmithOptions: {
+				experimentName: args.experimentName ?? `${subgraph}-eval`,
+				repetitions: args.repetitions,
+				concurrency: args.concurrency,
+				maxExamples: args.maxExamples,
+				filters: args.filters,
+				experimentMetadata: {
+					...buildCIMetadata(),
+					subgraph,
+				},
+			},
+			lifecycle,
+			logger,
+			outputDir: args.outputDir,
+			timeoutMs: args.timeoutMs,
+			regenerate: args.regenerate,
+			writeBack: args.writeBack,
+			llms: args.regenerate ? env.llms : undefined,
+			parsedNodeTypes: args.regenerate ? env.parsedNodeTypes : undefined,
+		});
+	}
+
+	if (args.webhookUrl) {
+		await sendWebhookNotification({
+			webhookUrl: args.webhookUrl,
+			webhookSecret: args.webhookSecret,
+			summary,
+			dataset: args.datasetFile ?? args.datasetName ?? 'local-dataset',
+			suite: args.suite,
+			metadata: { ...buildCIMetadata(), subgraph },
+			logger,
+		});
+	}
+}
+
+/**
+ * Main entry point for v2 evaluation CLI.
+ */
+export async function runV2Evaluation(): Promise<void> {
+	const args = parseEvaluationArgs();
+
+	if (args.backend === 'langsmith' && (args.prompt || args.promptsCsv || args.testCase)) {
+		throw new Error(
+			'LangSmith mode requires `--dataset` and does not support `--prompt`, `--prompts-csv`, or `--test-case`',
+		);
+	}
+
+	// Setup environment with per-stage model configuration
+	const logger = createLogger(args.verbose);
+	const lifecycle = createConsoleLifecycle({ verbose: args.verbose, logger });
+	const stageModels = argsToStageModels(args);
+	const env = await setupTestEnvironment(stageModels, logger);
+
+	// Validate LangSmith client early if langsmith backend is requested
+	if (args.backend === 'langsmith' && !env.lsClient) {
+		throw new Error('LangSmith client not initialized - check LANGSMITH_API_KEY');
+	}
+
+	// Subgraph evaluation mode
+	if (args.subgraph) {
+		await handleSubgraphMode(args, env, lifecycle, logger);
+		process.exit(0);
+	}
+
+	// Create workflow generator based on agent type
+	const generateWorkflow =
+		args.agent === AGENT_TYPES.CODE_BUILDER
+			? createCodeWorkflowBuilderGenerator(
+					env.parsedNodeTypes,
+					env.llms,
+					args.timeoutMs,
+					env.nodeDefinitionDirs,
+				)
+			: createWorkflowGenerator(env.parsedNodeTypes, env.llms, args.featureFlags);
+
+	// Create evaluators based on suite type
+	const evaluators = createEvaluators({
+		suite: args.suite,
+		judgeLlm: env.llms.judge,
+		parsedNodeTypes: env.parsedNodeTypes,
+		numJudges: args.numJudges,
+		checks: args.checks,
+	});
+
+	// Execution evaluator runs for all suites — validates workflows execute with pin data
+	evaluators.push(createExecutionEvaluator());
+
+	const llmCallLimiter = pLimit(args.concurrency);
+
+	// Merge console lifecycle with optional introspection analysis lifecycle
+	const mergedLifecycle = mergeLifecycles(
+		createConsoleLifecycle({ verbose: args.verbose, logger }),
+		args.suite === 'introspection'
+			? createIntrospectionAnalysisLifecycle({
+					judgeLlm: env.llms.judge,
+					outputDir: args.outputDir,
+					logger,
+				})
+			: undefined,
+	);
+	// Create pin data generator for mocking service node outputs in evaluations
+	const nodesBasePath = resolveNodesBasePath();
+	const pinDataGenerator = async (workflow: SimpleWorkflow) =>
+		await generateEvalPinData(workflow, {
+			llm: env.llms.judge,
+			nodeTypes: env.parsedNodeTypes,
+			nodesBasePath,
+			logger,
+		});
+
+	const baseConfig = {
+		generateWorkflow,
+		evaluators,
+		lifecycle: mergedLifecycle,
+		logger,
+		outputDir: args.outputDir,
+		outputCsv: args.outputCsv,
+		suite: args.suite,
+		timeoutMs: args.timeoutMs,
+		context: { llmCallLimiter },
+		passThreshold: args.suite === 'introspection' || args.suite === 'binary-checks' ? 0 : undefined,
+		pinDataGenerator,
+	};
+
+	const config: RunConfig =
+		args.backend === 'langsmith'
+			? {
+					...baseConfig,
+					mode: 'langsmith',
+					dataset: args.datasetName ?? getDefaultDatasetName(args.suite),
+					langsmithClient: env.lsClient!,
+					langsmithOptions: {
+						experimentName: args.experimentName ?? getDefaultExperimentName(args.suite),
+						repetitions: args.repetitions,
+						concurrency: args.concurrency,
+						maxExamples: args.maxExamples,
+						filters: args.filters,
+						experimentMetadata: {
+							...buildCIMetadata(),
+							...(args.suite === 'pairwise' && {
+								numJudges: args.numJudges,
+								scoringMethod: 'hierarchical',
+							}),
+						},
+					},
+				}
+			: {
+					...baseConfig,
+					mode: 'local',
+					dataset: loadTestCases(args),
+					concurrency: args.concurrency,
+				};
+
+	// Run evaluation
+	const summary = await runEvaluation(config);
+
+	if (args.webhookUrl) {
+		const dataset =
+			args.backend === 'langsmith'
+				? (args.datasetName ?? getDefaultDatasetName(args.suite))
+				: 'local-dataset';
+
+		await sendWebhookNotification({
+			webhookUrl: args.webhookUrl,
+			webhookSecret: args.webhookSecret,
+			summary,
+			dataset,
+			suite: args.suite,
+			metadata: { ...buildCIMetadata() },
+			logger,
+		});
+	}
+
+	// Always exit 0 on successful completion - pass/fail is informational, not an error
+	process.exit(0);
+}
+
+// Run if called directly
+if (require.main === module) {
+	runV2Evaluation().catch((error) => {
+		const logger = createLogger(true);
+		const message = error instanceof Error ? (error.stack ?? error.message) : String(error);
+		logger.error(`Evaluation failed: ${message}`);
+		process.exit(1);
+	});
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/webhook.ts

@@ -0,0 +1,263 @@
+/**
+ * Webhook utilities for sending evaluation results.
+ */
+
+import { createHmac, timingSafeEqual } from 'node:crypto';
+import dns from 'node:dns/promises';
+
+import type { RunSummary } from '../harness/harness-types';
+import type { EvalLogger } from '../harness/logger';
+
+/**
+ * Mask a webhook URL for safe logging (hide potential tokens in path/query).
+ */
+function maskWebhookUrl(webhookUrl: string): string {
+	const url = new URL(webhookUrl);
+	return `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}/***`;
+}
+
+/**
+ * Generate HMAC-SHA256 signature for webhook payload.
+ * Format: sha256=<hex-encoded-signature>
+ */
+export function generateWebhookSignature(payload: string, secret: string): string {
+	const signature = createHmac('sha256', secret).update(payload, 'utf8').digest('hex');
+	return `sha256=${signature}`;
+}
+
+/**
+ * Verify HMAC-SHA256 signature of a webhook payload.
+ * Uses timing-safe comparison to prevent timing attacks.
+ */
+export function verifyWebhookSignature(
+	payload: string,
+	signature: string,
+	secret: string,
+): boolean {
+	const expectedSignature = generateWebhookSignature(payload, secret);
+
+	// Ensure both signatures have same length before comparison
+	if (signature.length !== expectedSignature.length) {
+		return false;
+	}
+
+	return timingSafeEqual(Buffer.from(signature), Buffer.from(expectedSignature));
+}
+
+/**
+ * Webhook payload sent after evaluation completes.
+ */
+export interface WebhookPayload {
+	suite: string;
+	summary: {
+		totalExamples: number;
+		passed: number;
+		failed: number;
+		errors: number;
+		averageScore: number;
+	};
+	evaluatorAverages?: Record<string, number>;
+	totalDurationMs: number;
+	metadata: Record<string, unknown>;
+	/** LangSmith IDs for constructing comparison URLs (only available in langsmith mode) */
+	langsmith?: {
+		experimentName: string;
+		experimentId: string;
+		datasetId: string;
+		datasetName: string;
+	};
+}
+
+/**
+ * Check if an IP address is private/internal.
+ */
+function isPrivateIp(ip: string): boolean {
+	const ipv4PrivatePatterns = [
+		/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // 127.0.0.0/8 (loopback)
+		/^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // 10.0.0.0/8
+		/^172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}$/, // 172.16.0.0/12
+		/^192\.168\.\d{1,3}\.\d{1,3}$/, // 192.168.0.0/16
+		/^169\.254\.\d{1,3}\.\d{1,3}$/, // 169.254.0.0/16 (link-local)
+		/^0\.0\.0\.0$/, // 0.0.0.0
+	];
+
+	for (const pattern of ipv4PrivatePatterns) {
+		if (pattern.test(ip)) {
+			return true;
+		}
+	}
+
+	const ipLower = ip.toLowerCase();
+	if (
+		ipLower === '::1' || // loopback
+		ipLower.startsWith('fe80:') || // link-local
+		ipLower.startsWith('fc') || // unique local (fc00::/7)
+		ipLower.startsWith('fd') // unique local (fc00::/7)
+	) {
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Validate webhook URL for security (hostname-based checks only).
+ * - Must be HTTPS
+ * - Must not target localhost or private/internal IP addresses (SSRF prevention)
+ *
+ * Note: This performs synchronous hostname string validation.
+ * For full SSRF protection, use validateWebhookUrlWithDns() which also resolves DNS.
+ */
+export function validateWebhookUrl(webhookUrl: string): void {
+	const url = new URL(webhookUrl);
+
+	if (url.protocol !== 'https:') {
+		throw new Error(`Webhook URL must use HTTPS. Got: ${url.protocol}`);
+	}
+
+	const hostname = url.hostname.toLowerCase();
+
+	if (
+		hostname === 'localhost' ||
+		hostname === '127.0.0.1' ||
+		hostname === '::1' ||
+		hostname === '[::1]'
+	) {
+		throw new Error('Webhook URL cannot target localhost');
+	}
+
+	if (isPrivateIp(hostname)) {
+		throw new Error('Webhook URL cannot target private/internal IP addresses');
+	}
+
+	const blockedHostnames = ['internal', 'intranet', 'corp', 'private', 'local'];
+	for (const blocked of blockedHostnames) {
+		if (hostname === blocked || hostname.endsWith(`.${blocked}`)) {
+			throw new Error(`Webhook URL cannot target internal hostname: ${hostname}`);
+		}
+	}
+}
+
+/**
+ * Validate webhook URL with DNS resolution for comprehensive SSRF protection.
+ * Resolves the hostname and validates that resolved IPs are not private/internal.
+ */
+export async function validateWebhookUrlWithDns(webhookUrl: string): Promise<void> {
+	validateWebhookUrl(webhookUrl);
+
+	const url = new URL(webhookUrl);
+	const hostname = url.hostname.toLowerCase();
+
+	if (isPrivateIp(hostname)) {
+		throw new Error('Webhook URL cannot target private/internal IP addresses');
+	}
+
+	try {
+		const addresses = await dns.resolve(hostname);
+		for (const ip of addresses) {
+			if (isPrivateIp(ip)) {
+				throw new Error('Webhook URL hostname resolves to a private/internal IP address');
+			}
+		}
+
+		try {
+			const ipv6Addresses = await dns.resolve6(hostname);
+			for (const ip of ipv6Addresses) {
+				if (isPrivateIp(ip)) {
+					throw new Error('Webhook URL hostname resolves to a private/internal IP address');
+				}
+			}
+		} catch {
+			// IPv6 resolution may fail if no AAAA records exist, which is fine
+		}
+	} catch (error) {
+		if (error instanceof Error && error.message.includes('private/internal')) {
+			throw error;
+		}
+	}
+}
+
+/** Header name for HMAC signature */
+export const WEBHOOK_SIGNATURE_HEADER = 'X-Signature-256';
+
+/** Header name for timestamp (for replay attack prevention) */
+export const WEBHOOK_TIMESTAMP_HEADER = 'X-Timestamp';
+
+/**
+ * Send evaluation results to a webhook URL.
+ *
+ * @param params.webhookUrl - The URL to POST results to
+ * @param params.webhookSecret - Optional secret for HMAC signature (recommended for production)
+ * @param params.summary - Evaluation run summary
+ * @param params.dataset - Dataset name
+ * @param params.suite - Evaluation suite name
+ * @param params.metadata - Additional metadata to include
+ * @param params.logger - Logger instance
+ */
+export async function sendWebhookNotification(params: {
+	webhookUrl: string;
+	webhookSecret?: string;
+	summary: RunSummary;
+	dataset: string;
+	suite: string;
+	metadata: Record<string, unknown>;
+	logger: EvalLogger;
+}): Promise<void> {
+	const { webhookUrl, webhookSecret, summary, dataset, suite, metadata, logger } = params;
+
+	await validateWebhookUrlWithDns(webhookUrl);
+
+	const payload: WebhookPayload = {
+		suite,
+		summary: {
+			totalExamples: summary.totalExamples,
+			passed: summary.passed,
+			failed: summary.failed,
+			errors: summary.errors,
+			averageScore: summary.averageScore,
+		},
+		evaluatorAverages: summary.evaluatorAverages,
+		totalDurationMs: summary.totalDurationMs,
+		metadata,
+		langsmith: summary.langsmith
+			? {
+					...summary.langsmith,
+					datasetName: dataset,
+				}
+			: undefined,
+	};
+
+	const body = JSON.stringify(payload);
+
+	const headers: Record<string, string> = {
+		'Content-Type': 'application/json',
+	};
+
+	if (webhookSecret) {
+		const timestamp = Date.now().toString();
+		const signaturePayload = `${timestamp}.${body}`;
+		headers[WEBHOOK_SIGNATURE_HEADER] = generateWebhookSignature(signaturePayload, webhookSecret);
+		headers[WEBHOOK_TIMESTAMP_HEADER] = timestamp;
+		logger.info('Webhook request will be signed with HMAC-SHA256');
+	} else {
+		logger.warn(
+			'No webhook secret provided - request will not be signed. ' +
+				'Consider using --webhook-secret for production use.',
+		);
+	}
+
+	// Log masked URL to avoid exposing potential tokens in path/query
+	logger.info(`Sending results to webhook: ${maskWebhookUrl(webhookUrl)}`);
+
+	const response = await fetch(webhookUrl, {
+		method: 'POST',
+		headers,
+		body,
+	});
+
+	if (!response.ok) {
+		throw new Error(`Webhook request failed: ${response.status} ${response.statusText}`);
+	}
+
+	logger.info(`Webhook notification sent successfully (status: ${response.status})`);
+}