first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/support/constants.ts

@@ -0,0 +1,46 @@
+// ============================================================================
+// Agent Types
+// ============================================================================
+
+export const AGENT_TYPES = {
+	CODE_BUILDER: 'code-builder',
+	MULTI_AGENT: 'multi-agent',
+} as const;
+
+// ============================================================================
+// Evaluation Type Identifiers
+// ============================================================================
+
+export const EVAL_TYPES = {
+	PAIRWISE_LOCAL: 'pairwise-local',
+	PAIRWISE_LANGSMITH: 'pairwise-langsmith',
+	LANGSMITH: 'langsmith-evals',
+} as const;
+
+export const EVAL_USERS = {
+	PAIRWISE_LOCAL: 'pairwise-local-user',
+	LANGSMITH: 'langsmith-eval-user',
+} as const;
+
+export const TRACEABLE_NAMES = {
+	PAIRWISE_EVALUATION: 'pairwise_evaluation',
+	WORKFLOW_GENERATION: 'workflow_generation',
+} as const;
+
+// ============================================================================
+// Default Values
+// ============================================================================
+
+export const DEFAULTS = {
+	NUM_JUDGES: 3,
+	EXPERIMENT_NAME: 'pairwise-evals',
+	LLM_JUDGE_EXPERIMENT_NAME: 'workflow-builder-evaluation',
+	CONCURRENCY: 5,
+	REPETITIONS: 1,
+	/** Per-operation timeout (generation / evaluator) */
+	TIMEOUT_MS: 20 * 60 * 1000,
+	DATASET_NAME: 'notion-pairwise-workflows',
+	FEATURE_FLAGS: {
+		templateExamples: false,
+	},
+} as const;

packages/@n8n/ai-workflow-builder.ee/evaluations/support/environment.ts

@@ -0,0 +1,337 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
+import { MemorySaver } from '@langchain/langgraph';
+import fs from 'fs';
+import { Client } from 'langsmith/client';
+import type { INodeTypeDescription } from 'n8n-workflow';
+import path from 'path';
+
+import { DEFAULT_MODEL, getApiKeyEnvVar, MODEL_FACTORIES, type ModelId } from '@/llm-config';
+import type { BuilderFeatureFlags } from '@/workflow-builder-agent';
+import { WorkflowBuilderAgent } from '@/workflow-builder-agent';
+
+import { loadNodesFromFile } from './load-nodes';
+import type { EvalLogger } from '../harness/logger';
+import {
+	createTraceFilters,
+	isMinimalTracingEnabled,
+	type TraceFilters,
+} from '../langsmith/trace-filters';
+
+/** Maximum memory for trace queue (3GB) */
+const MAX_INGEST_MEMORY_BYTES = 3 * 1024 * 1024 * 1024;
+
+// ============================================================================
+// Stage Models Configuration
+// ============================================================================
+
+/**
+ * Configuration for per-stage model selection.
+ * All fields except 'default' are optional - unspecified stages use the default model.
+ */
+export interface StageModels {
+	/** Default model for all stages */
+	default: ModelId;
+	/** Model for supervisor stage (routing decisions) */
+	supervisor?: ModelId;
+	/** Model for responder stage (final user responses) */
+	responder?: ModelId;
+	/** Model for discovery stage (node discovery) */
+	discovery?: ModelId;
+	/** Model for builder stage (workflow structure and configuration) */
+	builder?: ModelId;
+	/** Model for parameter updater (within builder) */
+	parameterUpdater?: ModelId;
+	/** Model for planner stage (plan mode) */
+	planner?: ModelId;
+	/** Model for LLM judge evaluation */
+	judge?: ModelId;
+}
+
+/**
+ * Resolved LLM instances for each stage.
+ * All fields are populated (using default model as fallback).
+ */
+export interface ResolvedStageLLMs {
+	default: BaseChatModel;
+	supervisor: BaseChatModel;
+	responder: BaseChatModel;
+	discovery: BaseChatModel;
+	builder: BaseChatModel;
+	parameterUpdater: BaseChatModel;
+	planner: BaseChatModel;
+	judge: BaseChatModel;
+}
+
+export interface TestEnvironment {
+	parsedNodeTypes: INodeTypeDescription[];
+	/** Resolved LLM instances for each stage */
+	llms: ResolvedStageLLMs;
+	tracer?: LangChainTracer;
+	lsClient?: Client;
+	/** Trace filtering utilities (only present when minimal tracing is enabled) */
+	traceFilters?: TraceFilters;
+	/** Directories containing generated node definition files */
+	nodeDefinitionDirs: string[];
+}
+
+/**
+ * Sets up an LLM with proper configuration
+ * @param modelId - Model identifier (defaults to DEFAULT_MODEL)
+ * @returns Configured LLM instance
+ * @throws Error if the required API key environment variable is not set
+ */
+export async function setupLLM(modelId: ModelId = DEFAULT_MODEL): Promise<BaseChatModel> {
+	const envVar = getApiKeyEnvVar(modelId);
+	const apiKey = process.env[envVar];
+	if (!apiKey) {
+		throw new Error(`${envVar} environment variable is required for model ${modelId}`);
+	}
+	const factory = MODEL_FACTORIES[modelId];
+	return await factory({ apiKey });
+}
+
+/**
+ * Resolves all stage models to LLM instances.
+ * Unspecified stages fall back to the default model.
+ * @param stageModels - Per-stage model configuration
+ * @returns Resolved LLM instances for each stage
+ */
+export async function resolveStageModels(stageModels: StageModels): Promise<ResolvedStageLLMs> {
+	const defaultLLM = await setupLLM(stageModels.default);
+
+	// For stages without specific model, use default
+	// For parameter updater, fall back to builder if not specified
+	const builderLLM = stageModels.builder ? await setupLLM(stageModels.builder) : defaultLLM;
+
+	return {
+		default: defaultLLM,
+		supervisor: stageModels.supervisor ? await setupLLM(stageModels.supervisor) : defaultLLM,
+		responder: stageModels.responder ? await setupLLM(stageModels.responder) : defaultLLM,
+		discovery: stageModels.discovery ? await setupLLM(stageModels.discovery) : defaultLLM,
+		builder: builderLLM,
+		parameterUpdater: stageModels.parameterUpdater
+			? await setupLLM(stageModels.parameterUpdater)
+			: builderLLM,
+		planner: stageModels.planner ? await setupLLM(stageModels.planner) : defaultLLM,
+		judge: stageModels.judge ? await setupLLM(stageModels.judge) : defaultLLM,
+	};
+}
+
+/**
+ * Creates a LangChain tracer for monitoring agent execution
+ * @param projectName - Name of the LangSmith project
+ * @returns LangChainTracer instance or undefined if API key not provided
+ */
+export function createTracer(client: Client, projectName: string): LangChainTracer | undefined {
+	return new LangChainTracer({
+		client,
+		projectName,
+	});
+}
+
+/**
+ * Result of creating a LangSmith client with optional filtering.
+ */
+export interface LangsmithClientResult {
+	client: Client;
+	/** Trace filters (only present when minimal tracing is enabled) */
+	traceFilters?: TraceFilters;
+}
+
+/**
+ * Creates a Langsmith client if API key is available.
+ * By default, minimal tracing is enabled to reduce payload sizes and avoid 403 errors.
+ * Set LANGSMITH_MINIMAL_TRACING=false to disable filtering and get full traces.
+ * @param logger - Optional logger for trace filter output
+ * @returns LangSmith client with optional trace filters, or undefined if no API key
+ */
+export function createLangsmithClient(logger?: EvalLogger): LangsmithClientResult | undefined {
+	const apiKey = process.env.LANGSMITH_API_KEY;
+	if (!apiKey) {
+		return undefined;
+	}
+
+	const minimalTracing = isMinimalTracingEnabled();
+
+	if (!minimalTracing) {
+		return { client: new Client({ apiKey }) };
+	}
+
+	// Create closure-scoped filters for this client instance
+	const traceFilters = createTraceFilters(logger);
+
+	const client = new Client({
+		apiKey,
+		// Filter large fields from traces to avoid 403 payload errors
+		hideInputs: traceFilters.filterInputs,
+		hideOutputs: traceFilters.filterOutputs,
+		// Increase queue memory limit for high-concurrency evals
+		maxIngestMemoryBytes: MAX_INGEST_MEMORY_BYTES,
+	});
+
+	return { client, traceFilters };
+}
+
+/**
+ * Resolve built-in node definition directories from installed node packages.
+ * Mirrors `WorkflowBuilderService.resolveBuiltinNodeDefinitionDirs()` for use
+ * in the eval harness where the DI container is not available.
+ */
+export function resolveBuiltinNodeDefinitionDirs(): string[] {
+	// In a pnpm monorepo, n8n-nodes-base and n8n-nodes-langchain are not direct
+	// dependencies of ai-workflow-builder.ee, so bare require.resolve() fails.
+	// Resolve from packages/cli which has them as dependencies.
+	const repoRoot = findRepoRoot(__dirname);
+	const resolvePaths = repoRoot ? [path.join(repoRoot, 'packages', 'cli')] : undefined;
+
+	const dirs: string[] = [];
+	for (const packageId of ['n8n-nodes-base', '@n8n/n8n-nodes-langchain']) {
+		try {
+			const packageJsonPath = require.resolve(`${packageId}/package.json`, {
+				paths: resolvePaths,
+			});
+			const distDir = path.dirname(packageJsonPath);
+			const nodeDefsDir = path.join(distDir, 'dist', 'node-definitions');
+			if (fs.existsSync(nodeDefsDir)) {
+				dirs.push(nodeDefsDir);
+			}
+		} catch {
+			// Package not installed, skip
+		}
+	}
+	if (dirs.length === 0) {
+		console.error('[NODE-DEFS] No node definition dirs resolved — get_node_types will fail');
+	}
+	return dirs;
+}
+
+/** Walk up from startDir to find the monorepo root (contains pnpm-workspace.yaml). */
+export function findRepoRoot(startDir: string): string | undefined {
+	let dir = startDir;
+	while (dir !== path.dirname(dir)) {
+		if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) {
+			return dir;
+		}
+		dir = path.dirname(dir);
+	}
+	return undefined;
+}
+
+/**
+ * Resolve the path to packages/nodes-base/nodes/ for __schema__ resolution.
+ * Returns undefined if the path doesn't exist (e.g. running outside the monorepo).
+ */
+export function resolveNodesBasePath(): string | undefined {
+	const repoRoot = findRepoRoot(__dirname);
+	if (!repoRoot) return undefined;
+	const p = path.join(repoRoot, 'packages', 'nodes-base', 'nodes');
+	return fs.existsSync(p) ? p : undefined;
+}
+
+/**
+ * Sets up the test environment with LLM, nodes, and tracing
+ * @param stageModels - Per-stage model configuration (optional, uses default model if not provided)
+ * @param logger - Optional logger for trace filter output
+ * @returns Test environment configuration
+ */
+export async function setupTestEnvironment(
+	stageModels?: StageModels,
+	logger?: EvalLogger,
+): Promise<TestEnvironment> {
+	const parsedNodeTypes = loadNodesFromFile();
+
+	// Use provided stage models or default configuration
+	const models: StageModels = stageModels ?? { default: DEFAULT_MODEL };
+	const llms = await resolveStageModels(models);
+
+	const lsClientResult = createLangsmithClient(logger);
+
+	const lsClient = lsClientResult?.client;
+	const traceFilters = lsClientResult?.traceFilters;
+	const tracer = lsClient ? createTracer(lsClient, 'workflow-builder-evaluation') : undefined;
+
+	return {
+		parsedNodeTypes,
+		llms,
+		tracer,
+		lsClient,
+		traceFilters,
+		nodeDefinitionDirs: resolveBuiltinNodeDefinitionDirs(),
+	};
+}
+
+export interface CreateAgentOptions {
+	parsedNodeTypes: INodeTypeDescription[];
+	/** Per-stage LLMs resolved from model configuration */
+	llms: ResolvedStageLLMs;
+	tracer?: LangChainTracer;
+	featureFlags?: BuilderFeatureFlags;
+	experimentName?: string;
+}
+
+/**
+ * Creates a new WorkflowBuilderAgent instance
+ * @param options - Agent configuration options
+ * @returns Configured WorkflowBuilderAgent
+ */
+export function createAgent(options: CreateAgentOptions): WorkflowBuilderAgent {
+	const { parsedNodeTypes, llms, tracer, featureFlags, experimentName } = options;
+
+	return new WorkflowBuilderAgent({
+		parsedNodeTypes,
+		stageLLMs: {
+			supervisor: llms.supervisor,
+			responder: llms.responder,
+			discovery: llms.discovery,
+			builder: llms.builder,
+			parameterUpdater: llms.parameterUpdater,
+			planner: llms.planner,
+		},
+		checkpointer: new MemorySaver(),
+		tracer,
+		featureFlags,
+		runMetadata: {
+			featureFlags: featureFlags ?? {},
+			experimentName,
+		},
+	});
+}
+
+/**
+ * Get concurrency limit from environment
+ * @returns Concurrency limit (defaults to 5)
+ */
+export function getConcurrencyLimit(): number {
+	const envConcurrency = process.env.EVALUATION_CONCURRENCY;
+	if (envConcurrency) {
+		const parsed = parseInt(envConcurrency, 10);
+		if (!isNaN(parsed) && parsed > 0) {
+			return parsed;
+		}
+	}
+	return 5;
+}
+
+/**
+ * Check if test cases should be generated
+ * @returns True if test cases should be generated
+ */
+export function shouldGenerateTestCases(): boolean {
+	return process.env.GENERATE_TEST_CASES === 'true';
+}
+
+/**
+ * How many test cases to generate based on environment variable
+ * @returns Number of test cases to generate (defaults to 10)
+ */
+export function howManyTestCasesToGenerate(): number {
+	const envCount = process.env.GENERATE_TEST_CASES_COUNT;
+	if (envCount) {
+		const parsed = parseInt(envCount, 10);
+		if (!isNaN(parsed) && parsed > 0) {
+			return parsed;
+		}
+	}
+	return 10; // Default to 10 if not specified
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/load-nodes.test.ts

@@ -0,0 +1,173 @@
+import { readFileSync, existsSync } from 'fs';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+// We need to mock the fs module before importing the module under test
+jest.mock('fs', () => ({
+	readFileSync: jest.fn(),
+	existsSync: jest.fn(),
+}));
+
+const mockedReadFileSync = readFileSync as jest.MockedFunction<typeof readFileSync>;
+const mockedExistsSync = existsSync as jest.MockedFunction<typeof existsSync>;
+
+// Import after mocking
+import { loadNodesFromFile } from './load-nodes';
+
+describe('loadNodesFromFile', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+		// Default: legacy path exists
+		mockedExistsSync.mockImplementation((path) => {
+			return String(path).endsWith('nodes.json');
+		});
+	});
+
+	describe('version handling', () => {
+		it('should keep all version entries for a node type so older versions can be validated', () => {
+			// This is the real-world scenario: removeDuplicates has two entries
+			// - Entry 1: version [1, 1.1], defaultVersion 2
+			// - Entry 2: version [2], defaultVersion 2
+			// A workflow using typeVersion 1.1 should still be able to validate
+			const nodesData: Array<Partial<INodeTypeDescription>> = [
+				{
+					name: 'n8n-nodes-base.removeDuplicates',
+					displayName: 'Remove Duplicates',
+					version: [1, 1.1],
+					defaultVersion: 2,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+				{
+					name: 'n8n-nodes-base.removeDuplicates',
+					displayName: 'Remove Duplicates',
+					version: [2],
+					defaultVersion: 2,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+			];
+
+			mockedReadFileSync.mockReturnValue(JSON.stringify(nodesData));
+
+			const result = loadNodesFromFile();
+
+			// The result should contain entries that cover ALL versions (1, 1.1, and 2)
+			// so that workflows using any of these versions can be validated
+			const removeDuplicatesEntries = result.filter(
+				(n) => n.name === 'n8n-nodes-base.removeDuplicates',
+			);
+
+			// We need at least the versions to be available for lookup
+			// Either keep both entries, or merge the versions
+			const allVersions = removeDuplicatesEntries.flatMap((entry) =>
+				Array.isArray(entry.version) ? entry.version : [entry.version],
+			);
+
+			expect(allVersions).toContain(1);
+			expect(allVersions).toContain(1.1);
+			expect(allVersions).toContain(2);
+		});
+
+		it('should handle single-version nodes correctly', () => {
+			const nodesData: Array<Partial<INodeTypeDescription>> = [
+				{
+					name: 'n8n-nodes-base.code',
+					displayName: 'Code',
+					version: 1,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+			];
+
+			mockedReadFileSync.mockReturnValue(JSON.stringify(nodesData));
+
+			const result = loadNodesFromFile();
+			const codeNode = result.find((n) => n.name === 'n8n-nodes-base.code');
+
+			expect(codeNode).toBeDefined();
+			expect(codeNode?.version).toBe(1);
+		});
+
+		it('should handle nodes with version array but no defaultVersion', () => {
+			const nodesData: Array<Partial<INodeTypeDescription>> = [
+				{
+					name: 'n8n-nodes-base.httpRequest',
+					displayName: 'HTTP Request',
+					version: [1, 2, 3],
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+			];
+
+			mockedReadFileSync.mockReturnValue(JSON.stringify(nodesData));
+
+			const result = loadNodesFromFile();
+			const httpNode = result.find((n) => n.name === 'n8n-nodes-base.httpRequest');
+
+			expect(httpNode).toBeDefined();
+			// All versions should be available
+			const versions = Array.isArray(httpNode?.version) ? httpNode.version : [httpNode?.version];
+			expect(versions).toContain(1);
+			expect(versions).toContain(2);
+			expect(versions).toContain(3);
+		});
+	});
+
+	describe('filtering', () => {
+		it('should filter out ignored node types', () => {
+			const nodesData: Array<Partial<INodeTypeDescription>> = [
+				{
+					name: '@n8n/n8n-nodes-langchain.toolVectorStore',
+					displayName: 'Vector Store Tool',
+					version: 1,
+					inputs: [],
+					outputs: ['ai_tool'],
+				},
+				{
+					name: 'n8n-nodes-base.code',
+					displayName: 'Code',
+					version: 1,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+			];
+
+			mockedReadFileSync.mockReturnValue(JSON.stringify(nodesData));
+
+			const result = loadNodesFromFile();
+
+			expect(
+				result.find((n) => n.name === '@n8n/n8n-nodes-langchain.toolVectorStore'),
+			).toBeUndefined();
+			expect(result.find((n) => n.name === 'n8n-nodes-base.code')).toBeDefined();
+		});
+
+		it('should filter out hidden nodes except dataTable', () => {
+			const nodesData: Array<Partial<INodeTypeDescription>> = [
+				{
+					name: 'n8n-nodes-base.hiddenNode',
+					displayName: 'Hidden Node',
+					version: 1,
+					hidden: true,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+				{
+					name: 'n8n-nodes-base.dataTable',
+					displayName: 'Data Table',
+					version: 1,
+					hidden: true,
+					inputs: ['main'],
+					outputs: ['main'],
+				},
+			];
+
+			mockedReadFileSync.mockReturnValue(JSON.stringify(nodesData));
+
+			const result = loadNodesFromFile();
+
+			expect(result.find((n) => n.name === 'n8n-nodes-base.hiddenNode')).toBeUndefined();
+			expect(result.find((n) => n.name === 'n8n-nodes-base.dataTable')).toBeDefined();
+		});
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/support/load-nodes.ts

@@ -0,0 +1,80 @@
+import { readFileSync, existsSync } from 'fs';
+import { jsonParse, type INodeTypeDescription } from 'n8n-workflow';
+import { join } from 'path';
+
+interface NodeWithVersion extends INodeTypeDescription {
+	version: number | number[];
+	defaultVersion?: number;
+}
+
+// These types are ignored because they tend to cause issues when generating workflows
+// Same as in ai-workflow-builder-agent.service.ts
+const IGNORED_TYPES = new Set([
+	'@n8n/n8n-nodes-langchain.toolVectorStore',
+	'@n8n/n8n-nodes-langchain.documentGithubLoader',
+	'@n8n/n8n-nodes-langchain.code',
+]);
+
+// Parse disabled nodes from environment variable (comma-separated)
+function getDisabledNodes(): Set<string> {
+	const disabledNodesEnv = process.env.N8N_EVALS_DISABLED_NODES ?? '';
+	return new Set(
+		disabledNodesEnv
+			.split(',')
+			.map((s) => s.trim())
+			.filter(Boolean),
+	);
+}
+
+/**
+ * Filter node types similar to production service:
+ * - Remove ignored types (hardcoded)
+ * - Remove disabled types (from env var)
+ * - Remove hidden nodes (except DataTable)
+ * - Merge tool nodes with their non-tool counterparts
+ */
+function filterNodeTypes(
+	nodeTypes: INodeTypeDescription[],
+	disabledNodes: Set<string>,
+): INodeTypeDescription[] {
+	const visibleNodeTypes = nodeTypes.filter(
+		(nodeType) =>
+			!IGNORED_TYPES.has(nodeType.name) &&
+			!disabledNodes.has(nodeType.name) &&
+			// Filter out hidden nodes, except for the Data Table node which has custom hiding logic
+			(nodeType.hidden !== true || nodeType.name === 'n8n-nodes-base.dataTable'),
+	);
+
+	return visibleNodeTypes.map((nodeType) => {
+		// If the node type is a tool, merge it with the corresponding non-tool node type
+		const isTool = nodeType.name.endsWith('Tool');
+		if (!isTool) return nodeType;
+
+		const nonToolNode = nodeTypes.find((nt) => nt.name === nodeType.name.replace('Tool', ''));
+		if (!nonToolNode) return nodeType;
+
+		return { ...nonToolNode, ...nodeType };
+	});
+}
+
+export function loadNodesFromFile(): INodeTypeDescription[] {
+	const preferredPath = join(__dirname, '..', '.data', 'nodes.json');
+	const legacyPath = join(__dirname, '..', 'nodes.json');
+	const nodesPath = existsSync(preferredPath) ? preferredPath : legacyPath;
+
+	if (!existsSync(nodesPath)) {
+		throw new Error(
+			`nodes.json not found at ${nodesPath}. ` +
+				'Run n8n and export node definitions to evaluations/.data/nodes.json',
+		);
+	}
+
+	const nodesData = readFileSync(nodesPath, 'utf-8');
+	const allNodes = jsonParse<NodeWithVersion[]>(nodesData);
+
+	// Keep all version entries instead of selecting only the latest version.
+	// Workflows may use older node versions (e.g., removeDuplicates v1.1), and
+	// discarding those entries causes "Node type not found" validation errors.
+	const disabledNodes = getDisabledNodes();
+	return filterNodeTypes(allNodes, disabledNodes);
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts

@@ -0,0 +1,491 @@
+import type { INode, INodeTypeDescription } from 'n8n-workflow';
+
+import {
+	identifyPinDataNodes,
+	buildSchemaContexts,
+	workflowToMermaid,
+	generateEvalPinData,
+} from './pin-data-generator';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeNode(overrides: Partial<INode> & { name: string; type: string }): INode {
+	return {
+		name: overrides.name,
+		type: overrides.type,
+		typeVersion: overrides.typeVersion ?? 1,
+		position: overrides.position ?? [0, 0],
+		parameters: overrides.parameters ?? {},
+		id: overrides.id ?? overrides.name,
+		...(overrides.disabled !== undefined ? { disabled: overrides.disabled } : {}),
+	} as INode;
+}
+
+function makeNodeType(
+	name: string,
+	opts?: { credentials?: Array<{ name: string }> },
+): INodeTypeDescription {
+	return {
+		displayName: name,
+		name,
+		group: ['transform'],
+		version: 1,
+		description: '',
+		defaults: { name },
+		inputs: ['main'],
+		outputs: ['main'],
+		properties: [],
+		...(opts?.credentials ? { credentials: opts.credentials } : {}),
+	} as unknown as INodeTypeDescription;
+}
+
+// ---------------------------------------------------------------------------
+// identifyPinDataNodes
+// ---------------------------------------------------------------------------
+
+describe('identifyPinDataNodes', () => {
+	const nodeTypes: INodeTypeDescription[] = [
+		makeNodeType('n8n-nodes-base.slack', { credentials: [{ name: 'slackApi' }] }),
+		makeNodeType('n8n-nodes-base.gmail', { credentials: [{ name: 'gmailOAuth2' }] }),
+		makeNodeType('n8n-nodes-base.set'),
+		makeNodeType('n8n-nodes-base.if'),
+		makeNodeType('n8n-nodes-base.httpRequest'),
+		makeNodeType('n8n-nodes-base.webhook'),
+	];
+
+	it('should include service nodes with credentials', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(1);
+		expect(result[0].name).toBe('Slack');
+	});
+
+	it('should exclude utility nodes that can run without infrastructure', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Set', type: 'n8n-nodes-base.set' }),
+				makeNode({ name: 'If', type: 'n8n-nodes-base.if' }),
+				makeNode({ name: 'Merge', type: 'n8n-nodes-base.merge' }),
+			],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(0);
+	});
+
+	it('should include Code and ExecuteCommand nodes (require task runner)', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Code', type: 'n8n-nodes-base.code' }),
+				makeNode({ name: 'Exec', type: 'n8n-nodes-base.executeCommand' }),
+			],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(2);
+		expect(result.map((n) => n.name)).toEqual(['Code', 'Exec']);
+	});
+
+	it('should include DataTable nodes', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Table', type: 'n8n-nodes-base.dataTable' })],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(1);
+		expect(result[0].name).toBe('Table');
+	});
+
+	it('should include root AI nodes targeted by ai_* connections', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Chat Trigger', type: '@n8n/n8n-nodes-langchain.chatTrigger' }),
+				makeNode({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' }),
+				makeNode({ name: 'OpenAI', type: '@n8n/n8n-nodes-langchain.lmOpenAi' }),
+				makeNode({ name: 'Tool', type: '@n8n/n8n-nodes-langchain.toolCalculator' }),
+			],
+			connections: {
+				'Chat Trigger': {
+					main: [[{ node: 'Agent', type: 'main', index: 0 }]],
+				},
+				OpenAI: {
+					ai_languageModel: [[{ node: 'Agent', type: 'ai_languageModel', index: 0 }]],
+				},
+				Tool: {
+					ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]],
+				},
+			},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		const names = result.map((n) => n.name);
+		// Agent is a root AI node (target of ai_* connections) → pinned
+		expect(names).toContain('Agent');
+		// Sub-nodes (OpenAI, Tool) are not pinned — they won't execute
+		// because the root is pinned
+		expect(names).not.toContain('OpenAI');
+		expect(names).not.toContain('Tool');
+	});
+
+	it('should not treat nodes targeted only by main connections as AI roots', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Trigger', type: 'n8n-nodes-base.manualTrigger' }),
+				makeNode({ name: 'Set', type: 'n8n-nodes-base.set' }),
+			],
+			connections: {
+				Trigger: {
+					main: [[{ node: 'Set', type: 'main', index: 0 }]],
+				},
+			},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(0);
+	});
+
+	it('should include HTTP Request and Webhook nodes even without credentials', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'HTTP', type: 'n8n-nodes-base.httpRequest' }),
+				makeNode({ name: 'Hook', type: 'n8n-nodes-base.webhook' }),
+			],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should skip disabled nodes', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack', disabled: true })],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(0);
+	});
+
+	it('should handle mixed utility and service nodes', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Trigger', type: 'n8n-nodes-base.manualTrigger' }),
+				makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' }),
+				makeNode({ name: 'Set', type: 'n8n-nodes-base.set' }),
+				makeNode({ name: 'Gmail', type: 'n8n-nodes-base.gmail' }),
+			],
+			connections: {},
+		};
+
+		const result = identifyPinDataNodes(workflow, nodeTypes);
+		expect(result).toHaveLength(2);
+		expect(result.map((n) => n.name)).toEqual(['Slack', 'Gmail']);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// buildSchemaContexts
+// ---------------------------------------------------------------------------
+
+describe('buildSchemaContexts', () => {
+	it('should extract resource and operation from node parameters', () => {
+		const nodes: INode[] = [
+			makeNode({
+				name: 'Slack',
+				type: 'n8n-nodes-base.slack',
+				typeVersion: 2,
+				parameters: { resource: 'message', operation: 'send' },
+			}),
+		];
+
+		const contexts = buildSchemaContexts(nodes);
+		expect(contexts).toHaveLength(1);
+		expect(contexts[0]).toMatchObject({
+			nodeName: 'Slack',
+			nodeType: 'n8n-nodes-base.slack',
+			typeVersion: 2,
+			resource: 'message',
+			operation: 'send',
+		});
+	});
+
+	it('should handle nodes without resource/operation', () => {
+		const nodes: INode[] = [
+			makeNode({
+				name: 'HTTP',
+				type: 'n8n-nodes-base.httpRequest',
+				parameters: { url: 'https://example.com' },
+			}),
+		];
+
+		const contexts = buildSchemaContexts(nodes);
+		expect(contexts[0].resource).toBeUndefined();
+		expect(contexts[0].operation).toBeUndefined();
+	});
+
+	it('should not include schema when nodesBasePath is not provided', () => {
+		const nodes: INode[] = [
+			makeNode({
+				name: 'Slack',
+				type: 'n8n-nodes-base.slack',
+				parameters: { resource: 'message', operation: 'send' },
+			}),
+		];
+
+		const contexts = buildSchemaContexts(nodes);
+		expect(contexts[0].schema).toBeUndefined();
+	});
+});
+
+// ---------------------------------------------------------------------------
+// workflowToMermaid
+// ---------------------------------------------------------------------------
+
+describe('workflowToMermaid', () => {
+	it('should generate mermaid flowchart for a simple workflow', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Trigger', type: 'n8n-nodes-base.manualTrigger' }),
+				makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack', typeVersion: 2 }),
+			],
+			connections: {
+				Trigger: {
+					main: [[{ node: 'Slack', type: 'main', index: 0 }]],
+				},
+			},
+		};
+
+		const mermaid = workflowToMermaid(workflow);
+		expect(mermaid).toContain('flowchart LR');
+		expect(mermaid).toContain('Trigger');
+		expect(mermaid).toContain('Slack');
+		expect(mermaid).toContain('-->');
+	});
+
+	it('should include resource and operation in labels', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({
+					name: 'Slack',
+					type: 'n8n-nodes-base.slack',
+					typeVersion: 2,
+					parameters: { resource: 'message', operation: 'send' },
+				}),
+			],
+			connections: {},
+		};
+
+		const mermaid = workflowToMermaid(workflow);
+		expect(mermaid).toContain('resource:message');
+		expect(mermaid).toContain('op:send');
+	});
+
+	it('should handle empty connections', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Trigger', type: 'n8n-nodes-base.manualTrigger' })],
+			connections: {},
+		};
+
+		const mermaid = workflowToMermaid(workflow);
+		expect(mermaid).toContain('flowchart LR');
+		expect(mermaid).not.toContain('-->');
+	});
+
+	it('should handle multi-output connections', () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'If', type: 'n8n-nodes-base.if' }),
+				makeNode({ name: 'True', type: 'n8n-nodes-base.set' }),
+				makeNode({ name: 'False', type: 'n8n-nodes-base.set' }),
+			],
+			connections: {
+				If: {
+					main: [
+						[{ node: 'True', type: 'main', index: 0 }],
+						[{ node: 'False', type: 'main', index: 0 }],
+					],
+				},
+			},
+		};
+
+		const mermaid = workflowToMermaid(workflow);
+		// Should have two connection lines
+		const arrowCount = (mermaid.match(/-->/g) ?? []).length;
+		expect(arrowCount).toBe(2);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// generateEvalPinData (with mocked LLM)
+// ---------------------------------------------------------------------------
+
+describe('generateEvalPinData', () => {
+	const nodeTypes: INodeTypeDescription[] = [
+		makeNodeType('n8n-nodes-base.slack', { credentials: [{ name: 'slackApi' }] }),
+		makeNodeType('n8n-nodes-base.linear', { credentials: [{ name: 'linearApi' }] }),
+		makeNodeType('n8n-nodes-base.set'),
+	];
+
+	function createMockLLM(responseContent: string) {
+		return {
+			invoke: jest.fn().mockResolvedValue({ content: responseContent }),
+		} as never;
+	}
+
+	it('should return pin data for service nodes', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Linear', type: 'n8n-nodes-base.linear' }),
+				makeNode({ name: 'Set', type: 'n8n-nodes-base.set' }),
+				makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' }),
+			],
+			connections: {},
+		};
+
+		const llmResponse = JSON.stringify({
+			Linear: [{ json: { id: 'issue-123', title: 'Test Issue' } }],
+			Slack: [{ json: { ok: true, channel: 'C123' } }],
+		});
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM(llmResponse),
+			nodeTypes,
+		});
+
+		expect(Object.keys(result)).toEqual(['Linear', 'Slack']);
+		expect(result.Linear).toHaveLength(1);
+		expect(result.Slack).toHaveLength(1);
+	});
+
+	it('should return empty object when no service nodes exist', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Set', type: 'n8n-nodes-base.set' })],
+			connections: {},
+		};
+
+		const llm = createMockLLM('{}');
+
+		const result = await generateEvalPinData(workflow, { llm, nodeTypes });
+
+		expect(result).toEqual({});
+		// LLM should not be called
+		expect((llm as unknown as { invoke: jest.Mock }).invoke).not.toHaveBeenCalled();
+	});
+
+	it('should handle markdown-fenced JSON response', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		const llmResponse = '```json\n{"Slack": [{"json": {"ok": true}}]}\n```';
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM(llmResponse),
+			nodeTypes,
+		});
+
+		expect(result.Slack).toHaveLength(1);
+	});
+
+	it('should wrap raw objects in { json: ... } format', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		// LLM returns items without the { json: ... } wrapper
+		const llmResponse = JSON.stringify({
+			Slack: [{ ok: true, channel: 'C123' }],
+		});
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM(llmResponse),
+			nodeTypes,
+		});
+
+		expect(result.Slack).toHaveLength(1);
+		expect(result.Slack[0]).toHaveProperty('json');
+	});
+
+	it('should return empty object on LLM failure', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		const llm = {
+			invoke: jest.fn().mockRejectedValue(new Error('LLM error')),
+		} as never;
+
+		const result = await generateEvalPinData(workflow, { llm, nodeTypes });
+		expect(result).toEqual({});
+	});
+
+	it('should return empty object on invalid JSON response', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM('not valid json at all'),
+			nodeTypes,
+		});
+
+		expect(result).toEqual({});
+	});
+
+	it('should skip nodes not present in LLM response', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [
+				makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' }),
+				makeNode({ name: 'Linear', type: 'n8n-nodes-base.linear' }),
+			],
+			connections: {},
+		};
+
+		// LLM only returns data for Slack, not Linear
+		const llmResponse = JSON.stringify({
+			Slack: [{ json: { ok: true } }],
+		});
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM(llmResponse),
+			nodeTypes,
+		});
+
+		expect(Object.keys(result)).toEqual(['Slack']);
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts

@@ -0,0 +1,510 @@
+/**
+ * LLM-based pin data generator for evaluations.
+ *
+ * Generates realistic mock output data for service nodes in a workflow
+ * via a single LLM call, ensuring cross-node data consistency.
+ * Works with both the code-based and multi-agent builder outputs.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import { existsSync, readFileSync, readdirSync } from 'fs';
+import {
+	jsonParse,
+	type IDataObject,
+	type INode,
+	type INodeTypeDescription,
+	type IPinData,
+} from 'n8n-workflow';
+import { join } from 'path';
+
+import type { SimpleWorkflow } from '../../src/types/workflow';
+import type { EvalLogger } from '../harness/logger';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface PinDataGeneratorOptions {
+	/** LLM to use for generating mock data */
+	llm: BaseChatModel;
+	/** Loaded node type descriptions (from env.parsedNodeTypes) */
+	nodeTypes: INodeTypeDescription[];
+	/** Path to packages/nodes-base/nodes/ for __schema__ resolution */
+	nodesBasePath?: string;
+	/** Logger for verbose output */
+	logger?: EvalLogger;
+}
+
+interface NodeSchemaContext {
+	nodeName: string;
+	nodeType: string;
+	typeVersion: number;
+	resource?: string;
+	operation?: string;
+	schema?: Record<string, unknown>;
+}
+
+// ---------------------------------------------------------------------------
+// Utility node deny-list
+// ---------------------------------------------------------------------------
+
+/**
+ * Nodes that define credentials in their type description but don't actually
+ * call external APIs, so they don't need pin data. All other non-service nodes
+ * are excluded naturally because they have no credentials defined.
+ */
+const NON_SERVICE_NODES_WITH_CREDENTIALS = new Set([
+	'n8n-nodes-base.wait', // optional webhook-resumption auth
+	'n8n-nodes-base.respondToWebhook', // optional JWT signing when responding
+]);
+
+// ---------------------------------------------------------------------------
+// Node identification
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a set of node names that are targets of AI-type connections
+ * (ai_languageModel, ai_tool, ai_memory, etc.). These are root AI nodes
+ * (e.g. Agent, Chain) whose sub-nodes can't be individually pinned.
+ * Pinning the root prevents sub-node execution entirely.
+ */
+function findAiRootNodeNames(workflow: SimpleWorkflow): Set<string> {
+	const roots = new Set<string>();
+	for (const nodeConns of Object.values(workflow.connections)) {
+		for (const [connType, outputs] of Object.entries(nodeConns)) {
+			if (!connType.startsWith('ai_')) continue;
+			if (!Array.isArray(outputs)) continue;
+			for (const group of outputs) {
+				if (!Array.isArray(group)) continue;
+				for (const conn of group) {
+					if (conn?.node) roots.add(conn.node);
+				}
+			}
+		}
+	}
+	return roots;
+}
+
+/**
+ * Identify which nodes in a workflow need pin data.
+ * In eval context, we pin all service/API nodes since none have real credentials.
+ */
+export function identifyPinDataNodes(
+	workflow: SimpleWorkflow,
+	nodeTypes: INodeTypeDescription[],
+): INode[] {
+	const nodeTypeMap = new Map(nodeTypes.map((nt) => [nt.name, nt]));
+	const aiRootNodes = findAiRootNodeNames(workflow);
+
+	return workflow.nodes.filter((node) => {
+		// Skip disabled nodes
+		if (node.disabled) return false;
+
+		// Pin root AI nodes (Agent, Chain, etc.) — their sub-nodes (tools,
+		// memory, LLMs) can't run without real providers in the eval context.
+		if (aiRootNodes.has(node.name)) return true;
+
+		// Check if the node type definition has credentials (→ it's a service node)
+		const typeDesc = nodeTypeMap.get(node.type);
+		if (typeDesc?.credentials && typeDesc.credentials.length > 0) {
+			// Exclude nodes that define credentials for local/optional auth, not external APIs
+			return !NON_SERVICE_NODES_WITH_CREDENTIALS.has(node.type);
+		}
+
+		// Nodes that require infrastructure unavailable in the eval context:
+		// - HTTP/Webhook: optional credentials, may call external APIs
+		// - DataTable: requires the data-table module
+		// - Code/ExecuteCommand: require a task runner to execute
+		if (
+			node.type === 'n8n-nodes-base.httpRequest' ||
+			node.type === 'n8n-nodes-base.webhook' ||
+			node.type === 'n8n-nodes-base.dataTable' ||
+			node.type === 'n8n-nodes-base.code' ||
+			node.type === 'n8n-nodes-base.executeCommand'
+		) {
+			return true;
+		}
+
+		return false;
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Schema resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a map from node type name (e.g., "n8n-nodes-base.linear") to the
+ * directory containing its __schema__ folder by scanning .node.ts files.
+ * Cached per nodesBasePath.
+ */
+const schemaMapCache = new Map<string, Map<string, string>>();
+
+function buildSchemaMap(nodesBasePath: string): Map<string, string> {
+	const cached = schemaMapCache.get(nodesBasePath);
+	if (cached) return cached;
+
+	const result = new Map<string, string>();
+
+	function scanDir(dir: string) {
+		try {
+			for (const entry of readdirSync(dir, { withFileTypes: true })) {
+				if (!entry.isDirectory()) continue;
+
+				const entryPath = join(dir, entry.name);
+				const schemaDir = join(entryPath, '__schema__');
+
+				if (existsSync(schemaDir)) {
+					// Find .node.ts/.node.js files to extract the node type name
+					const nodeFiles = readdirSync(entryPath).filter(
+						(f) => f.endsWith('.node.ts') || f.endsWith('.node.js'),
+					);
+					for (const nodeFile of nodeFiles) {
+						try {
+							const content = readFileSync(join(entryPath, nodeFile), 'utf-8');
+							const nameMatch = content.match(/name:\s*['"]([^'"]+)['"]/);
+							if (nameMatch) {
+								result.set(`n8n-nodes-base.${nameMatch[1]}`, entryPath);
+							}
+						} catch {
+							// Skip files that can't be read
+						}
+					}
+				}
+
+				// Recurse into subdirectories (e.g., Aws/S3, Google/Gmail)
+				scanDir(entryPath);
+			}
+		} catch {
+			// Directory doesn't exist or can't be read
+		}
+	}
+
+	scanDir(nodesBasePath);
+	schemaMapCache.set(nodesBasePath, result);
+	return result;
+}
+
+/**
+ * Normalize a version number to a semver-like string for directory matching.
+ * 1 → "1.0.0", 1.1 → "1.1.0", 2 → "2.0.0"
+ */
+function normalizeVersion(version: number): string {
+	const str = String(version);
+	const parts = str.split('.');
+	while (parts.length < 3) parts.push('0');
+	return parts.join('.');
+}
+
+/**
+ * Resolve the __schema__ JSON Schema for a node's output, if available.
+ */
+export function resolveSchemaForNode(
+	nodeType: string,
+	typeVersion: number,
+	resource: string | undefined,
+	operation: string | undefined,
+	nodesBasePath: string,
+): Record<string, unknown> | undefined {
+	const schemaMap = buildSchemaMap(nodesBasePath);
+	const nodeDir = schemaMap.get(nodeType);
+	if (!nodeDir) return undefined;
+
+	const schemaBaseDir = join(nodeDir, '__schema__');
+	if (!existsSync(schemaBaseDir)) return undefined;
+
+	const versionStr = normalizeVersion(typeVersion);
+
+	// Try exact version first, then fall back to available versions
+	const versionDirs = [
+		`v${versionStr}`,
+		...readdirSync(schemaBaseDir)
+			.filter((d) => d.startsWith('v'))
+			.sort()
+			.reverse(),
+	];
+
+	for (const vDir of [...new Set(versionDirs)]) {
+		const versionPath = join(schemaBaseDir, vDir);
+		if (!existsSync(versionPath)) continue;
+
+		// Build path with resource and operation
+		const parts = [versionPath, resource, operation ? `${operation}.json` : undefined].filter(
+			Boolean,
+		) as string[];
+
+		const schemaFile = join(...parts);
+		if (existsSync(schemaFile)) {
+			try {
+				return JSON.parse(readFileSync(schemaFile, 'utf-8')) as Record<string, unknown>;
+			} catch {
+				return undefined;
+			}
+		}
+	}
+
+	return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Schema context building
+// ---------------------------------------------------------------------------
+
+/**
+ * Build schema context for all pin-data-eligible nodes.
+ */
+export function buildSchemaContexts(nodes: INode[], nodesBasePath?: string): NodeSchemaContext[] {
+	return nodes.map((node) => {
+		const params = node.parameters as Record<string, unknown> | undefined;
+		const resource = typeof params?.resource === 'string' ? params.resource : undefined;
+		const operation = typeof params?.operation === 'string' ? params.operation : undefined;
+
+		let schema: Record<string, unknown> | undefined;
+		if (nodesBasePath) {
+			schema = resolveSchemaForNode(
+				node.type,
+				node.typeVersion,
+				resource,
+				operation,
+				nodesBasePath,
+			);
+		}
+
+		return {
+			nodeName: node.name,
+			nodeType: node.type,
+			typeVersion: node.typeVersion,
+			resource,
+			operation,
+			schema,
+		};
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Mermaid diagram generation
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert a workflow's nodes and connections to a mermaid flowchart string.
+ * Includes node type, version, resource, and operation info in labels.
+ */
+export function workflowToMermaid(workflow: SimpleWorkflow): string {
+	const lines: string[] = ['flowchart LR'];
+
+	// Build a map for safe mermaid IDs
+	const nodeIdMap = new Map<string, string>();
+	workflow.nodes.forEach((node, i) => {
+		nodeIdMap.set(node.name, `n${i}`);
+	});
+
+	// Add node declarations with labels
+	for (const node of workflow.nodes) {
+		const id = nodeIdMap.get(node.name)!;
+		const params = node.parameters as Record<string, unknown> | undefined;
+		const resource = typeof params?.resource === 'string' ? params.resource : undefined;
+		const operation = typeof params?.operation === 'string' ? params.operation : undefined;
+
+		const shortType = node.type.split('.').pop() ?? node.type;
+		let label = `${node.name} (${shortType} v${node.typeVersion}`;
+		if (resource) label += `, resource:${resource}`;
+		if (operation) label += `, op:${operation}`;
+		label += ')';
+
+		lines.push(`  ${id}["${label}"]`);
+	}
+
+	// Add connections
+	const { connections } = workflow;
+	for (const [sourceName, nodeConns] of Object.entries(connections)) {
+		const sourceId = nodeIdMap.get(sourceName);
+		if (!sourceId) continue;
+
+		for (const [, outputConnections] of Object.entries(nodeConns)) {
+			if (!Array.isArray(outputConnections)) continue;
+
+			for (const outputGroup of outputConnections) {
+				if (!Array.isArray(outputGroup)) continue;
+
+				for (const conn of outputGroup) {
+					if (!conn?.node) continue;
+					const targetId = nodeIdMap.get(conn.node);
+					if (targetId) {
+						lines.push(`  ${sourceId} --> ${targetId}`);
+					}
+				}
+			}
+		}
+	}
+
+	return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// LLM prompt construction
+// ---------------------------------------------------------------------------
+
+const SYSTEM_PROMPT = `You are a test data generator for n8n workflow automation. Generate realistic mock API response data for service nodes in a workflow.
+
+RULES:
+1. Data must be consistent across nodes. If node A creates an entity with id "abc-123", downstream nodes referencing that entity must use "abc-123".
+2. Generate 1-2 items per node.
+3. When a JSON Schema is provided, follow its structure exactly.
+4. When no schema is provided, generate a realistic response based on the node type, resource, and operation.
+5. Use realistic but clearly fake values (e.g., "jane@example.com", "proj_abc123", "2024-01-15T10:30:00Z").
+6. Return ONLY a valid JSON object, no explanation or markdown fencing.`;
+
+function buildUserPrompt(workflow: SimpleWorkflow, contexts: NodeSchemaContext[]): string {
+	const mermaid = workflowToMermaid(workflow);
+
+	const sections: string[] = [
+		'Generate mock output data for service nodes in this workflow.',
+		'',
+		'## Workflow Graph',
+		'',
+		'```mermaid',
+		mermaid,
+		'```',
+		'',
+		'## Nodes Requiring Mock Data',
+	];
+
+	for (const ctx of contexts) {
+		sections.push('');
+		sections.push(`### ${ctx.nodeName} (${ctx.nodeType} v${ctx.typeVersion})`);
+		if (ctx.resource || ctx.operation) {
+			const parts: string[] = [];
+			if (ctx.resource) parts.push(`Resource: ${ctx.resource}`);
+			if (ctx.operation) parts.push(`Operation: ${ctx.operation}`);
+			sections.push(`- ${parts.join(' | ')}`);
+		}
+		if (ctx.schema) {
+			const schemaStr = JSON.stringify(ctx.schema, null, 2);
+			// Truncate very large schemas to keep prompt manageable
+			const truncated = schemaStr.length > 3000 ? schemaStr.slice(0, 3000) + '\n...' : schemaStr;
+			sections.push('- Output JSON Schema:');
+			sections.push('```json');
+			sections.push(truncated);
+			sections.push('```');
+		} else {
+			sections.push('(no schema available — generate based on API knowledge)');
+		}
+	}
+
+	sections.push('');
+	sections.push('## Expected Output Format');
+	sections.push('');
+	sections.push(
+		'Return a JSON object where each key is the exact node name and the value is an array of items, each wrapped in a "json" key:',
+	);
+	sections.push('');
+	sections.push('```json');
+	sections.push('{');
+	for (let i = 0; i < Math.min(contexts.length, 2); i++) {
+		const ctx = contexts[i];
+		const comma = i < Math.min(contexts.length, 2) - 1 ? ',' : '';
+		sections.push(`  "${ctx.nodeName}": [{ "json": { ... } }]${comma}`);
+	}
+	if (contexts.length > 2) {
+		sections.push('  ...');
+	}
+	sections.push('}');
+	sections.push('```');
+
+	return sections.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Response parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse the LLM response into IPinData format.
+ * Handles both `{ "json": {...} }` wrapped and unwrapped items.
+ */
+function parsePinDataResponse(responseText: string, expectedNodes: string[]): IPinData {
+	// Strip markdown code fences if present
+	let cleaned = responseText.trim();
+	if (cleaned.startsWith('```')) {
+		cleaned = cleaned.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
+	}
+
+	const parsed = jsonParse<Record<string, unknown>>(cleaned);
+	const pinData: IPinData = {};
+
+	for (const nodeName of expectedNodes) {
+		const nodeData = parsed[nodeName];
+		if (!Array.isArray(nodeData) || nodeData.length === 0) continue;
+
+		pinData[nodeName] = nodeData.map((item: unknown) => {
+			if (typeof item === 'object' && item !== null && 'json' in item) {
+				return item as { json: IDataObject };
+			}
+			// Wrap raw objects in { json: ... }
+			return { json: (item ?? {}) as IDataObject };
+		});
+	}
+
+	return pinData;
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate pin data for all service nodes in a workflow using an LLM.
+ * Produces consistent cross-node mock data in a single LLM call.
+ *
+ * @returns IPinData map (node name → execution data items). Returns {} on failure.
+ */
+export async function generateEvalPinData(
+	workflow: SimpleWorkflow,
+	options: PinDataGeneratorOptions,
+): Promise<IPinData> {
+	const { llm, nodeTypes, nodesBasePath, logger } = options;
+
+	// 1. Identify which nodes need pin data
+	const eligibleNodes = identifyPinDataNodes(workflow, nodeTypes);
+	if (eligibleNodes.length === 0) {
+		logger?.verbose('  Pin data: no service nodes found, skipping');
+		return {};
+	}
+
+	logger?.verbose(`  Pin data: generating for ${eligibleNodes.length} node(s)`);
+
+	// 2. Build schema contexts (with optional __schema__ enrichment)
+	const contexts = buildSchemaContexts(eligibleNodes, nodesBasePath);
+	const schemasFound = contexts.filter((c) => c.schema).length;
+	if (schemasFound > 0) {
+		logger?.verbose(`  Pin data: found __schema__ for ${schemasFound}/${contexts.length} node(s)`);
+	}
+
+	// 3. Build prompt and call LLM
+	const userPrompt = buildUserPrompt(workflow, contexts);
+	const expectedNodeNames = contexts.map((c) => c.nodeName);
+
+	try {
+		const response = await llm.invoke([
+			new SystemMessage(SYSTEM_PROMPT),
+			new HumanMessage(userPrompt),
+		]);
+
+		const responseText =
+			typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
+
+		const pinData = parsePinDataResponse(responseText, expectedNodeNames);
+
+		const generatedCount = Object.keys(pinData).length;
+		logger?.verbose(
+			`  Pin data: generated for ${generatedCount}/${expectedNodeNames.length} node(s)`,
+		);
+
+		return pinData;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		logger?.warn(`  Pin data generation failed: ${message}`);
+		return {};
+	}
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/report-generator.ts

@@ -0,0 +1,208 @@
+/**
+ * Markdown Report Generator
+ *
+ * Generates human-readable markdown reports from evaluation results.
+ */
+
+import { feedbackKey } from '../harness/feedback';
+import type { ExampleResult, RunSummary } from '../harness/harness-types';
+import {
+	groupByEvaluator,
+	selectScoringItems,
+	calculateFiniteAverage,
+} from '../harness/score-calculator';
+
+/**
+ * Violation severity levels.
+ */
+export type ViolationSeverity = 'critical' | 'major' | 'minor';
+
+/**
+ * Options for report generation.
+ */
+export interface ReportOptions {
+	/** Include detailed per-test results (default: false) */
+	includeDetails?: boolean;
+	/** Include violation breakdown (default: true) */
+	includeViolations?: boolean;
+}
+
+/**
+ * Metrics calculated from evaluation results.
+ */
+export interface ReportMetrics {
+	/** Average score per evaluator */
+	evaluatorAverages: Record<string, number>;
+	/** Count of violations by severity */
+	violationCounts: { critical: number; major: number; minor: number };
+}
+
+/** Maximum prompt length before truncation */
+const MAX_PROMPT_LENGTH = 80;
+
+/**
+ * Extract violation severity from a feedback comment.
+ *
+ * Looks for markers like [CRITICAL], [MAJOR], [MINOR] in the comment.
+ *
+ * @param comment - The feedback comment to parse
+ * @returns The violation severity or null if not found
+ */
+export function extractViolationSeverity(comment?: string): ViolationSeverity | null {
+	if (!comment) return null;
+
+	const lowerComment = comment.toLowerCase();
+
+	if (lowerComment.includes('[critical]')) return 'critical';
+	if (lowerComment.includes('[major]')) return 'major';
+	if (lowerComment.includes('[minor]')) return 'minor';
+
+	return null;
+}
+
+/**
+ * Format a number as a percentage string.
+ */
+function formatPercentage(value: number, decimals = 1): string {
+	if (!Number.isFinite(value)) return 'N/A';
+	return `${(value * 100).toFixed(decimals)}%`;
+}
+
+/**
+ * Truncate a string to a maximum length with ellipsis.
+ */
+function truncate(str: string, maxLength: number): string {
+	if (str.length <= maxLength) return str;
+	return str.slice(0, maxLength - 3) + '...';
+}
+
+/**
+ * Calculate report metrics from evaluation results.
+ *
+ * @param results - Array of example results
+ * @returns Calculated metrics including evaluator averages and violation counts
+ */
+export function calculateReportMetrics(results: ExampleResult[]): ReportMetrics {
+	const okResults = results.filter((r) => r.status !== 'error');
+
+	// Calculate evaluator averages (per-example, then average; avoids key-count skew)
+	const evaluatorScores: Record<string, number[]> = {};
+	for (const result of okResults) {
+		const grouped = groupByEvaluator(result.feedback);
+		for (const [evaluator, items] of Object.entries(grouped)) {
+			if (!evaluatorScores[evaluator]) evaluatorScores[evaluator] = [];
+			evaluatorScores[evaluator].push(calculateFiniteAverage(selectScoringItems(items)));
+		}
+	}
+
+	const evaluatorAverages: Record<string, number> = {};
+	for (const [evaluator, scores] of Object.entries(evaluatorScores)) {
+		evaluatorAverages[evaluator] = scores.reduce((sum, s) => sum + s, 0) / scores.length;
+	}
+
+	// Count violations by severity
+	const violationCounts = { critical: 0, major: 0, minor: 0 };
+	for (const result of okResults) {
+		for (const feedback of result.feedback) {
+			const severity = extractViolationSeverity(feedback.comment);
+			if (severity) {
+				violationCounts[severity]++;
+			}
+		}
+	}
+
+	return {
+		evaluatorAverages,
+		violationCounts,
+	};
+}
+
+/**
+ * Generate a markdown report from evaluation results.
+ *
+ * @param results - Array of example results
+ * @param summary - Run summary with totals
+ * @param options - Report generation options
+ * @returns Formatted markdown string
+ */
+export function generateMarkdownReport(
+	results: ExampleResult[],
+	summary: RunSummary,
+	options: ReportOptions = {},
+): string {
+	const { includeDetails = false, includeViolations = true } = options;
+
+	const metrics = calculateReportMetrics(results);
+	const passRate = summary.totalExamples > 0 ? summary.passed / summary.totalExamples : 0;
+
+	let report = `# AI Workflow Builder Evaluation Report
+
+## Summary
+- Total Tests: ${summary.totalExamples}
+- Passed: ${summary.passed} (${formatPercentage(passRate)})
+- Failed: ${summary.failed}
+- Errors: ${summary.errors}
+- Average Score: ${formatPercentage(summary.averageScore)}
+- Total Duration: ${(summary.totalDurationMs / 1000).toFixed(1)}s
+
+`;
+
+	// Evaluator Averages
+	if (Object.keys(metrics.evaluatorAverages).length > 0) {
+		report += `## Evaluator Averages
+`;
+		for (const [evaluator, avg] of Object.entries(metrics.evaluatorAverages)) {
+			report += `- ${evaluator}: ${formatPercentage(avg)}
+`;
+		}
+		report += '\n';
+	}
+
+	// Violations Summary
+	if (includeViolations) {
+		const { critical, major, minor } = metrics.violationCounts;
+		report += `## Violations Summary
+- Critical: ${critical}
+- Major: ${major}
+- Minor: ${minor}
+
+`;
+	}
+
+	// Detailed Results
+	if (includeDetails && results.length > 0) {
+		report += `## Detailed Results
+
+`;
+		for (const result of results) {
+			const promptPreview = truncate(result.prompt, MAX_PROMPT_LENGTH);
+			const resultScore = result.score;
+
+			report += `### Test ${result.index}: ${promptPreview}
+- **Status**: ${result.status}
+- **Score**: ${formatPercentage(resultScore)}
+- **Duration**: ${result.durationMs}ms
+`;
+
+			if (result.error) {
+				report += `- **Error**: ${result.error}
+`;
+			}
+
+			if (result.feedback.length > 0) {
+				report += `- **Feedback**:
+`;
+				for (const fb of result.feedback) {
+					const scoreStr = formatPercentage(fb.score);
+					const commentStr = fb.comment ? ` - ${fb.comment}` : '';
+					report += `  - [${feedbackKey(fb)}] ${scoreStr}${commentStr}
+`;
+				}
+			}
+
+			report += '\n';
+		}
+	}
+
+	return report;
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/test-case-generator.ts

@@ -0,0 +1,179 @@
+/**
+ * Test Case Generator
+ *
+ * Generates test cases for workflow evaluation using LLM with structured output.
+ * For default test cases, see fixtures/default-prompts.csv.
+ */
+
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import { z } from 'zod';
+
+/**
+ * Options for test case generation.
+ */
+export interface TestCaseGeneratorOptions {
+	/** Number of test cases to generate (default: 10) */
+	count?: number;
+	/** Focus areas for generation */
+	focus?: string;
+	/** Complexity distribution */
+	complexity?: 'balanced' | 'simple' | 'complex';
+}
+
+/**
+ * A test case generated by the LLM.
+ */
+export interface GeneratedTestCase {
+	id: string;
+	name: string;
+	summary: string;
+	prompt: string;
+}
+
+/**
+ * Test case generator interface.
+ */
+export interface TestCaseGenerator {
+	/** Generate test cases */
+	generate(): Promise<GeneratedTestCase[]>;
+}
+
+/**
+ * Zod schema for structured output.
+ */
+const generatedTestCasesSchema = z.object({
+	testCases: z.array(
+		z.object({
+			id: z.string().describe('Unique identifier (e.g., "test_001")'),
+			name: z.string().describe('Short descriptive title'),
+			summary: z.string().describe('Brief description of what the workflow does'),
+			prompt: z.string().describe('User-facing prompt for workflow generation'),
+		}),
+	),
+});
+
+/** Inferred type from the Zod schema */
+type GeneratedTestCasesOutput = z.infer<typeof generatedTestCasesSchema>;
+
+/** Parse and validate LLM output using the Zod schema */
+function parseTestCasesOutput(value: unknown): GeneratedTestCasesOutput {
+	const parsed = generatedTestCasesSchema.safeParse(value);
+	if (!parsed.success) {
+		throw new Error(`Invalid LLM output: ${parsed.error.message}`);
+	}
+	return parsed.data;
+}
+
+/**
+ * System prompt for test case generation.
+ */
+const systemPrompt = `You are an expert at generating diverse test cases for an n8n workflow builder AI. Create test cases that cover various real-world scenarios and complexity levels.
+
+## Test Case Requirements:
+
+1. **Simple Test Cases**: Single operation workflows
+   - API calls
+   - Data transformations
+   - File operations
+   - Basic integrations
+
+2. **Medium Test Cases**: Multi-step workflows with logic
+   - Conditional logic (IF nodes)
+   - Data filtering and transformation
+   - Multiple API integrations
+   - Error handling
+
+3. **Complex Test Cases**: Advanced workflows
+   - Parallel execution branches
+   - Complex error handling and retry logic
+   - Multiple integrations with data synchronization
+   - Webhooks and event-driven flows
+
+## Guidelines:
+- Create realistic business scenarios
+- Include specific requirements that can be evaluated
+- Vary the domains (e-commerce, HR, marketing, DevOps, etc.)
+- Include both common and edge-case scenarios
+- Make prompts clear and unambiguous
+- Specify expected node types when possible
+
+## Output Format:
+Each test case should have:
+- Unique ID (e.g., "test_001")
+- Descriptive name
+- Brief description
+- Clear prompt that a user would give`;
+
+/**
+ * Get default focus based on complexity option.
+ */
+function getFocus(options?: TestCaseGeneratorOptions): string {
+	const complexity = options?.complexity ?? 'balanced';
+
+	if (options?.focus) {
+		return options.focus;
+	}
+
+	switch (complexity) {
+		case 'simple':
+			return 'simple, single-operation workflows like basic API calls, data transformations, and file operations';
+		case 'complex':
+			return 'complex, multi-step workflows with parallel execution, error handling, and multiple integrations';
+		case 'balanced':
+		default:
+			return 'balanced mix of API integrations, data processing, and automation scenarios';
+	}
+}
+
+/**
+ * Build the human message content.
+ */
+function buildHumanMessage(count: number, focus: string): string {
+	return `Generate ${count} diverse test cases for workflow generation evaluation.
+
+Focus on:
+${focus}
+
+Ensure a good mix of complexity levels and use cases.`;
+}
+
+/**
+ * Create a test case generator that uses LLM to generate test cases.
+ *
+ * @param llm - Language model to use for generation
+ * @param options - Generation options
+ * @returns A test case generator
+ *
+ * @example
+ * ```typescript
+ * const generator = createTestCaseGenerator(llm, { count: 20 });
+ * const testCases = await generator.generate();
+ * ```
+ */
+export function createTestCaseGenerator(
+	llm: BaseChatModel,
+	options?: TestCaseGeneratorOptions,
+): TestCaseGenerator {
+	const count = options?.count ?? 10;
+	const focus = getFocus(options);
+
+	// Create LLM with structured output
+	const llmWithStructuredOutput = llm.withStructuredOutput(generatedTestCasesSchema);
+
+	return {
+		async generate(): Promise<GeneratedTestCase[]> {
+			const humanMessage = buildHumanMessage(count, focus);
+
+			const rawResult = await llmWithStructuredOutput.invoke([
+				new SystemMessage(systemPrompt),
+				new HumanMessage(humanMessage),
+			]);
+
+			// Validate and parse the LLM output using Zod
+			const result = parseTestCasesOutput(rawResult);
+
+			return result.testCases;
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/workflow-executor.test.ts

@@ -0,0 +1,184 @@
+import type { INode, INodeType, INodeTypes, IPinData } from 'n8n-workflow';
+
+import { findRepoRoot } from './environment';
+import { executeWorkflowWithPinData, findTriggerByGroup } from './workflow-executor';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+
+jest.mock('./environment', () => ({
+	findRepoRoot: jest.fn(() => '/mock/repo/root'),
+}));
+
+jest.mock('@n8n/di', () => ({
+	Container: {
+		set: jest.fn(),
+		get: jest.fn(() => ({})),
+	},
+}));
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeSimpleWorkflow(nodes?: INode[]): SimpleWorkflow {
+	return {
+		name: 'Test Workflow',
+		nodes: nodes ?? [
+			{
+				name: 'Trigger',
+				type: 'n8n-nodes-base.manualTrigger',
+				typeVersion: 1,
+				position: [0, 0] as [number, number],
+				parameters: {},
+				id: 'trigger-1',
+			} as INode,
+		],
+		connections: {},
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('executeWorkflowWithPinData', () => {
+	describe('error handling — no monorepo root', () => {
+		beforeEach(() => {
+			jest.mocked(findRepoRoot).mockReturnValue(undefined);
+		});
+
+		it('should return success=false when monorepo root cannot be found', async () => {
+			const workflow = makeSimpleWorkflow();
+			const pinData: IPinData = {};
+
+			const result = await executeWorkflowWithPinData(workflow, pinData);
+
+			expect(result.success).toBe(false);
+			expect(result.error).toContain('Cannot find monorepo root');
+			expect(result.executedNodes).toEqual([]);
+		});
+
+		it('should always return a non-negative durationMs', async () => {
+			const result = await executeWorkflowWithPinData(makeSimpleWorkflow(), {});
+
+			expect(typeof result.durationMs).toBe('number');
+			expect(result.durationMs).toBeGreaterThanOrEqual(0);
+		});
+	});
+});
+
+describe('ExecutionResult shape', () => {
+	it('should include success, durationMs, and executedNodes', () => {
+		const result = {
+			success: true,
+			durationMs: 100,
+			executedNodes: ['Trigger', 'Set'],
+		};
+
+		expect(result).toHaveProperty('success');
+		expect(result).toHaveProperty('durationMs');
+		expect(result).toHaveProperty('executedNodes');
+	});
+
+	it('should support optional error and errorNode fields', () => {
+		const result = {
+			success: false,
+			error: 'Something went wrong',
+			errorNode: 'HTTP Request',
+			durationMs: 50,
+			executedNodes: ['Trigger'],
+		};
+
+		expect(result.success).toBe(false);
+		expect(result.error).toBe('Something went wrong');
+		expect(result.errorNode).toBe('HTTP Request');
+	});
+});
+
+// ---------------------------------------------------------------------------
+// findTriggerByGroup
+// ---------------------------------------------------------------------------
+
+describe('findTriggerByGroup', () => {
+	function makeNodeTypes(types: Record<string, { group: string[] }>): INodeTypes {
+		return {
+			getByName: jest.fn(),
+			getByNameAndVersion(type: string): INodeType {
+				const entry = types[type] ?? { group: ['transform'] };
+				return { description: { group: entry.group } } as unknown as INodeType;
+			},
+			getKnownTypes: jest.fn(),
+		};
+	}
+
+	it('should find a chat trigger node by its group', () => {
+		const nodes: INode[] = [
+			{
+				name: 'Chat Trigger',
+				type: '@n8n/n8n-nodes-langchain.chatTrigger',
+				typeVersion: 1,
+				position: [0, 0] as [number, number],
+				parameters: {},
+				id: '1',
+			} as INode,
+			{
+				name: 'Agent',
+				type: '@n8n/n8n-nodes-langchain.agent',
+				typeVersion: 1,
+				position: [200, 0] as [number, number],
+				parameters: {},
+				id: '2',
+			} as INode,
+		];
+
+		const nodeTypes = makeNodeTypes({
+			'@n8n/n8n-nodes-langchain.chatTrigger': { group: ['trigger'] },
+			'@n8n/n8n-nodes-langchain.agent': { group: ['transform'] },
+		});
+
+		const result = findTriggerByGroup(nodes, nodeTypes);
+		expect(result?.name).toBe('Chat Trigger');
+	});
+
+	it('should skip disabled trigger nodes', () => {
+		const nodes: INode[] = [
+			{
+				name: 'Disabled Trigger',
+				type: '@n8n/n8n-nodes-langchain.chatTrigger',
+				typeVersion: 1,
+				position: [0, 0] as [number, number],
+				parameters: {},
+				id: '1',
+				disabled: true,
+			} as INode,
+		];
+
+		const nodeTypes = makeNodeTypes({
+			'@n8n/n8n-nodes-langchain.chatTrigger': { group: ['trigger'] },
+		});
+
+		const result = findTriggerByGroup(nodes, nodeTypes);
+		expect(result).toBeUndefined();
+	});
+
+	it('should return undefined when no trigger nodes exist', () => {
+		const nodes: INode[] = [
+			{
+				name: 'Set',
+				type: 'n8n-nodes-base.set',
+				typeVersion: 1,
+				position: [0, 0] as [number, number],
+				parameters: {},
+				id: '1',
+			} as INode,
+		];
+
+		const nodeTypes = makeNodeTypes({});
+
+		const result = findTriggerByGroup(nodes, nodeTypes);
+		expect(result).toBeUndefined();
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/support/workflow-executor.ts

@@ -0,0 +1,377 @@
+/**
+ * Lightweight workflow executor for evaluations.
+ *
+ * Executes a SimpleWorkflow with pin data using the n8n execution engine.
+ * Service/API nodes use pin data (skipping real API calls). Utility nodes
+ * (Set, If, Code, etc.) actually execute using their compiled dist
+ * implementations, validating the workflow structure end-to-end.
+ *
+ * Node implementations are loaded directly from the compiled dist/ files in
+ * nodes-base and nodes-langchain, bypassing the DI-based loader infrastructure
+ * and its TypeScript path-alias issues.
+ *
+ * Uses path-based resolution to import WorkflowExecute and
+ * ExecutionLifecycleHooks from @n8n/core dist without adding it as a package
+ * dependency, following the NodeTestHarness pattern.
+ */
+
+import type {
+	IExecuteFunctions,
+	INode,
+	IPinData,
+	IRun,
+	INodeType,
+	INodeTypeDescription,
+	INodeTypes,
+	IVersionedNodeType,
+	IWorkflowExecuteAdditionalData,
+} from 'n8n-workflow';
+import { createDeferredPromise, createRunExecutionData, NodeHelpers, Workflow } from 'n8n-workflow';
+import path from 'path';
+
+import { findRepoRoot } from './environment';
+import type { SimpleWorkflow } from '../../src/types/workflow';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ExecutionResult {
+	success: boolean;
+	error?: string;
+	/** Node that caused the error, if identifiable */
+	errorNode?: string;
+	/** Total execution duration in ms */
+	durationMs: number;
+	/** Which nodes actually executed (in order) */
+	executedNodes: string[];
+}
+
+// ---------------------------------------------------------------------------
+// DistNodeTypes
+//
+// Implements INodeTypes by loading compiled node classes directly from the
+// dist/ directories of nodes-base and nodes-langchain packages.
+//
+// - Reads dist/known/nodes.json for the lazy-load index (no actual loading yet)
+// - On getByName/getByNameAndVersion: require()s the compiled .js file and
+//   instantiates the class on demand
+// - Unknown types fall back to a passthrough; service nodes never reach
+//   execute() because the engine short-circuits on pin data
+//
+// No DI container, no TypeScript source files, no path alias issues.
+// ---------------------------------------------------------------------------
+
+interface KnownNodeEntry {
+	className: string;
+	sourcePath: string;
+	packageDir: string;
+}
+
+class DistNodeTypes implements INodeTypes {
+	private readonly knownNodes: Map<string, KnownNodeEntry>;
+
+	private readonly cache = new Map<string, INodeType | IVersionedNodeType>();
+
+	constructor(
+		packages: Array<{
+			packagePrefix: string;
+			packageDir: string;
+			knownNodes: Record<string, { className: string; sourcePath: string }>;
+		}>,
+	) {
+		this.knownNodes = new Map();
+		for (const { packagePrefix, packageDir, knownNodes } of packages) {
+			for (const [shortName, info] of Object.entries(knownNodes)) {
+				// Workflow nodes use the full type name: "n8n-nodes-base.set"
+				// The known.nodes JSON uses just the short name: "set"
+				this.knownNodes.set(`${packagePrefix}.${shortName}`, { ...info, packageDir });
+			}
+		}
+	}
+
+	getByName(type: string): INodeType | IVersionedNodeType {
+		return this.loadNode(type);
+	}
+
+	getByNameAndVersion(type: string, version?: number): INodeType {
+		const loaded = this.loadNode(type);
+		if ('nodeVersions' in loaded) {
+			return NodeHelpers.getVersionedNodeType(loaded, version);
+		}
+		return loaded;
+	}
+
+	getKnownTypes() {
+		return {};
+	}
+
+	private loadNode(type: string): INodeType | IVersionedNodeType {
+		const cached = this.cache.get(type);
+		if (cached) return cached;
+
+		const known = this.knownNodes.get(type);
+		if (!known) {
+			// Unknown type — return a passthrough so the engine doesn't crash.
+			return {
+				description: {
+					displayName: type,
+					name: type,
+					group: ['transform'],
+					version: 1,
+					description: '',
+					defaults: { name: type },
+					inputs: ['main'],
+					outputs: ['main'],
+					properties: [],
+				} as unknown as INodeTypeDescription,
+				async execute(this: IExecuteFunctions) {
+					return [this.getInputData()];
+				},
+			};
+		}
+
+		// Load from compiled dist JS file — no path aliases, no DI
+		const filePath = path.join(known.packageDir, known.sourcePath);
+		// eslint-disable-next-line @typescript-eslint/no-require-imports
+		const mod = require(filePath) as Record<string, new () => INodeType | IVersionedNodeType>;
+		const NodeClass = mod[known.className];
+		const instance = new NodeClass();
+		this.cache.set(type, instance);
+		return instance;
+	}
+}
+
+let resolvedImports: ResolvedImports | undefined;
+
+interface ResolvedImports {
+	WorkflowExecute: new (
+		additionalData: IWorkflowExecuteAdditionalData,
+		mode: string,
+		runExecutionData: unknown,
+	) => { processRunExecutionData: (workflow: Workflow) => Promise<IRun> };
+	ExecutionLifecycleHooks: new (
+		mode: string,
+		executionId: string,
+		workflowData: unknown,
+	) => {
+		addHandler: (event: string, handler: (...args: unknown[]) => void) => void;
+	};
+	nodeTypes: DistNodeTypes;
+}
+
+function getPaths(): { corePath: string; nodesBasePath: string; langchainPath: string } {
+	const repoRoot = findRepoRoot(__dirname);
+	if (!repoRoot) {
+		throw new Error('Cannot find monorepo root — workflow execution requires the n8n monorepo');
+	}
+	return {
+		corePath: path.join(repoRoot, 'packages', 'core'),
+		nodesBasePath: path.join(repoRoot, 'packages', 'nodes-base'),
+		langchainPath: path.join(repoRoot, 'packages', '@n8n', 'nodes-langchain'),
+	};
+}
+
+async function resolveImports(): Promise<ResolvedImports> {
+	if (resolvedImports) return resolvedImports;
+
+	const { corePath, nodesBasePath, langchainPath } = getPaths();
+	const distPath = path.join(corePath, 'dist', 'execution-engine');
+
+	// Import WorkflowExecute and ExecutionLifecycleHooks from compiled dist
+	const weModule = (await import(path.join(distPath, 'workflow-execute.js'))) as Record<
+		string,
+		unknown
+	>;
+	const hookModule = (await import(path.join(distPath, 'execution-lifecycle-hooks.js'))) as Record<
+		string,
+		unknown
+	>;
+
+	// Load the lazy-load index from each package's dist/known/nodes.json.
+	// eslint-disable-next-line @typescript-eslint/no-require-imports
+	const nodesBaseKnown = require(path.join(nodesBasePath, 'dist', 'known', 'nodes.json')) as Record<
+		string,
+		{ className: string; sourcePath: string }
+	>;
+	// eslint-disable-next-line @typescript-eslint/no-require-imports
+	const langchainKnown = require(path.join(langchainPath, 'dist', 'known', 'nodes.json')) as Record<
+		string,
+		{ className: string; sourcePath: string }
+	>;
+
+	const nodeTypes = new DistNodeTypes([
+		{
+			packagePrefix: 'n8n-nodes-base',
+			packageDir: nodesBasePath,
+			knownNodes: nodesBaseKnown,
+		},
+		{
+			packagePrefix: '@n8n/n8n-nodes-langchain',
+			packageDir: langchainPath,
+			knownNodes: langchainKnown,
+		},
+	]);
+
+	resolvedImports = {
+		WorkflowExecute: weModule.WorkflowExecute as ResolvedImports['WorkflowExecute'],
+		ExecutionLifecycleHooks:
+			hookModule.ExecutionLifecycleHooks as ResolvedImports['ExecutionLifecycleHooks'],
+		nodeTypes,
+	};
+
+	return resolvedImports;
+}
+
+/*
+ * Proxy-based additionalData stub
+ *
+ * WorkflowExecute calls various methods on additionalData at runtime.
+ * we use a Proxy that returns a no-op function for any property access and
+ * allows specific values to be injected via the `overrides` map.
+ */
+function makeAdditionalDataStub(
+	overrides: Record<string, unknown>,
+): IWorkflowExecuteAdditionalData {
+	return new Proxy({} as IWorkflowExecuteAdditionalData, {
+		get(_target, prop: string) {
+			if (prop in overrides) return overrides[prop];
+			return () => undefined;
+		},
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Start node detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Fallback start-node finder for trigger nodes whose type description
+ * includes 'trigger' in its group (e.g. ChatTrigger, which is webhook-based
+ * and not recognised by Workflow.getStartNode()).
+ */
+export function findTriggerByGroup(nodes: INode[], nodeTypes: INodeTypes): INode | undefined {
+	return nodes.find((currentNode) => {
+		if (currentNode.disabled) return false;
+		const nt = nodeTypes.getByNameAndVersion(currentNode.type, currentNode.typeVersion);
+		return nt.description.group?.includes('trigger');
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Main execution function
+// ---------------------------------------------------------------------------
+
+/**
+ * Execute a workflow with pin data and return the result.
+ *
+ * Service/API nodes use pin data (skipping real API calls). Utility nodes
+ * (Set, If, Code, etc.) execute normally using their compiled dist
+ * implementations
+ *
+ * @param workflow - The workflow to execute
+ * @param pinData  - Pin data for service/API nodes
+ */
+export async function executeWorkflowWithPinData(
+	workflow: SimpleWorkflow,
+	pinData: IPinData,
+): Promise<ExecutionResult> {
+	const startTime = Date.now();
+	const executedNodes: string[] = [];
+
+	try {
+		const imports = await resolveImports();
+
+		// Create Workflow instance with real node implementations
+		const workflowInstance = new Workflow({
+			id: 'eval-execution',
+			nodes: workflow.nodes,
+			connections: workflow.connections,
+			nodeTypes: imports.nodeTypes,
+			active: false,
+		});
+
+		// Find start node. getStartNode() only recognises nodes with a trigger()/poll()
+		// method or those in STARTING_NODE_TYPES. Webhook-based triggers like ChatTrigger
+		// are missed, so fall back to any node whose description group includes 'trigger'.
+		const startNode = workflowInstance.getStartNode() ?? findTriggerByGroup(workflow.nodes, imports.nodeTypes);
+		if (!startNode) {
+			return {
+				success: false,
+				error: 'No start node found in workflow',
+				durationMs: Date.now() - startTime,
+				executedNodes: [],
+			};
+		}
+
+		// Set up execution lifecycle hooks
+		const hooks = new imports.ExecutionLifecycleHooks('trigger', '1', {} as never);
+		hooks.addHandler('nodeExecuteAfter', (nodeName: unknown) => {
+			if (typeof nodeName === 'string') {
+				executedNodes.push(nodeName);
+			}
+		});
+
+		const waitPromise = createDeferredPromise<IRun>();
+		hooks.addHandler('workflowExecuteAfter', (fullRunData: unknown) => {
+			waitPromise.resolve(fullRunData as IRun);
+		});
+
+		// Set up additional data with a Proxy stub so WorkflowExecute
+		const additionalData = makeAdditionalDataStub({ executionId: '1', hooks });
+
+		const runExecutionData = createRunExecutionData({
+			executionData: {
+				nodeExecutionStack: [
+					{
+						node: startNode,
+						data: { main: [[{ json: {} }]] },
+						source: null,
+					},
+				],
+			},
+			resultData: {
+				pinData,
+			},
+		});
+
+		const workflowExecute = new imports.WorkflowExecute(additionalData, 'manual', runExecutionData);
+		await workflowExecute.processRunExecutionData(workflowInstance);
+
+		const result = await waitPromise.promise;
+
+		// Check result for errors
+		const resultError = result.data?.resultData?.error;
+		const hasError = !!resultError;
+
+		return {
+			success: !hasError && result.status === 'success',
+			error: hasError ? ((resultError as Error)?.message ?? resultError) : undefined,
+			errorNode: hasError ? findErrorNode(result) : undefined,
+			durationMs: Date.now() - startTime,
+			executedNodes,
+		};
+	} catch (error) {
+		return {
+			success: false,
+			error: error instanceof Error ? error.message : String(error),
+			durationMs: Date.now() - startTime,
+			executedNodes,
+		};
+	}
+}
+
+/**
+ * Find the node that caused an execution error from run data.
+ */
+function findErrorNode(run: IRun): string | undefined {
+	const { runData } = run.data.resultData;
+	for (const [nodeName, nodeRuns] of Object.entries(runData)) {
+		for (const nodeRun of nodeRuns) {
+			if (nodeRun.error) {
+				return nodeName;
+			}
+		}
+	}
+	return undefined;
+}