first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/__tests__/index.test.ts

@@ -0,0 +1,57 @@
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import { createBinaryChecksEvaluator } from '../index';
+
+const mockNodeTypes: INodeTypeDescription[] = [];
+
+describe('createBinaryChecksEvaluator', () => {
+	it('creates evaluator with correct name', () => {
+		const evaluator = createBinaryChecksEvaluator({ nodeTypes: mockNodeTypes });
+		expect(evaluator.name).toBe('binary-checks');
+	});
+
+	it('runs deterministic checks without LLM', async () => {
+		const evaluator = createBinaryChecksEvaluator({ nodeTypes: mockNodeTypes });
+		const workflow = { name: 'test', nodes: [], connections: {} };
+		const feedback = await evaluator.evaluate(workflow, { prompt: 'test' });
+		expect(feedback.length).toBe(17); // 17 deterministic checks
+		expect(feedback.every((f) => f.evaluator === 'binary-checks')).toBe(true);
+		expect(feedback.every((f) => f.kind === 'metric')).toBe(true);
+		expect(feedback.every((f) => f.score === 0 || f.score === 1)).toBe(true);
+	});
+
+	it('filters checks with --checks option', async () => {
+		const evaluator = createBinaryChecksEvaluator({
+			nodeTypes: mockNodeTypes,
+			checks: ['has_nodes', 'has_trigger'],
+		});
+		const workflow = { name: 'test', nodes: [], connections: {} };
+		const feedback = await evaluator.evaluate(workflow, { prompt: 'test' });
+		expect(feedback.length).toBe(2);
+		expect(feedback.map((f) => f.metric).sort()).toEqual(['has_nodes', 'has_trigger']);
+	});
+
+	it('throws when all check names in filter are invalid', () => {
+		expect(() =>
+			createBinaryChecksEvaluator({
+				nodeTypes: mockNodeTypes,
+				checks: ['nonexistent_check'],
+			}),
+		).toThrow('No valid checks after filtering');
+	});
+
+	it('warns but continues when some check names are unrecognized', async () => {
+		const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
+		const evaluator = createBinaryChecksEvaluator({
+			nodeTypes: mockNodeTypes,
+			checks: ['has_nodes', 'nonexistent'],
+		});
+		expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('nonexistent'));
+		warnSpy.mockRestore();
+
+		const workflow = { name: 'test', nodes: [], connections: {} };
+		const feedback = await evaluator.evaluate(workflow, { prompt: 'test' });
+		expect(feedback.length).toBe(1);
+		expect(feedback[0].metric).toBe('has_nodes');
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/all-nodes-connected.ts

@@ -0,0 +1,50 @@
+import { getChildNodes, mapConnectionsByDestination } from 'n8n-workflow';
+
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+const STICKY_NOTE_TYPE = 'n8n-nodes-base.stickyNote';
+
+export const allNodesConnected: BinaryCheck = {
+	name: 'all_nodes_connected',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		// Filter out sticky notes — they are visual annotations, not part of the workflow graph
+		const activeNodes = (workflow.nodes ?? []).filter((n) => n.type !== STICKY_NOTE_TYPE);
+
+		if (activeNodes.length === 0) {
+			return { pass: true };
+		}
+
+		const connections = workflow.connections ?? {};
+
+		// Use n8n-workflow graph utilities to determine connectivity
+		const connectionsByDest = mapConnectionsByDestination(connections);
+
+		// Build full reachability set: nodes that appear as source or target in any connection,
+		// plus all nodes transitively reachable via getChildNodes (ALL connection types)
+		const connected = new Set<string>();
+		for (const sourceName of Object.keys(connections)) {
+			connected.add(sourceName);
+			for (const child of getChildNodes(connections, sourceName, 'ALL', -1)) {
+				connected.add(child);
+			}
+		}
+		for (const destName of Object.keys(connectionsByDest)) {
+			connected.add(destName);
+		}
+
+		const disconnected: string[] = [];
+		for (const node of activeNodes) {
+			if (!connected.has(node.name)) {
+				disconnected.push(node.name);
+			}
+		}
+
+		return {
+			pass: disconnected.length === 0,
+			...(disconnected.length > 0
+				? { comment: `Disconnected nodes: ${disconnected.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/create-validation-check.ts

@@ -0,0 +1,41 @@
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { ProgrammaticViolation } from '@/validation/types';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+type ValidateFn = (
+	workflow: SimpleWorkflow,
+	nodeTypes: INodeTypeDescription[],
+) => ProgrammaticViolation[];
+
+interface ValidationCheckConfig {
+	/** Check name in snake_case */
+	name: string;
+	/** Validation function to call. nodeTypes is always passed but can be ignored. */
+	validate: ValidateFn;
+	/** Optional filter to select specific violations. If omitted, all violations count. */
+	filter?: (violation: ProgrammaticViolation) => boolean;
+	/** Static comment when check fails. If omitted, violation descriptions are joined with '; '. */
+	failComment?: string;
+}
+
+/**
+ * Factory for binary checks that wrap existing validation functions.
+ *
+ * Eliminates boilerplate: call validator → filter violations → format comment.
+ */
+export function createValidationCheck(config: ValidationCheckConfig): BinaryCheck {
+	return {
+		name: config.name,
+		kind: 'deterministic',
+		// eslint-disable-next-line @typescript-eslint/require-await
+		async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+			const all = config.validate(workflow, ctx.nodeTypes);
+			const violations = config.filter ? all.filter(config.filter) : all;
+			if (violations.length === 0) return { pass: true };
+			const comment = config.failComment ?? violations.map((v) => v.description).join('; ');
+			return { pass: false, comment };
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/expressions-reference-existing-nodes.ts

@@ -0,0 +1,102 @@
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+/**
+ * Regex patterns to extract node names from n8n expression syntaxes.
+ * Mirrors ACCESS_PATTERNS from packages/workflow/src/node-reference-parser-utils.ts.
+ *
+ * Quoted patterns capture at group index 2; dot-notation captures at group index 1.
+ */
+const QUOTED_NODE_REFS: RegExp[] = [
+	/\$\(\s*(['"`])((?:\\.|(?!\1)[^\\])*)\1\s*\)/g, // $('Node Name')
+	/\$node\[\s*(['"])((?:\\.|(?!\1)[^\\])*)\1\s*\]/g, // $node["Node Name"]
+	/\$items\(\s*(['"])((?:\\.|(?!\1)[^\\])*)\1\s*[,)]/g, // $items("Node Name") or $items("Node Name", 0)
+];
+
+/** Legacy dot-notation: $node.NodeName.json... — name is a JS identifier after $node. */
+const DOT_NODE_REF = /\$node\.(\w+)\./g;
+
+/** Remove backslash escapes from a captured node name (e.g. `Node\'s` → `Node's`). */
+function unescapeNodeName(raw: string): string {
+	return raw.replace(/\\(.)/g, '$1');
+}
+
+/** Collect all matches from a global regex, returning captured groups at the given index. */
+function collectMatches(pattern: RegExp, text: string, groupIndex: number): string[] {
+	return Array.from(text.matchAll(pattern), (m) => m[groupIndex]);
+}
+
+/** Extract all referenced node names from an expression string. */
+function extractNodeNamesFromExpression(expression: string): string[] {
+	const names: string[] = [];
+
+	for (const pattern of QUOTED_NODE_REFS) {
+		for (const raw of collectMatches(pattern, expression, 2)) {
+			names.push(unescapeNodeName(raw));
+		}
+	}
+
+	// Legacy $node.Name dot-notation
+	for (const name of collectMatches(DOT_NODE_REF, expression, 1)) {
+		names.push(name);
+	}
+
+	return names;
+}
+
+/** Recursively extract all expression strings from node parameters. */
+function extractExpressionsFromParams(value: unknown, key?: string): string[] {
+	if (typeof value === 'string') {
+		// Expressions start with '=', jsCode fields are also expressions
+		if (value.charAt(0) === '=' || key === 'jsCode') {
+			return [value];
+		}
+		return [];
+	}
+
+	if (Array.isArray(value)) {
+		return value.flatMap((item) => extractExpressionsFromParams(item));
+	}
+
+	if (typeof value === 'object' && value !== null) {
+		return Object.entries(value).flatMap(([k, v]) => extractExpressionsFromParams(v, k));
+	}
+
+	return [];
+}
+
+export const expressionsReferenceExistingNodes: BinaryCheck = {
+	name: 'expressions_reference_existing_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const existingNodeNames = new Set(workflow.nodes.map((n) => n.name));
+		const invalid: string[] = [];
+
+		for (const node of workflow.nodes) {
+			if (!node.parameters) continue;
+
+			const expressions = extractExpressionsFromParams(node.parameters);
+			for (const expr of expressions) {
+				const referencedNames = extractNodeNamesFromExpression(expr);
+				for (const refName of referencedNames) {
+					if (!existingNodeNames.has(refName)) {
+						invalid.push(`"${refName}" (in node "${node.name}")`);
+					}
+				}
+			}
+		}
+
+		// Deduplicate
+		const unique = [...new Set(invalid)];
+
+		return {
+			pass: unique.length === 0,
+			...(unique.length > 0
+				? { comment: `Expressions reference non-existent nodes: ${unique.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/has-start-node.ts

@@ -0,0 +1,30 @@
+import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow';
+
+import { createNodeTypeMaps, getNodeTypeForNode } from '@/validation/utils/node-type-map';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+export const hasStartNode: BinaryCheck = {
+	name: 'has_start_node',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: false, comment: 'Workflow has no nodes' };
+		}
+
+		const connections = workflow.connections ?? {};
+		const { nodeTypeMap, nodeTypesByName } = createNodeTypeMaps(ctx.nodeTypes);
+
+		for (const node of workflow.nodes) {
+			const nodeType = getNodeTypeForNode(node, nodeTypeMap, nodeTypesByName);
+			if (!nodeType?.group.includes('trigger')) continue;
+
+			const children = getChildNodes(connections, node.name, NodeConnectionTypes.Main, 1);
+			if (children.length > 0) {
+				return { pass: true };
+			}
+		}
+
+		return { pass: false, comment: 'No trigger has a downstream node connected' };
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/index.ts

@@ -0,0 +1,40 @@
+import type { BinaryCheck } from '../types';
+import { allNodesConnected } from './all-nodes-connected';
+import { expressionsReferenceExistingNodes } from './expressions-reference-existing-nodes';
+import { hasStartNode } from './has-start-node';
+import { noEmptySetNodes } from './no-empty-set-nodes';
+import { noUnnecessaryCodeNodes } from './no-unnecessary-code-nodes';
+import { noUnreachableNodes } from './no-unreachable-nodes';
+import {
+	agentHasDynamicPrompt,
+	agentHasLanguageModel,
+	hasNodes,
+	hasTrigger,
+	memoryProperlyConnected,
+	noHardcodedCredentials,
+	noInvalidFromAi,
+	toolsHaveParameters,
+	validOptionsValues,
+	validRequiredParameters,
+	vectorStoreHasEmbeddings,
+} from './validation-checks';
+
+export const DETERMINISTIC_CHECKS: BinaryCheck[] = [
+	hasNodes,
+	allNodesConnected,
+	noUnreachableNodes,
+	hasTrigger,
+	noEmptySetNodes,
+	agentHasDynamicPrompt,
+	agentHasLanguageModel,
+	memoryProperlyConnected,
+	vectorStoreHasEmbeddings,
+	hasStartNode,
+	noHardcodedCredentials,
+	noUnnecessaryCodeNodes,
+	expressionsReferenceExistingNodes,
+	validRequiredParameters,
+	validOptionsValues,
+	noInvalidFromAi,
+	toolsHaveParameters,
+];

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-empty-set-nodes.ts

@@ -0,0 +1,54 @@
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Check if a Set node has values configured in either format:
+ * - v3.x+: parameters.assignments.assignments (array)
+ * - v2.x:  parameters.fields.values (array)
+ */
+function hasSetNodeValues(parameters: unknown): boolean {
+	if (!isRecord(parameters)) return false;
+
+	// v3.x+ format: assignments.assignments
+	const assignments = parameters.assignments;
+	if (isRecord(assignments)) {
+		if (Array.isArray(assignments.assignments) && assignments.assignments.length > 0) return true;
+	}
+
+	// v2.x format: fields.values
+	const fields = parameters.fields;
+	if (isRecord(fields)) {
+		if (Array.isArray(fields.values) && fields.values.length > 0) return true;
+	}
+
+	return false;
+}
+
+export const noEmptySetNodes: BinaryCheck = {
+	name: 'no_empty_set_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const emptySetNodes: string[] = [];
+		for (const node of workflow.nodes) {
+			if (node.type !== 'n8n-nodes-base.set') continue;
+
+			if (!hasSetNodeValues(node.parameters)) {
+				emptySetNodes.push(node.name);
+			}
+		}
+
+		return {
+			pass: emptySetNodes.length === 0,
+			...(emptySetNodes.length > 0
+				? { comment: `Empty Set nodes (no assignments): ${emptySetNodes.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-unnecessary-code-nodes.ts

@@ -0,0 +1,23 @@
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+export const noUnnecessaryCodeNodes: BinaryCheck = {
+	name: 'no_unnecessary_code_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		if (ctx.annotations?.code_necessary === true) {
+			return { pass: true, comment: 'Code marked as necessary by annotation' };
+		}
+
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const codeNodes = workflow.nodes.filter((n) => n.type === 'n8n-nodes-base.code');
+		return {
+			pass: codeNodes.length === 0,
+			...(codeNodes.length > 0
+				? { comment: `Unnecessary code nodes: ${codeNodes.map((n) => n.name).join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-unreachable-nodes.ts

@@ -0,0 +1,110 @@
+import type { INodeTypeDescription } from 'n8n-workflow';
+import { getChildNodes } from 'n8n-workflow';
+
+import { createNodeTypeMaps, getNodeTypeForNode } from '@/validation/utils/node-type-map';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+const STICKY_NOTE_TYPE = 'n8n-nodes-base.stickyNote';
+
+function findTriggerNames(workflow: SimpleWorkflow, nodeTypes: INodeTypeDescription[]): string[] {
+	const { nodeTypeMap, nodeTypesByName } = createNodeTypeMaps(nodeTypes);
+	const triggers: string[] = [];
+	for (const node of workflow.nodes) {
+		const nodeType = getNodeTypeForNode(node, nodeTypeMap, nodeTypesByName);
+		if (nodeType?.group.includes('trigger')) {
+			triggers.push(node.name);
+		}
+	}
+	return triggers;
+}
+
+/** Check if a node has an ai_* output connection to any node in the reachable set. */
+function connectsToReachableViaAi(
+	nodeConns: SimpleWorkflow['connections'][string],
+	reachable: Set<string>,
+): boolean {
+	return Object.entries(nodeConns).some(
+		([connType, connGroups]) =>
+			connType.startsWith('ai_') &&
+			connGroups.some((group) => group?.some((conn) => conn?.node && reachable.has(conn.node))),
+	);
+}
+
+/**
+ * Find sub-nodes that connect TO already-reachable nodes via ai_* outputs.
+ * Repeats until no new nodes are discovered (fixpoint), so deeply nested
+ * sub-nodes (e.g., Tool → AgentTool → Agent) are handled.
+ *
+ * Returns a new Set containing the original reachable nodes plus discovered sub-nodes.
+ */
+function expandReachableWithSubNodes(
+	reachable: ReadonlySet<string>,
+	connections: SimpleWorkflow['connections'],
+	allNodeNames: string[],
+): Set<string> {
+	const expanded = new Set(reachable);
+	let changed = true;
+	while (changed) {
+		changed = false;
+		for (const nodeName of allNodeNames) {
+			if (expanded.has(nodeName)) continue;
+
+			const nodeConns = connections[nodeName];
+			if (!nodeConns) continue;
+
+			if (connectsToReachableViaAi(nodeConns, expanded)) {
+				expanded.add(nodeName);
+				changed = true;
+			}
+		}
+	}
+	return expanded;
+}
+
+export const noUnreachableNodes: BinaryCheck = {
+	name: 'no_unreachable_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		// Filter out sticky notes — they are visual annotations, not part of the workflow graph
+		const activeNodes = (workflow.nodes ?? []).filter((n) => n.type !== STICKY_NOTE_TYPE);
+
+		if (activeNodes.length === 0) {
+			return { pass: true };
+		}
+
+		const connections = workflow.connections ?? {};
+		const triggers = findTriggerNames(workflow, ctx.nodeTypes);
+
+		// No triggers means all nodes are unreachable — fail explicitly
+		if (triggers.length === 0) {
+			const nodeNames = activeNodes.map((n) => n.name);
+			return {
+				pass: false,
+				comment: `No trigger found — all nodes unreachable: ${nodeNames.join(', ')}`,
+			};
+		}
+
+		// Forward BFS from triggers using n8n-workflow's getChildNodes (ALL connection types)
+		const reachable = new Set<string>(triggers);
+		for (const trigger of triggers) {
+			for (const child of getChildNodes(connections, trigger, 'ALL', -1)) {
+				reachable.add(child);
+			}
+		}
+
+		// Iteratively expand: sub-nodes connect TO parents via ai_* outputs.
+		// Repeat until stable so nested sub-nodes (Tool → AgentTool → Agent) are found.
+		const allNodeNames = activeNodes.map((n) => n.name);
+		const expanded = expandReachableWithSubNodes(reachable, connections, allNodeNames);
+
+		const unreachable = activeNodes.filter((n) => !expanded.has(n.name)).map((n) => n.name);
+
+		return {
+			pass: unreachable.length === 0,
+			...(unreachable.length > 0
+				? { comment: `Unreachable nodes: ${unreachable.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/validation-checks.ts

@@ -0,0 +1,93 @@
+/**
+ * Binary checks that wrap existing validation functions.
+ *
+ * Each check calls a validator from @/validation/checks, optionally filters
+ * the violations, and returns a binary pass/fail result.
+ */
+
+import {
+	validateAgentPrompt,
+	validateConnections,
+	validateCredentials,
+	validateFromAi,
+	validateNodes,
+	validateParameters,
+	validateTools,
+	validateTrigger,
+} from '@/validation/checks';
+
+import { createValidationCheck } from './create-validation-check';
+
+export const hasNodes = createValidationCheck({
+	name: 'has_nodes',
+	validate: validateNodes,
+	filter: (v) => v.name === 'workflow-has-no-nodes',
+	failComment: 'Workflow has no nodes',
+});
+
+export const hasTrigger = createValidationCheck({
+	name: 'has_trigger',
+	validate: validateTrigger,
+	filter: (v) => v.name === 'workflow-has-no-trigger',
+	failComment: 'Workflow has no trigger node',
+});
+
+export const agentHasDynamicPrompt = createValidationCheck({
+	name: 'agent_has_dynamic_prompt',
+	validate: (workflow) => validateAgentPrompt(workflow),
+	filter: (v) => v.name === 'agent-static-prompt',
+});
+
+export const agentHasLanguageModel = createValidationCheck({
+	name: 'agent_has_language_model',
+	validate: validateConnections,
+	filter: (v) =>
+		v.name === 'node-missing-required-input' &&
+		v.metadata?.missingType === 'ai_languageModel' &&
+		(v.metadata?.nodeType?.includes('langchain.agent') ?? false),
+	failComment: 'Agent node missing required language model connection',
+});
+
+export const memoryProperlyConnected = createValidationCheck({
+	name: 'memory_properly_connected',
+	validate: validateConnections,
+	filter: (v) => v.name === 'sub-node-not-connected' && v.metadata?.outputType === 'ai_memory',
+	failComment: 'Memory node not properly connected to parent node',
+});
+
+export const vectorStoreHasEmbeddings = createValidationCheck({
+	name: 'vector_store_has_embeddings',
+	validate: validateConnections,
+	filter: (v) =>
+		v.name === 'node-missing-required-input' &&
+		v.metadata?.missingType === 'ai_embedding' &&
+		(v.metadata?.nodeType?.includes('vectorStore') ?? false),
+	failComment: 'Vector store node missing required embeddings connection',
+});
+
+export const noHardcodedCredentials = createValidationCheck({
+	name: 'no_hardcoded_credentials',
+	validate: (workflow) => validateCredentials(workflow),
+});
+
+export const validRequiredParameters = createValidationCheck({
+	name: 'valid_required_parameters',
+	validate: validateParameters,
+	filter: (v) => v.name === 'node-missing-required-parameter',
+});
+
+export const validOptionsValues = createValidationCheck({
+	name: 'valid_options_values',
+	validate: validateParameters,
+	filter: (v) => v.name === 'node-invalid-options-value',
+});
+
+export const noInvalidFromAi = createValidationCheck({
+	name: 'no_invalid_from_ai',
+	validate: validateFromAi,
+});
+
+export const toolsHaveParameters = createValidationCheck({
+	name: 'tools_have_parameters',
+	validate: validateTools,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/index.ts

@@ -0,0 +1,104 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { DETERMINISTIC_CHECKS } from './checks';
+import { LLM_CHECKS } from './llm-checks';
+import type { BinaryCheck, BinaryCheckContext } from './types';
+import type { EvaluationContext, Evaluator, Feedback } from '../../harness/harness-types';
+
+export interface BinaryChecksEvaluatorOptions {
+	nodeTypes: INodeTypeDescription[];
+	llm?: BaseChatModel;
+	checks?: string[];
+}
+
+const EVALUATOR_NAME = 'binary-checks';
+
+/**
+ * Create a binary-checks evaluator that runs deterministic and optional LLM checks.
+ *
+ * Each check emits one Feedback with score 0 or 1.
+ */
+export function createBinaryChecksEvaluator(
+	options: BinaryChecksEvaluatorOptions,
+): Evaluator<EvaluationContext> {
+	const allChecks: BinaryCheck[] = [...DETERMINISTIC_CHECKS, ...(options.llm ? LLM_CHECKS : [])];
+
+	const allCheckNames = allChecks.map((c) => c.name);
+
+	let selectedChecks: BinaryCheck[];
+
+	if (options.checks && options.checks.length > 0) {
+		const validNames = new Set(allCheckNames);
+		const unrecognized = options.checks.filter((name) => !validNames.has(name));
+
+		for (const name of unrecognized) {
+			console.warn(`Warning: unrecognized check name "${name}" in --checks filter`);
+		}
+
+		selectedChecks = allChecks.filter((c) => options.checks!.includes(c.name));
+
+		if (selectedChecks.length === 0) {
+			throw new Error(
+				`No valid checks after filtering. Requested: ${options.checks.join(', ')}. Available: ${allCheckNames.join(', ')}`,
+			);
+		}
+	} else {
+		selectedChecks = allChecks;
+	}
+
+	return {
+		name: EVALUATOR_NAME,
+
+		async evaluate(workflow: SimpleWorkflow, ctx: EvaluationContext): Promise<Feedback[]> {
+			const checkCtx: BinaryCheckContext = {
+				prompt: ctx.prompt,
+				nodeTypes: options.nodeTypes,
+				annotations: ctx.annotations,
+				llm: options.llm,
+				// Intentionally omit llmCallLimiter: binary-checks LLM judges are small,
+				// cheap calls that should run in parallel, not throttled by the shared limiter.
+				timeoutMs: ctx.timeoutMs,
+			};
+
+			const results = await Promise.allSettled(
+				selectedChecks.map(async (check) => {
+					try {
+						const result = await check.run(workflow, checkCtx);
+						return { check, result };
+					} catch (error) {
+						const message = error instanceof Error ? error.message : String(error);
+						return { check, result: { pass: false, comment: `Error: ${message}` } };
+					}
+				}),
+			);
+
+			return results.map((settled, i) => {
+				if (settled.status === 'fulfilled') {
+					const { check, result } = settled.value;
+					return {
+						evaluator: EVALUATOR_NAME,
+						metric: check.name,
+						score: result.pass ? 1 : 0,
+						kind: 'metric' as const,
+						...(result.comment ? { comment: result.comment } : {}),
+					};
+				}
+
+				// Should not happen since we catch inside, but handle gracefully
+				const check = selectedChecks[i];
+				const reason =
+					settled.reason instanceof Error ? settled.reason.message : String(settled.reason);
+				return {
+					evaluator: EVALUATOR_NAME,
+					metric: check.name,
+					score: 0,
+					kind: 'metric' as const,
+					comment: `Unexpected error: ${reason}`,
+				};
+			});
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/__tests__/create-llm-check.test.ts

@@ -0,0 +1,21 @@
+import { createLlmCheck } from '../create-llm-check';
+
+describe('createLlmCheck', () => {
+	it('returns pass: true with "Skipped" comment when no LLM provided', async () => {
+		const check = createLlmCheck({
+			name: 'test_check',
+			systemPrompt: 'test',
+			humanTemplate: 'test {userPrompt} {generatedWorkflow} {referenceSection}',
+		});
+
+		expect(check.name).toBe('test_check');
+		expect(check.kind).toBe('llm');
+
+		const result = await check.run(
+			{ name: 'test', nodes: [], connections: {} },
+			{ prompt: 'test', nodeTypes: [] },
+		);
+		expect(result.pass).toBe(true);
+		expect(result.comment).toBe('Skipped: no LLM');
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/correct-node-operations.ts

@@ -0,0 +1,28 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const correctNodeOperations = createLlmCheck({
+	name: 'correct_node_operations',
+	systemPrompt: `You are an evaluator checking whether n8n workflow nodes use the correct resource and operation settings.
+
+For each node that has resource/operation parameters, verify:
+1. The resource matches what the node SHOULD operate on given its name and the workflow's purpose
+2. The operation matches the intended action (get, getAll, create, update, delete, etc.)
+3. Two nodes that should do different things are NOT configured identically
+
+Common mistakes to catch:
+- A node named "Get Captions" but configured with resource: "video" instead of resource: "caption"
+- A node that should create records but uses operation: "get"
+- Two nodes configured identically when they should fetch different resources
+
+Nodes without resource/operation parameters (triggers, Set, Merge, AI agents, LLM models) should be skipped.
+
+Respond with pass: true ONLY if all resource/operation combinations are correct for the workflow's intent.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Check each node's resource and operation parameters. Are they all correct for this workflow's purpose?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/create-llm-check.ts

@@ -0,0 +1,46 @@
+import { runWithOptionalLimiter, withTimeout } from '../../../harness/evaluation-helpers';
+import { createEvaluatorChain, invokeEvaluatorChain } from '../../llm-judge/evaluators/base';
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+import { binaryJudgeResultSchema } from './schemas';
+
+const REASONING_FIRST_SUFFIX = `
+
+IMPORTANT: Write your full reasoning FIRST. Only AFTER completing your analysis, decide on pass or fail based on what you wrote. Do not decide pass/fail before reasoning.`;
+
+export function createLlmCheck(options: {
+	name: string;
+	systemPrompt: string;
+	humanTemplate: string;
+}): BinaryCheck {
+	const systemPrompt = options.systemPrompt + REASONING_FIRST_SUFFIX;
+
+	return {
+		name: options.name,
+		kind: 'llm',
+		async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+			if (!ctx.llm) {
+				return { pass: true, comment: 'Skipped: no LLM' };
+			}
+
+			const chain = createEvaluatorChain(
+				ctx.llm,
+				binaryJudgeResultSchema,
+				systemPrompt,
+				options.humanTemplate,
+			);
+
+			const result = await runWithOptionalLimiter(async () => {
+				return await withTimeout({
+					promise: invokeEvaluatorChain(chain, {
+						userPrompt: ctx.prompt,
+						generatedWorkflow: workflow,
+					}),
+					timeoutMs: ctx.timeoutMs,
+					label: `binary-checks:${options.name}`,
+				});
+			}, ctx.llmCallLimiter);
+
+			return { pass: result.pass, comment: result.reasoning };
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/descriptive-node-names.ts

@@ -0,0 +1,24 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const descriptiveNodeNames = createLlmCheck({
+	name: 'descriptive_node_names',
+	systemPrompt: `You are an evaluator checking whether n8n workflow nodes have meaningful, descriptive names.
+Check:
+- Are node names descriptive of their purpose (e.g., "Send Welcome Email" vs "HTTP Request")?
+- Do names avoid default/generic names like "HTTP Request", "Code", "Set", "IF"?
+- Are names specific enough to understand the workflow at a glance?
+
+Note: Trigger nodes commonly keep their default names (e.g., "When clicking 'Test workflow'"), which is acceptable.
+A few default names in a simple workflow is acceptable; the check is about overall naming quality.
+
+Respond with pass: true if node names are generally descriptive and meaningful, false otherwise.
+Provide clear reasoning for your decision.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Do the nodes in this workflow have descriptive, meaningful names?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/fulfills-user-request.ts

@@ -0,0 +1,25 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const fulfillsUserRequest = createLlmCheck({
+	name: 'fulfills_user_request',
+	systemPrompt: `You are a strict evaluator checking whether an n8n workflow fulfills a user's request.
+
+For each feature the user explicitly asked for, check:
+1. Is there a node of the correct TYPE for that feature? (e.g., YouTube node for YouTube operations)
+2. Is that node configured with the correct RESOURCE and OPERATION? (e.g., resource: "caption" for fetching captions, not resource: "video")
+3. Is the node actually CONNECTED in the workflow flow?
+
+A node that exists but is misconfigured does NOT count as fulfilling the requirement.
+For example, a YouTube node with resource: "video" does NOT fulfill a request to "fetch captions" — captions require resource: "caption".
+
+Be binary: pass ONLY if every explicitly requested feature has a correctly-typed AND correctly-configured node.
+Do NOT pass just because a node with the right name exists — verify its actual parameters.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+For each feature the user requested, is there a correctly configured node? List each requirement and whether it's met.`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/handles-multiple-items.ts

@@ -0,0 +1,32 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const handlesMultipleItems = createLlmCheck({
+	name: 'handles_multiple_items',
+	systemPrompt: `You are an evaluator checking whether an n8n workflow handles multiple items correctly.
+
+Important n8n context:
+- Most n8n nodes automatically process ALL incoming items one by one — this is correct default behavior
+- A workflow designed for single-item processing (manual trigger + one record) does NOT need batch handling
+- Merge nodes with "combineByPosition" are correct for merging parallel single-item branches
+- AI Agent nodes process one item at a time, which is normal
+
+Only FAIL if there is a clear structural problem:
+- A node configured with multipleFiles/multiple inputs but no downstream handling for arrays
+- A splitInBatches that's clearly needed but missing (e.g., sending individual API calls for each item in a large list)
+- An aggregate node producing an array that downstream nodes don't handle
+
+Do NOT fail for:
+- Single-item workflows (manual trigger processing one record)
+- Workflows where n8n's automatic item-by-item processing is sufficient
+- Chatbot/agent workflows that process one message at a time
+
+Respond with pass: true if the workflow handles items correctly for its use case.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Does this workflow handle multiple items correctly for its intended use case?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/index.ts

@@ -0,0 +1,14 @@
+import type { BinaryCheck } from '../types';
+import { correctNodeOperations } from './correct-node-operations';
+import { descriptiveNodeNames } from './descriptive-node-names';
+import { fulfillsUserRequest } from './fulfills-user-request';
+import { handlesMultipleItems } from './handles-multiple-items';
+import { validDataFlow } from './valid-data-flow';
+
+export const LLM_CHECKS: BinaryCheck[] = [
+	fulfillsUserRequest,
+	correctNodeOperations,
+	validDataFlow,
+	handlesMultipleItems,
+	descriptiveNodeNames,
+];

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/schemas.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod';
+
+export const binaryJudgeResultSchema = z.object({
+	reasoning: z
+		.string()
+		.describe('Step-by-step analysis. Write this FIRST, BEFORE deciding pass/fail.'),
+	pass: z
+		.boolean()
+		.describe(
+			'Final verdict derived from the reasoning above. true = all criteria met, false = at least one issue found.',
+		),
+});
+
+export type BinaryJudgeResult = z.infer<typeof binaryJudgeResultSchema>;

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/valid-data-flow.ts

@@ -0,0 +1,36 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const validDataFlow = createLlmCheck({
+	name: 'valid_data_flow',
+	systemPrompt: `You are an evaluator checking whether expressions in an n8n workflow reference fields that actually exist upstream.
+
+For each expression in node parameters, check:
+1. \`{{ $json.fieldName }}\` — does the immediately upstream node output this field?
+2. \`$('NodeName').item.json.field\` — does that node exist, and does it output that field?
+3. Cross-references between nodes are consistent (field set in one node matches what's read in another)
+
+Important n8n context:
+- Manual Trigger and Schedule Trigger nodes output an empty object — they do NOT provide custom fields unless a Set node is placed after them
+- YouTube video.get returns \`snippet.title\`, \`snippet.description\`, etc. — NOT \`caption\` or \`transcript\`
+- Set nodes output exactly the fields defined in their assignments
+- AI Agent nodes output \`{ output: string }\`
+- Merge nodes combine fields from all inputs
+
+Focus on CRITICAL issues only:
+- Expressions referencing fields that clearly don't exist upstream (e.g., \`$json.transcript\` when no node produces a transcript field)
+- Expressions referencing nodes that don't exist in the workflow
+
+Do NOT fail for:
+- Minor field name case differences
+- Fields that might be available through n8n's built-in variables (\`$execution\`, \`$workflow\`, etc.)
+
+Respond with pass: true if there are no critical data flow issues.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Check each expression in the workflow. Do they reference fields that exist upstream?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/types.ts

@@ -0,0 +1,29 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { SimpleWorkflow } from '@/types';
+
+import type { LlmCallLimiter } from '../../harness/harness-types';
+
+export interface BinaryCheckResult {
+	pass: boolean;
+	comment?: string;
+}
+
+export interface BinaryCheckContext {
+	prompt: string;
+	nodeTypes: INodeTypeDescription[];
+	annotations?: Record<string, unknown>;
+	llm?: BaseChatModel;
+	llmCallLimiter?: LlmCallLimiter;
+	timeoutMs?: number;
+}
+
+export interface BinaryCheck {
+	name: string;
+	kind: 'deterministic' | 'llm';
+	run(workflow: SimpleWorkflow, ctx: BinaryCheckContext): Promise<BinaryCheckResult>;
+}
+
+// Re-export for convenience
+export type { SimpleWorkflow };