first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/all-nodes-connected.ts

@@ -0,0 +1,50 @@
+import { getChildNodes, mapConnectionsByDestination } from 'n8n-workflow';
+
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+const STICKY_NOTE_TYPE = 'n8n-nodes-base.stickyNote';
+
+export const allNodesConnected: BinaryCheck = {
+	name: 'all_nodes_connected',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		// Filter out sticky notes — they are visual annotations, not part of the workflow graph
+		const activeNodes = (workflow.nodes ?? []).filter((n) => n.type !== STICKY_NOTE_TYPE);
+
+		if (activeNodes.length === 0) {
+			return { pass: true };
+		}
+
+		const connections = workflow.connections ?? {};
+
+		// Use n8n-workflow graph utilities to determine connectivity
+		const connectionsByDest = mapConnectionsByDestination(connections);
+
+		// Build full reachability set: nodes that appear as source or target in any connection,
+		// plus all nodes transitively reachable via getChildNodes (ALL connection types)
+		const connected = new Set<string>();
+		for (const sourceName of Object.keys(connections)) {
+			connected.add(sourceName);
+			for (const child of getChildNodes(connections, sourceName, 'ALL', -1)) {
+				connected.add(child);
+			}
+		}
+		for (const destName of Object.keys(connectionsByDest)) {
+			connected.add(destName);
+		}
+
+		const disconnected: string[] = [];
+		for (const node of activeNodes) {
+			if (!connected.has(node.name)) {
+				disconnected.push(node.name);
+			}
+		}
+
+		return {
+			pass: disconnected.length === 0,
+			...(disconnected.length > 0
+				? { comment: `Disconnected nodes: ${disconnected.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/create-validation-check.ts

@@ -0,0 +1,41 @@
+import type { INodeTypeDescription } from 'n8n-workflow';
+
+import type { ProgrammaticViolation } from '@/validation/types';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+type ValidateFn = (
+	workflow: SimpleWorkflow,
+	nodeTypes: INodeTypeDescription[],
+) => ProgrammaticViolation[];
+
+interface ValidationCheckConfig {
+	/** Check name in snake_case */
+	name: string;
+	/** Validation function to call. nodeTypes is always passed but can be ignored. */
+	validate: ValidateFn;
+	/** Optional filter to select specific violations. If omitted, all violations count. */
+	filter?: (violation: ProgrammaticViolation) => boolean;
+	/** Static comment when check fails. If omitted, violation descriptions are joined with '; '. */
+	failComment?: string;
+}
+
+/**
+ * Factory for binary checks that wrap existing validation functions.
+ *
+ * Eliminates boilerplate: call validator → filter violations → format comment.
+ */
+export function createValidationCheck(config: ValidationCheckConfig): BinaryCheck {
+	return {
+		name: config.name,
+		kind: 'deterministic',
+		// eslint-disable-next-line @typescript-eslint/require-await
+		async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+			const all = config.validate(workflow, ctx.nodeTypes);
+			const violations = config.filter ? all.filter(config.filter) : all;
+			if (violations.length === 0) return { pass: true };
+			const comment = config.failComment ?? violations.map((v) => v.description).join('; ');
+			return { pass: false, comment };
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/expressions-reference-existing-nodes.ts

@@ -0,0 +1,102 @@
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+/**
+ * Regex patterns to extract node names from n8n expression syntaxes.
+ * Mirrors ACCESS_PATTERNS from packages/workflow/src/node-reference-parser-utils.ts.
+ *
+ * Quoted patterns capture at group index 2; dot-notation captures at group index 1.
+ */
+const QUOTED_NODE_REFS: RegExp[] = [
+	/\$\(\s*(['"`])((?:\\.|(?!\1)[^\\])*)\1\s*\)/g, // $('Node Name')
+	/\$node\[\s*(['"])((?:\\.|(?!\1)[^\\])*)\1\s*\]/g, // $node["Node Name"]
+	/\$items\(\s*(['"])((?:\\.|(?!\1)[^\\])*)\1\s*[,)]/g, // $items("Node Name") or $items("Node Name", 0)
+];
+
+/** Legacy dot-notation: $node.NodeName.json... — name is a JS identifier after $node. */
+const DOT_NODE_REF = /\$node\.(\w+)\./g;
+
+/** Remove backslash escapes from a captured node name (e.g. `Node\'s` → `Node's`). */
+function unescapeNodeName(raw: string): string {
+	return raw.replace(/\\(.)/g, '$1');
+}
+
+/** Collect all matches from a global regex, returning captured groups at the given index. */
+function collectMatches(pattern: RegExp, text: string, groupIndex: number): string[] {
+	return Array.from(text.matchAll(pattern), (m) => m[groupIndex]);
+}
+
+/** Extract all referenced node names from an expression string. */
+function extractNodeNamesFromExpression(expression: string): string[] {
+	const names: string[] = [];
+
+	for (const pattern of QUOTED_NODE_REFS) {
+		for (const raw of collectMatches(pattern, expression, 2)) {
+			names.push(unescapeNodeName(raw));
+		}
+	}
+
+	// Legacy $node.Name dot-notation
+	for (const name of collectMatches(DOT_NODE_REF, expression, 1)) {
+		names.push(name);
+	}
+
+	return names;
+}
+
+/** Recursively extract all expression strings from node parameters. */
+function extractExpressionsFromParams(value: unknown, key?: string): string[] {
+	if (typeof value === 'string') {
+		// Expressions start with '=', jsCode fields are also expressions
+		if (value.charAt(0) === '=' || key === 'jsCode') {
+			return [value];
+		}
+		return [];
+	}
+
+	if (Array.isArray(value)) {
+		return value.flatMap((item) => extractExpressionsFromParams(item));
+	}
+
+	if (typeof value === 'object' && value !== null) {
+		return Object.entries(value).flatMap(([k, v]) => extractExpressionsFromParams(v, k));
+	}
+
+	return [];
+}
+
+export const expressionsReferenceExistingNodes: BinaryCheck = {
+	name: 'expressions_reference_existing_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const existingNodeNames = new Set(workflow.nodes.map((n) => n.name));
+		const invalid: string[] = [];
+
+		for (const node of workflow.nodes) {
+			if (!node.parameters) continue;
+
+			const expressions = extractExpressionsFromParams(node.parameters);
+			for (const expr of expressions) {
+				const referencedNames = extractNodeNamesFromExpression(expr);
+				for (const refName of referencedNames) {
+					if (!existingNodeNames.has(refName)) {
+						invalid.push(`"${refName}" (in node "${node.name}")`);
+					}
+				}
+			}
+		}
+
+		// Deduplicate
+		const unique = [...new Set(invalid)];
+
+		return {
+			pass: unique.length === 0,
+			...(unique.length > 0
+				? { comment: `Expressions reference non-existent nodes: ${unique.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/has-start-node.ts

@@ -0,0 +1,30 @@
+import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow';
+
+import { createNodeTypeMaps, getNodeTypeForNode } from '@/validation/utils/node-type-map';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+export const hasStartNode: BinaryCheck = {
+	name: 'has_start_node',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: false, comment: 'Workflow has no nodes' };
+		}
+
+		const connections = workflow.connections ?? {};
+		const { nodeTypeMap, nodeTypesByName } = createNodeTypeMaps(ctx.nodeTypes);
+
+		for (const node of workflow.nodes) {
+			const nodeType = getNodeTypeForNode(node, nodeTypeMap, nodeTypesByName);
+			if (!nodeType?.group.includes('trigger')) continue;
+
+			const children = getChildNodes(connections, node.name, NodeConnectionTypes.Main, 1);
+			if (children.length > 0) {
+				return { pass: true };
+			}
+		}
+
+		return { pass: false, comment: 'No trigger has a downstream node connected' };
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/index.ts

@@ -0,0 +1,40 @@
+import type { BinaryCheck } from '../types';
+import { allNodesConnected } from './all-nodes-connected';
+import { expressionsReferenceExistingNodes } from './expressions-reference-existing-nodes';
+import { hasStartNode } from './has-start-node';
+import { noEmptySetNodes } from './no-empty-set-nodes';
+import { noUnnecessaryCodeNodes } from './no-unnecessary-code-nodes';
+import { noUnreachableNodes } from './no-unreachable-nodes';
+import {
+	agentHasDynamicPrompt,
+	agentHasLanguageModel,
+	hasNodes,
+	hasTrigger,
+	memoryProperlyConnected,
+	noHardcodedCredentials,
+	noInvalidFromAi,
+	toolsHaveParameters,
+	validOptionsValues,
+	validRequiredParameters,
+	vectorStoreHasEmbeddings,
+} from './validation-checks';
+
+export const DETERMINISTIC_CHECKS: BinaryCheck[] = [
+	hasNodes,
+	allNodesConnected,
+	noUnreachableNodes,
+	hasTrigger,
+	noEmptySetNodes,
+	agentHasDynamicPrompt,
+	agentHasLanguageModel,
+	memoryProperlyConnected,
+	vectorStoreHasEmbeddings,
+	hasStartNode,
+	noHardcodedCredentials,
+	noUnnecessaryCodeNodes,
+	expressionsReferenceExistingNodes,
+	validRequiredParameters,
+	validOptionsValues,
+	noInvalidFromAi,
+	toolsHaveParameters,
+];

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-empty-set-nodes.ts

@@ -0,0 +1,54 @@
+import type { BinaryCheck, SimpleWorkflow } from '../types';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Check if a Set node has values configured in either format:
+ * - v3.x+: parameters.assignments.assignments (array)
+ * - v2.x:  parameters.fields.values (array)
+ */
+function hasSetNodeValues(parameters: unknown): boolean {
+	if (!isRecord(parameters)) return false;
+
+	// v3.x+ format: assignments.assignments
+	const assignments = parameters.assignments;
+	if (isRecord(assignments)) {
+		if (Array.isArray(assignments.assignments) && assignments.assignments.length > 0) return true;
+	}
+
+	// v2.x format: fields.values
+	const fields = parameters.fields;
+	if (isRecord(fields)) {
+		if (Array.isArray(fields.values) && fields.values.length > 0) return true;
+	}
+
+	return false;
+}
+
+export const noEmptySetNodes: BinaryCheck = {
+	name: 'no_empty_set_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow) {
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const emptySetNodes: string[] = [];
+		for (const node of workflow.nodes) {
+			if (node.type !== 'n8n-nodes-base.set') continue;
+
+			if (!hasSetNodeValues(node.parameters)) {
+				emptySetNodes.push(node.name);
+			}
+		}
+
+		return {
+			pass: emptySetNodes.length === 0,
+			...(emptySetNodes.length > 0
+				? { comment: `Empty Set nodes (no assignments): ${emptySetNodes.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-unnecessary-code-nodes.ts

@@ -0,0 +1,23 @@
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+export const noUnnecessaryCodeNodes: BinaryCheck = {
+	name: 'no_unnecessary_code_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		if (ctx.annotations?.code_necessary === true) {
+			return { pass: true, comment: 'Code marked as necessary by annotation' };
+		}
+
+		if (!workflow.nodes || workflow.nodes.length === 0) {
+			return { pass: true };
+		}
+
+		const codeNodes = workflow.nodes.filter((n) => n.type === 'n8n-nodes-base.code');
+		return {
+			pass: codeNodes.length === 0,
+			...(codeNodes.length > 0
+				? { comment: `Unnecessary code nodes: ${codeNodes.map((n) => n.name).join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/no-unreachable-nodes.ts

@@ -0,0 +1,110 @@
+import type { INodeTypeDescription } from 'n8n-workflow';
+import { getChildNodes } from 'n8n-workflow';
+
+import { createNodeTypeMaps, getNodeTypeForNode } from '@/validation/utils/node-type-map';
+
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+
+const STICKY_NOTE_TYPE = 'n8n-nodes-base.stickyNote';
+
+function findTriggerNames(workflow: SimpleWorkflow, nodeTypes: INodeTypeDescription[]): string[] {
+	const { nodeTypeMap, nodeTypesByName } = createNodeTypeMaps(nodeTypes);
+	const triggers: string[] = [];
+	for (const node of workflow.nodes) {
+		const nodeType = getNodeTypeForNode(node, nodeTypeMap, nodeTypesByName);
+		if (nodeType?.group.includes('trigger')) {
+			triggers.push(node.name);
+		}
+	}
+	return triggers;
+}
+
+/** Check if a node has an ai_* output connection to any node in the reachable set. */
+function connectsToReachableViaAi(
+	nodeConns: SimpleWorkflow['connections'][string],
+	reachable: Set<string>,
+): boolean {
+	return Object.entries(nodeConns).some(
+		([connType, connGroups]) =>
+			connType.startsWith('ai_') &&
+			connGroups.some((group) => group?.some((conn) => conn?.node && reachable.has(conn.node))),
+	);
+}
+
+/**
+ * Find sub-nodes that connect TO already-reachable nodes via ai_* outputs.
+ * Repeats until no new nodes are discovered (fixpoint), so deeply nested
+ * sub-nodes (e.g., Tool → AgentTool → Agent) are handled.
+ *
+ * Returns a new Set containing the original reachable nodes plus discovered sub-nodes.
+ */
+function expandReachableWithSubNodes(
+	reachable: ReadonlySet<string>,
+	connections: SimpleWorkflow['connections'],
+	allNodeNames: string[],
+): Set<string> {
+	const expanded = new Set(reachable);
+	let changed = true;
+	while (changed) {
+		changed = false;
+		for (const nodeName of allNodeNames) {
+			if (expanded.has(nodeName)) continue;
+
+			const nodeConns = connections[nodeName];
+			if (!nodeConns) continue;
+
+			if (connectsToReachableViaAi(nodeConns, expanded)) {
+				expanded.add(nodeName);
+				changed = true;
+			}
+		}
+	}
+	return expanded;
+}
+
+export const noUnreachableNodes: BinaryCheck = {
+	name: 'no_unreachable_nodes',
+	kind: 'deterministic',
+	async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+		// Filter out sticky notes — they are visual annotations, not part of the workflow graph
+		const activeNodes = (workflow.nodes ?? []).filter((n) => n.type !== STICKY_NOTE_TYPE);
+
+		if (activeNodes.length === 0) {
+			return { pass: true };
+		}
+
+		const connections = workflow.connections ?? {};
+		const triggers = findTriggerNames(workflow, ctx.nodeTypes);
+
+		// No triggers means all nodes are unreachable — fail explicitly
+		if (triggers.length === 0) {
+			const nodeNames = activeNodes.map((n) => n.name);
+			return {
+				pass: false,
+				comment: `No trigger found — all nodes unreachable: ${nodeNames.join(', ')}`,
+			};
+		}
+
+		// Forward BFS from triggers using n8n-workflow's getChildNodes (ALL connection types)
+		const reachable = new Set<string>(triggers);
+		for (const trigger of triggers) {
+			for (const child of getChildNodes(connections, trigger, 'ALL', -1)) {
+				reachable.add(child);
+			}
+		}
+
+		// Iteratively expand: sub-nodes connect TO parents via ai_* outputs.
+		// Repeat until stable so nested sub-nodes (Tool → AgentTool → Agent) are found.
+		const allNodeNames = activeNodes.map((n) => n.name);
+		const expanded = expandReachableWithSubNodes(reachable, connections, allNodeNames);
+
+		const unreachable = activeNodes.filter((n) => !expanded.has(n.name)).map((n) => n.name);
+
+		return {
+			pass: unreachable.length === 0,
+			...(unreachable.length > 0
+				? { comment: `Unreachable nodes: ${unreachable.join(', ')}` }
+				: {}),
+		};
+	},
+};

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/checks/validation-checks.ts

@@ -0,0 +1,93 @@
+/**
+ * Binary checks that wrap existing validation functions.
+ *
+ * Each check calls a validator from @/validation/checks, optionally filters
+ * the violations, and returns a binary pass/fail result.
+ */
+
+import {
+	validateAgentPrompt,
+	validateConnections,
+	validateCredentials,
+	validateFromAi,
+	validateNodes,
+	validateParameters,
+	validateTools,
+	validateTrigger,
+} from '@/validation/checks';
+
+import { createValidationCheck } from './create-validation-check';
+
+export const hasNodes = createValidationCheck({
+	name: 'has_nodes',
+	validate: validateNodes,
+	filter: (v) => v.name === 'workflow-has-no-nodes',
+	failComment: 'Workflow has no nodes',
+});
+
+export const hasTrigger = createValidationCheck({
+	name: 'has_trigger',
+	validate: validateTrigger,
+	filter: (v) => v.name === 'workflow-has-no-trigger',
+	failComment: 'Workflow has no trigger node',
+});
+
+export const agentHasDynamicPrompt = createValidationCheck({
+	name: 'agent_has_dynamic_prompt',
+	validate: (workflow) => validateAgentPrompt(workflow),
+	filter: (v) => v.name === 'agent-static-prompt',
+});
+
+export const agentHasLanguageModel = createValidationCheck({
+	name: 'agent_has_language_model',
+	validate: validateConnections,
+	filter: (v) =>
+		v.name === 'node-missing-required-input' &&
+		v.metadata?.missingType === 'ai_languageModel' &&
+		(v.metadata?.nodeType?.includes('langchain.agent') ?? false),
+	failComment: 'Agent node missing required language model connection',
+});
+
+export const memoryProperlyConnected = createValidationCheck({
+	name: 'memory_properly_connected',
+	validate: validateConnections,
+	filter: (v) => v.name === 'sub-node-not-connected' && v.metadata?.outputType === 'ai_memory',
+	failComment: 'Memory node not properly connected to parent node',
+});
+
+export const vectorStoreHasEmbeddings = createValidationCheck({
+	name: 'vector_store_has_embeddings',
+	validate: validateConnections,
+	filter: (v) =>
+		v.name === 'node-missing-required-input' &&
+		v.metadata?.missingType === 'ai_embedding' &&
+		(v.metadata?.nodeType?.includes('vectorStore') ?? false),
+	failComment: 'Vector store node missing required embeddings connection',
+});
+
+export const noHardcodedCredentials = createValidationCheck({
+	name: 'no_hardcoded_credentials',
+	validate: (workflow) => validateCredentials(workflow),
+});
+
+export const validRequiredParameters = createValidationCheck({
+	name: 'valid_required_parameters',
+	validate: validateParameters,
+	filter: (v) => v.name === 'node-missing-required-parameter',
+});
+
+export const validOptionsValues = createValidationCheck({
+	name: 'valid_options_values',
+	validate: validateParameters,
+	filter: (v) => v.name === 'node-invalid-options-value',
+});
+
+export const noInvalidFromAi = createValidationCheck({
+	name: 'no_invalid_from_ai',
+	validate: validateFromAi,
+});
+
+export const toolsHaveParameters = createValidationCheck({
+	name: 'tools_have_parameters',
+	validate: validateTools,
+});