first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/__tests__/create-llm-check.test.ts

@@ -0,0 +1,21 @@
+import { createLlmCheck } from '../create-llm-check';
+
+describe('createLlmCheck', () => {
+	it('returns pass: true with "Skipped" comment when no LLM provided', async () => {
+		const check = createLlmCheck({
+			name: 'test_check',
+			systemPrompt: 'test',
+			humanTemplate: 'test {userPrompt} {generatedWorkflow} {referenceSection}',
+		});
+
+		expect(check.name).toBe('test_check');
+		expect(check.kind).toBe('llm');
+
+		const result = await check.run(
+			{ name: 'test', nodes: [], connections: {} },
+			{ prompt: 'test', nodeTypes: [] },
+		);
+		expect(result.pass).toBe(true);
+		expect(result.comment).toBe('Skipped: no LLM');
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/correct-node-operations.ts

@@ -0,0 +1,28 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const correctNodeOperations = createLlmCheck({
+	name: 'correct_node_operations',
+	systemPrompt: `You are an evaluator checking whether n8n workflow nodes use the correct resource and operation settings.
+
+For each node that has resource/operation parameters, verify:
+1. The resource matches what the node SHOULD operate on given its name and the workflow's purpose
+2. The operation matches the intended action (get, getAll, create, update, delete, etc.)
+3. Two nodes that should do different things are NOT configured identically
+
+Common mistakes to catch:
+- A node named "Get Captions" but configured with resource: "video" instead of resource: "caption"
+- A node that should create records but uses operation: "get"
+- Two nodes configured identically when they should fetch different resources
+
+Nodes without resource/operation parameters (triggers, Set, Merge, AI agents, LLM models) should be skipped.
+
+Respond with pass: true ONLY if all resource/operation combinations are correct for the workflow's intent.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Check each node's resource and operation parameters. Are they all correct for this workflow's purpose?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/create-llm-check.ts

@@ -0,0 +1,46 @@
+import { runWithOptionalLimiter, withTimeout } from '../../../harness/evaluation-helpers';
+import { createEvaluatorChain, invokeEvaluatorChain } from '../../llm-judge/evaluators/base';
+import type { BinaryCheck, BinaryCheckContext, SimpleWorkflow } from '../types';
+import { binaryJudgeResultSchema } from './schemas';
+
+const REASONING_FIRST_SUFFIX = `
+
+IMPORTANT: Write your full reasoning FIRST. Only AFTER completing your analysis, decide on pass or fail based on what you wrote. Do not decide pass/fail before reasoning.`;
+
+export function createLlmCheck(options: {
+	name: string;
+	systemPrompt: string;
+	humanTemplate: string;
+}): BinaryCheck {
+	const systemPrompt = options.systemPrompt + REASONING_FIRST_SUFFIX;
+
+	return {
+		name: options.name,
+		kind: 'llm',
+		async run(workflow: SimpleWorkflow, ctx: BinaryCheckContext) {
+			if (!ctx.llm) {
+				return { pass: true, comment: 'Skipped: no LLM' };
+			}
+
+			const chain = createEvaluatorChain(
+				ctx.llm,
+				binaryJudgeResultSchema,
+				systemPrompt,
+				options.humanTemplate,
+			);
+
+			const result = await runWithOptionalLimiter(async () => {
+				return await withTimeout({
+					promise: invokeEvaluatorChain(chain, {
+						userPrompt: ctx.prompt,
+						generatedWorkflow: workflow,
+					}),
+					timeoutMs: ctx.timeoutMs,
+					label: `binary-checks:${options.name}`,
+				});
+			}, ctx.llmCallLimiter);
+
+			return { pass: result.pass, comment: result.reasoning };
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/descriptive-node-names.ts

@@ -0,0 +1,24 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const descriptiveNodeNames = createLlmCheck({
+	name: 'descriptive_node_names',
+	systemPrompt: `You are an evaluator checking whether n8n workflow nodes have meaningful, descriptive names.
+Check:
+- Are node names descriptive of their purpose (e.g., "Send Welcome Email" vs "HTTP Request")?
+- Do names avoid default/generic names like "HTTP Request", "Code", "Set", "IF"?
+- Are names specific enough to understand the workflow at a glance?
+
+Note: Trigger nodes commonly keep their default names (e.g., "When clicking 'Test workflow'"), which is acceptable.
+A few default names in a simple workflow is acceptable; the check is about overall naming quality.
+
+Respond with pass: true if node names are generally descriptive and meaningful, false otherwise.
+Provide clear reasoning for your decision.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Do the nodes in this workflow have descriptive, meaningful names?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/fulfills-user-request.ts

@@ -0,0 +1,25 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const fulfillsUserRequest = createLlmCheck({
+	name: 'fulfills_user_request',
+	systemPrompt: `You are a strict evaluator checking whether an n8n workflow fulfills a user's request.
+
+For each feature the user explicitly asked for, check:
+1. Is there a node of the correct TYPE for that feature? (e.g., YouTube node for YouTube operations)
+2. Is that node configured with the correct RESOURCE and OPERATION? (e.g., resource: "caption" for fetching captions, not resource: "video")
+3. Is the node actually CONNECTED in the workflow flow?
+
+A node that exists but is misconfigured does NOT count as fulfilling the requirement.
+For example, a YouTube node with resource: "video" does NOT fulfill a request to "fetch captions" — captions require resource: "caption".
+
+Be binary: pass ONLY if every explicitly requested feature has a correctly-typed AND correctly-configured node.
+Do NOT pass just because a node with the right name exists — verify its actual parameters.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+For each feature the user requested, is there a correctly configured node? List each requirement and whether it's met.`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/handles-multiple-items.ts

@@ -0,0 +1,32 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const handlesMultipleItems = createLlmCheck({
+	name: 'handles_multiple_items',
+	systemPrompt: `You are an evaluator checking whether an n8n workflow handles multiple items correctly.
+
+Important n8n context:
+- Most n8n nodes automatically process ALL incoming items one by one — this is correct default behavior
+- A workflow designed for single-item processing (manual trigger + one record) does NOT need batch handling
+- Merge nodes with "combineByPosition" are correct for merging parallel single-item branches
+- AI Agent nodes process one item at a time, which is normal
+
+Only FAIL if there is a clear structural problem:
+- A node configured with multipleFiles/multiple inputs but no downstream handling for arrays
+- A splitInBatches that's clearly needed but missing (e.g., sending individual API calls for each item in a large list)
+- An aggregate node producing an array that downstream nodes don't handle
+
+Do NOT fail for:
+- Single-item workflows (manual trigger processing one record)
+- Workflows where n8n's automatic item-by-item processing is sufficient
+- Chatbot/agent workflows that process one message at a time
+
+Respond with pass: true if the workflow handles items correctly for its use case.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Does this workflow handle multiple items correctly for its intended use case?`,
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/index.ts

@@ -0,0 +1,14 @@
+import type { BinaryCheck } from '../types';
+import { correctNodeOperations } from './correct-node-operations';
+import { descriptiveNodeNames } from './descriptive-node-names';
+import { fulfillsUserRequest } from './fulfills-user-request';
+import { handlesMultipleItems } from './handles-multiple-items';
+import { validDataFlow } from './valid-data-flow';
+
+export const LLM_CHECKS: BinaryCheck[] = [
+	fulfillsUserRequest,
+	correctNodeOperations,
+	validDataFlow,
+	handlesMultipleItems,
+	descriptiveNodeNames,
+];

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/schemas.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod';
+
+export const binaryJudgeResultSchema = z.object({
+	reasoning: z
+		.string()
+		.describe('Step-by-step analysis. Write this FIRST, BEFORE deciding pass/fail.'),
+	pass: z
+		.boolean()
+		.describe(
+			'Final verdict derived from the reasoning above. true = all criteria met, false = at least one issue found.',
+		),
+});
+
+export type BinaryJudgeResult = z.infer<typeof binaryJudgeResultSchema>;

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/binary-checks/llm-checks/valid-data-flow.ts

@@ -0,0 +1,36 @@
+import { createLlmCheck } from './create-llm-check';
+
+export const validDataFlow = createLlmCheck({
+	name: 'valid_data_flow',
+	systemPrompt: `You are an evaluator checking whether expressions in an n8n workflow reference fields that actually exist upstream.
+
+For each expression in node parameters, check:
+1. \`{{ $json.fieldName }}\` — does the immediately upstream node output this field?
+2. \`$('NodeName').item.json.field\` — does that node exist, and does it output that field?
+3. Cross-references between nodes are consistent (field set in one node matches what's read in another)
+
+Important n8n context:
+- Manual Trigger and Schedule Trigger nodes output an empty object — they do NOT provide custom fields unless a Set node is placed after them
+- YouTube video.get returns \`snippet.title\`, \`snippet.description\`, etc. — NOT \`caption\` or \`transcript\`
+- Set nodes output exactly the fields defined in their assignments
+- AI Agent nodes output \`{ output: string }\`
+- Merge nodes combine fields from all inputs
+
+Focus on CRITICAL issues only:
+- Expressions referencing fields that clearly don't exist upstream (e.g., \`$json.transcript\` when no node produces a transcript field)
+- Expressions referencing nodes that don't exist in the workflow
+
+Do NOT fail for:
+- Minor field name case differences
+- Fields that might be available through n8n's built-in variables (\`$execution\`, \`$workflow\`, etc.)
+
+Respond with pass: true if there are no critical data flow issues.`,
+	humanTemplate: `User Request: {userPrompt}
+
+Generated Workflow:
+{generatedWorkflow}
+
+{referenceSection}
+
+Check each expression in the workflow. Do they reference fields that exist upstream?`,
+});