first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-chain.ts

@@ -0,0 +1,156 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import { z } from 'zod';
+
+import type { EvalCriteria } from './judge-panel';
+import { prompt } from '../../../src/prompts/builder';
+import type { SimpleWorkflow } from '../../../src/types/workflow';
+import { createEvaluatorChain, invokeEvaluatorChain } from '../llm-judge/evaluators/base';
+
+export interface PairwiseEvaluationInput {
+	evalCriteria: EvalCriteria;
+	workflowJSON: SimpleWorkflow;
+}
+
+const pairwiseEvaluationLLMResultSchema = z.object({
+	violations: z
+		.array(
+			z.object({
+				rule: z.string(),
+				justification: z.string(),
+			}),
+		)
+		.describe(
+			'List of criteria that were violated, this must be passed as a JSON array not a string.',
+		),
+	passes: z
+		.array(
+			z.object({
+				rule: z.string(),
+				justification: z.string(),
+			}),
+		)
+		.describe('The criterion that was passed, this must be passed as a JSON array not a string.'),
+});
+
+export type PairwiseEvaluationResult = z.infer<typeof pairwiseEvaluationLLMResultSchema> & {
+	/** True only if ALL criteria passed (no violations) */
+	primaryPass: boolean;
+	/** Ratio of passed criteria to total criteria (0-1) */
+	diagnosticScore: number;
+};
+
+const EVALUATOR_SYSTEM_PROMPT = prompt()
+	.section(
+		'role',
+		'You are an expert n8n workflow auditor. Your task is to strictly evaluate a candidate workflow against a provided set of requirements.',
+	)
+	.section(
+		'role_definition',
+		`- You are objective, precise, and evidence-based.
+- You do not assume functionality that is not explicitly configured in the JSON.
+- You verify every claim against the actual node configurations, connections, and parameters.`,
+	)
+	.section(
+		'clarifications',
+		`When evaluating criteria about "provider-specific nodes" or "using a specific AI provider":
+
+- Provider-specific nodes (e.g., n8n-nodes-langchain.openAi, n8n-nodes-langchain.anthropic) are standalone nodes that directly call a provider's API.
+- Chat model sub-nodes (e.g., @n8n/n8n-nodes-langchain.lmChatAnthropic, @n8n/n8n-nodes-langchain.lmChatOpenAi) are NOT provider-specific nodes. They are required infrastructure for connecting generic nodes like the AI Agent to a language model.
+
+If a criterion says "do not use provider-specific nodes" or similar, the presence of lmChat* sub-nodes should NOT count as a violation - these are necessary connectors, not provider-specific workflow nodes.
+
+When evaluating whether a specific node type has been used:
+- The "@n8n/" prefix in node types is OPTIONAL - ignore it when comparing
+- "@n8n/n8n-nodes-langchain.chatTrigger" and "n8n-nodes-langchain.chatTrigger" are the SAME node type
+- This applies regardless of which form appears in the criteria or the workflow`,
+	)
+	.section(
+		'constraints',
+		`- Judge ONLY against the provided evaluation criteria. Do not apply external "best practices" unless explicitly asked.
+- If a criterion is "not verifiable" from the JSON alone (e.g., requires runtime data), mark it as a violation and explain why.
+- For every pass or violation, you MUST cite the specific node name or parameter that serves as evidence.
+- Do not hallucinate nodes or parameters.`,
+	)
+	.build();
+
+const humanTemplate = prompt()
+	.section(
+		'task_context',
+		'Analyze the following n8n workflow against the provided checklist of criteria.',
+	)
+	.section('evaluation_criteria', '{userPrompt}')
+	.section('workflow_candidate', '{generatedWorkflow}')
+	.section(
+		'instructions',
+		`1. Read the <evaluation_criteria> carefully. It contains <do> and <dont> criteria.
+2. For each criterion:
+    - Search for evidence in the <workflow_candidate>.
+    - Classify as PASS or VIOLATION using the rules below.
+    - Provide a clear 'justification' citing the evidence (e.g., "Node 'HTTP Request' has method set to 'GET'").
+3. Output the result as a structured JSON with 'violations' and 'passes'.`,
+	)
+	.section(
+		'classification_rules',
+		`CRITICAL: Understand how to classify each criterion correctly:
+
+For <do> criteria (positive requirements like "Use X" or "Include Y"):
+- PASS: The required element IS present in the workflow
+- VIOLATION: The required element is NOT present in the workflow
+
+For <dont> criteria (anti-patterns to avoid):
+- PASS: The forbidden element is NOT present (the anti-pattern was avoided)
+- VIOLATION: The forbidden element IS present (the anti-pattern was used)
+
+Example: <dont>Use code node to organize data</dont>
+- If NO code node exists for organizing data → PASS (anti-pattern avoided)
+- If a code node IS used for organizing data → VIOLATION (anti-pattern present)`,
+	)
+	.build();
+
+export async function evaluateWorkflowPairwise(
+	llm: BaseChatModel,
+	input: PairwiseEvaluationInput,
+	config?: RunnableConfig,
+): Promise<PairwiseEvaluationResult> {
+	const dos = input.evalCriteria?.dos ?? '';
+	const donts = input.evalCriteria?.donts ?? '';
+
+	const doLines = dos.split('\n').filter((line) => line.trim().length > 0);
+	const dontLines = donts.split('\n').filter((line) => line.trim().length > 0);
+
+	const criteriaBuilder = prompt({ format: 'xml' });
+	for (const line of doLines) {
+		criteriaBuilder.section('do', line.trim());
+	}
+	for (const line of dontLines) {
+		criteriaBuilder.section('dont', line.trim());
+	}
+	const criteriaList = criteriaBuilder.build();
+
+	const chain = createEvaluatorChain(
+		llm,
+		pairwiseEvaluationLLMResultSchema,
+		EVALUATOR_SYSTEM_PROMPT,
+		humanTemplate,
+	);
+
+	const result = await invokeEvaluatorChain(
+		chain,
+		{
+			userPrompt: criteriaList,
+			generatedWorkflow: input.workflowJSON,
+		},
+		config,
+	);
+
+	const totalRules = result.passes.length + result.violations.length;
+	const diagnosticScore = totalRules > 0 ? result.passes.length / totalRules : 0;
+	const primaryPass = result.violations.length === 0;
+
+	return {
+		...result,
+		primaryPass,
+		diagnosticScore,
+	};
+}