first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/index.ts

@@ -0,0 +1,124 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { runJudgePanel, type EvalCriteria } from './judge-panel';
+import { PAIRWISE_METRICS } from './metrics';
+import type {
+	DisplayLine,
+	EvaluationContext,
+	Evaluator,
+	Feedback,
+} from '../../harness/harness-types';
+
+/**
+ * Options for creating a pairwise evaluator.
+ */
+export interface PairwiseEvaluatorOptions {
+	/** Number of judges to run (default: 3) */
+	numJudges?: number;
+}
+
+/**
+ * Create a pairwise evaluator that uses a panel of judges.
+ * Each judge evaluates the workflow against dos/donts criteria.
+ *
+ * @param llm - Language model for evaluation
+ * @param options - Configuration options
+ * @returns An evaluator that produces feedback from pairwise evaluation
+ *
+ * @example
+ * ```typescript
+ * const evaluator = createPairwiseEvaluator(llm, { numJudges: 3 });
+ * const feedback = await evaluator.evaluate(workflow, { dos, donts });
+ * ```
+ */
+export function createPairwiseEvaluator(
+	llm: BaseChatModel,
+	options?: PairwiseEvaluatorOptions,
+): Evaluator<EvaluationContext> {
+	const numJudges = options?.numJudges ?? 3;
+
+	return {
+		name: 'pairwise',
+
+		async evaluate(workflow: SimpleWorkflow, ctx: EvaluationContext): Promise<Feedback[]> {
+			const evalCriteria: EvalCriteria = {
+				dos: ctx?.dos,
+				donts: ctx?.donts,
+			};
+
+			const result = await runJudgePanel(llm, workflow, evalCriteria, numJudges, {
+				llmCallLimiter: ctx.llmCallLimiter,
+				timeoutMs: ctx.timeoutMs,
+			});
+
+			const feedback: Feedback[] = [];
+
+			const totalViolations = result.judgeResults.reduce((sum, r) => sum + r.violations.length, 0);
+			const totalPasses = result.judgeResults.reduce((sum, r) => sum + r.passes.length, 0);
+
+			// Primary metrics
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_PRIMARY,
+				score: result.majorityPass ? 1 : 0,
+				kind: 'score',
+				comment: `${result.primaryPasses}/${numJudges} judges passed`,
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC,
+				score: result.avgDiagnosticScore,
+				kind: 'metric',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_JUDGES_PASSED,
+				score: result.primaryPasses,
+				kind: 'detail',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_PASSES,
+				score: totalPasses,
+				kind: 'detail',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_VIOLATIONS,
+				score: totalViolations,
+				kind: 'detail',
+			});
+
+			// Individual judge results
+			for (let i = 0; i < result.judgeResults.length; i++) {
+				const judge = result.judgeResults[i];
+				const violationSummary =
+					judge.violations.length > 0
+						? judge.violations.map((v) => `[${v.rule}] ${v.justification}`).join('; ')
+						: undefined;
+
+				// Pre-format display lines for verbose logging
+				const displayLines: DisplayLine[] = [];
+				if (judge.violations.length > 0) {
+					for (const v of judge.violations) {
+						displayLines.push({ text: `[${v.rule}]`, color: 'yellow' });
+						displayLines.push({ text: v.justification, color: 'red' });
+					}
+				}
+
+				feedback.push({
+					evaluator: 'pairwise',
+					metric: `judge${i + 1}`,
+					score: judge.primaryPass ? 1 : 0,
+					kind: 'detail',
+					comment: violationSummary,
+					details: displayLines.length > 0 ? { displayLines } : undefined,
+				});
+			}
+
+			return feedback;
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-chain.test.ts

@@ -0,0 +1,97 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { evaluateWorkflowPairwise, type PairwiseEvaluationInput } from './judge-chain';
+import * as baseEvaluator from '../llm-judge/evaluators/base';
+
+// Mock the base evaluator module
+jest.mock('../llm-judge/evaluators/base', () => ({
+	createEvaluatorChain: jest.fn(),
+	invokeEvaluatorChain: jest.fn(),
+}));
+
+describe('evaluateWorkflowPairwise', () => {
+	const mockLlm = mock<BaseChatModel>();
+
+	const mockWorkflow: SimpleWorkflow = {
+		nodes: [],
+		connections: {},
+		name: 'Test Workflow',
+	};
+
+	const input: PairwiseEvaluationInput = {
+		evalCriteria: {
+			dos: 'Do this',
+			donts: "Don't do that",
+		},
+		workflowJSON: mockWorkflow,
+	};
+
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	it('should return structured result from invokeEvaluatorChain', async () => {
+		const mockResult = {
+			violations: [],
+			passes: [
+				{ rule: 'Do this', justification: 'Done' },
+				{ rule: "Don't do that", justification: 'Not done' },
+			],
+		};
+
+		jest.mocked(baseEvaluator.invokeEvaluatorChain).mockResolvedValue(mockResult);
+
+		const result = await evaluateWorkflowPairwise(mockLlm, input);
+
+		expect(result).toEqual({
+			...mockResult,
+			primaryPass: true,
+			diagnosticScore: 1,
+		});
+		expect(baseEvaluator.createEvaluatorChain).toHaveBeenCalledWith(
+			mockLlm,
+			expect.anything(), // schema
+			expect.stringContaining('expert n8n workflow auditor'), // system prompt
+			expect.stringContaining('<task_context>'), // human template
+		);
+		expect(baseEvaluator.invokeEvaluatorChain).toHaveBeenCalledWith(
+			undefined, // The chain (undefined because createEvaluatorChain mock returns undefined)
+			expect.objectContaining({
+				userPrompt: expect.stringContaining('<do>'),
+				generatedWorkflow: input.workflowJSON,
+			}),
+			undefined, // config parameter (not passed in this test)
+		);
+	});
+
+	it('should calculate diagnosticScore correctly with violations', async () => {
+		const mockResult = {
+			violations: [{ rule: "Don't do that", justification: 'Did it' }],
+			passes: [{ rule: 'Do this', justification: 'Done' }],
+		};
+
+		jest.mocked(baseEvaluator.invokeEvaluatorChain).mockResolvedValue(mockResult);
+
+		const result = await evaluateWorkflowPairwise(mockLlm, input);
+
+		expect(result.primaryPass).toBe(false);
+		expect(result.diagnosticScore).toBe(0.5);
+	});
+
+	it('should return diagnosticScore 0 when no rules evaluated', async () => {
+		const mockResult = {
+			violations: [],
+			passes: [],
+		};
+
+		jest.mocked(baseEvaluator.invokeEvaluatorChain).mockResolvedValue(mockResult);
+
+		const result = await evaluateWorkflowPairwise(mockLlm, input);
+
+		expect(result.primaryPass).toBe(true);
+		expect(result.diagnosticScore).toBe(0);
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-chain.ts

@@ -0,0 +1,156 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import { z } from 'zod';
+
+import type { EvalCriteria } from './judge-panel';
+import { prompt } from '../../../src/prompts/builder';
+import type { SimpleWorkflow } from '../../../src/types/workflow';
+import { createEvaluatorChain, invokeEvaluatorChain } from '../llm-judge/evaluators/base';
+
+export interface PairwiseEvaluationInput {
+	evalCriteria: EvalCriteria;
+	workflowJSON: SimpleWorkflow;
+}
+
+const pairwiseEvaluationLLMResultSchema = z.object({
+	violations: z
+		.array(
+			z.object({
+				rule: z.string(),
+				justification: z.string(),
+			}),
+		)
+		.describe(
+			'List of criteria that were violated, this must be passed as a JSON array not a string.',
+		),
+	passes: z
+		.array(
+			z.object({
+				rule: z.string(),
+				justification: z.string(),
+			}),
+		)
+		.describe('The criterion that was passed, this must be passed as a JSON array not a string.'),
+});
+
+export type PairwiseEvaluationResult = z.infer<typeof pairwiseEvaluationLLMResultSchema> & {
+	/** True only if ALL criteria passed (no violations) */
+	primaryPass: boolean;
+	/** Ratio of passed criteria to total criteria (0-1) */
+	diagnosticScore: number;
+};
+
+const EVALUATOR_SYSTEM_PROMPT = prompt()
+	.section(
+		'role',
+		'You are an expert n8n workflow auditor. Your task is to strictly evaluate a candidate workflow against a provided set of requirements.',
+	)
+	.section(
+		'role_definition',
+		`- You are objective, precise, and evidence-based.
+- You do not assume functionality that is not explicitly configured in the JSON.
+- You verify every claim against the actual node configurations, connections, and parameters.`,
+	)
+	.section(
+		'clarifications',
+		`When evaluating criteria about "provider-specific nodes" or "using a specific AI provider":
+
+- Provider-specific nodes (e.g., n8n-nodes-langchain.openAi, n8n-nodes-langchain.anthropic) are standalone nodes that directly call a provider's API.
+- Chat model sub-nodes (e.g., @n8n/n8n-nodes-langchain.lmChatAnthropic, @n8n/n8n-nodes-langchain.lmChatOpenAi) are NOT provider-specific nodes. They are required infrastructure for connecting generic nodes like the AI Agent to a language model.
+
+If a criterion says "do not use provider-specific nodes" or similar, the presence of lmChat* sub-nodes should NOT count as a violation - these are necessary connectors, not provider-specific workflow nodes.
+
+When evaluating whether a specific node type has been used:
+- The "@n8n/" prefix in node types is OPTIONAL - ignore it when comparing
+- "@n8n/n8n-nodes-langchain.chatTrigger" and "n8n-nodes-langchain.chatTrigger" are the SAME node type
+- This applies regardless of which form appears in the criteria or the workflow`,
+	)
+	.section(
+		'constraints',
+		`- Judge ONLY against the provided evaluation criteria. Do not apply external "best practices" unless explicitly asked.
+- If a criterion is "not verifiable" from the JSON alone (e.g., requires runtime data), mark it as a violation and explain why.
+- For every pass or violation, you MUST cite the specific node name or parameter that serves as evidence.
+- Do not hallucinate nodes or parameters.`,
+	)
+	.build();
+
+const humanTemplate = prompt()
+	.section(
+		'task_context',
+		'Analyze the following n8n workflow against the provided checklist of criteria.',
+	)
+	.section('evaluation_criteria', '{userPrompt}')
+	.section('workflow_candidate', '{generatedWorkflow}')
+	.section(
+		'instructions',
+		`1. Read the <evaluation_criteria> carefully. It contains <do> and <dont> criteria.
+2. For each criterion:
+    - Search for evidence in the <workflow_candidate>.
+    - Classify as PASS or VIOLATION using the rules below.
+    - Provide a clear 'justification' citing the evidence (e.g., "Node 'HTTP Request' has method set to 'GET'").
+3. Output the result as a structured JSON with 'violations' and 'passes'.`,
+	)
+	.section(
+		'classification_rules',
+		`CRITICAL: Understand how to classify each criterion correctly:
+
+For <do> criteria (positive requirements like "Use X" or "Include Y"):
+- PASS: The required element IS present in the workflow
+- VIOLATION: The required element is NOT present in the workflow
+
+For <dont> criteria (anti-patterns to avoid):
+- PASS: The forbidden element is NOT present (the anti-pattern was avoided)
+- VIOLATION: The forbidden element IS present (the anti-pattern was used)
+
+Example: <dont>Use code node to organize data</dont>
+- If NO code node exists for organizing data → PASS (anti-pattern avoided)
+- If a code node IS used for organizing data → VIOLATION (anti-pattern present)`,
+	)
+	.build();
+
+export async function evaluateWorkflowPairwise(
+	llm: BaseChatModel,
+	input: PairwiseEvaluationInput,
+	config?: RunnableConfig,
+): Promise<PairwiseEvaluationResult> {
+	const dos = input.evalCriteria?.dos ?? '';
+	const donts = input.evalCriteria?.donts ?? '';
+
+	const doLines = dos.split('\n').filter((line) => line.trim().length > 0);
+	const dontLines = donts.split('\n').filter((line) => line.trim().length > 0);
+
+	const criteriaBuilder = prompt({ format: 'xml' });
+	for (const line of doLines) {
+		criteriaBuilder.section('do', line.trim());
+	}
+	for (const line of dontLines) {
+		criteriaBuilder.section('dont', line.trim());
+	}
+	const criteriaList = criteriaBuilder.build();
+
+	const chain = createEvaluatorChain(
+		llm,
+		pairwiseEvaluationLLMResultSchema,
+		EVALUATOR_SYSTEM_PROMPT,
+		humanTemplate,
+	);
+
+	const result = await invokeEvaluatorChain(
+		chain,
+		{
+			userPrompt: criteriaList,
+			generatedWorkflow: input.workflowJSON,
+		},
+		config,
+	);
+
+	const totalRules = result.passes.length + result.violations.length;
+	const diagnosticScore = totalRules > 0 ? result.passes.length / totalRules : 0;
+	const primaryPass = result.violations.length === 0;
+
+	return {
+		...result,
+		primaryPass,
+		diagnosticScore,
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-panel.test.ts

@@ -0,0 +1,46 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { mock } from 'jest-mock-extended';
+import pLimit from 'p-limit';
+
+import type { SimpleWorkflow } from '@/types/workflow';
+
+import { runJudgePanel } from './judge-panel';
+
+const mockEvaluateWorkflowPairwise = jest.fn();
+
+jest.mock('./judge-chain', () => ({
+	evaluateWorkflowPairwise: (...args: unknown[]): unknown => mockEvaluateWorkflowPairwise(...args),
+}));
+
+function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
+	return { name, nodes: [], connections: {} };
+}
+
+describe('runJudgePanel()', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	it('should respect llmCallLimiter concurrency', async () => {
+		let active = 0;
+		let maxActive = 0;
+
+		mockEvaluateWorkflowPairwise.mockImplementation(async () => {
+			active++;
+			maxActive = Math.max(maxActive, active);
+			await new Promise((r) => setTimeout(r, 20));
+			active--;
+			return { violations: [], passes: [], primaryPass: true, diagnosticScore: 1 };
+		});
+
+		const llm = mock<BaseChatModel>();
+		const workflow = createMockWorkflow();
+
+		await runJudgePanel(llm, workflow, { dos: 'Do X', donts: 'Do not Y' }, 5, {
+			llmCallLimiter: pLimit(2),
+		});
+
+		expect(maxActive).toBeLessThanOrEqual(2);
+		expect(mockEvaluateWorkflowPairwise).toHaveBeenCalledTimes(5);
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-panel.ts

@@ -0,0 +1,151 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { RunnableConfig } from '@langchain/core/runnables';
+
+import { evaluateWorkflowPairwise, type PairwiseEvaluationResult } from './judge-chain';
+import type { SimpleWorkflow } from '../../../src/types/workflow';
+import {
+	getTracingCallbacks,
+	runWithOptionalLimiter,
+	withTimeout,
+} from '../../harness/evaluation-helpers';
+import type { EvaluationContext } from '../../harness/harness-types';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/** Evaluation criteria - at least one of dos or donts should be provided */
+export interface EvalCriteria {
+	dos?: string;
+	donts?: string;
+}
+
+export interface JudgePanelTiming {
+	/** Total time for all judges in milliseconds */
+	totalMs: number;
+	/** Time per judge in milliseconds */
+	perJudgeMs: number[];
+}
+
+export interface JudgePanelResult {
+	judgeResults: PairwiseEvaluationResult[];
+	primaryPasses: number;
+	majorityPass: boolean;
+	avgDiagnosticScore: number;
+	/** Timing information (only populated when timing is tracked) */
+	timing?: JudgePanelTiming;
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * Calculate minimum judges needed for majority (e.g., 2 for 3 judges, 3 for 5 judges)
+ * @param numJudges - Number of judges (must be >= 1)
+ * @throws Error if numJudges < 1
+ */
+export function getMajorityThreshold(numJudges: number): number {
+	if (numJudges < 1) {
+		throw new Error(`getMajorityThreshold requires numJudges >= 1, got ${numJudges}`);
+	}
+	return Math.ceil(numJudges / 2);
+}
+
+// ============================================================================
+// Judge Panel Execution
+// ============================================================================
+
+export interface JudgePanelOptions {
+	/** Experiment name for metadata */
+	experimentName?: string;
+	/** Optional limiter for LLM calls (shared across harness) */
+	llmCallLimiter?: EvaluationContext['llmCallLimiter'];
+	/** Optional timeout for each judge call */
+	timeoutMs?: number;
+}
+
+/**
+ * Run a panel of judges on a workflow.
+ * Executes judges in parallel and aggregates their results.
+ *
+ * @param llm - Language model for evaluation
+ * @param workflow - Workflow to evaluate
+ * @param evalCriteria - Evaluation criteria (dos/donts)
+ * @param numJudges - Number of judges to run
+ * @param options - Optional metadata for tracing
+ * @returns Aggregated judge panel results
+ */
+export async function runJudgePanel(
+	llm: BaseChatModel,
+	workflow: SimpleWorkflow,
+	evalCriteria: EvalCriteria,
+	numJudges: number,
+	options?: JudgePanelOptions,
+): Promise<JudgePanelResult> {
+	const { experimentName, llmCallLimiter, timeoutMs } = options ?? {};
+	const panelStartTime = Date.now();
+
+	// Bridge LangSmith traceable context to LangChain callbacks
+	const callbacks = await getTracingCallbacks();
+
+	// Run all judges in parallel, tracking timing for each
+	const judgeTimings: number[] = [];
+	const judgeResults = await Promise.all(
+		Array.from({ length: numJudges }, async (_, judgeIndex) => {
+			const runJudge = async (): Promise<PairwiseEvaluationResult> => {
+				const judgeStartTime = Date.now();
+
+				// Build config with callbacks for proper trace context propagation
+				const config: RunnableConfig = {
+					runName: `judge_${judgeIndex + 1}`,
+					metadata: {
+						...(experimentName && { experiment_name: experimentName }),
+					},
+					callbacks,
+				};
+
+				const result = await withTimeout({
+					promise: evaluateWorkflowPairwise(llm, { workflowJSON: workflow, evalCriteria }, config),
+					timeoutMs,
+					label: `pairwise:judge${judgeIndex + 1}`,
+				});
+				judgeTimings[judgeIndex] = Date.now() - judgeStartTime;
+				return result;
+			};
+
+			return await runWithOptionalLimiter(runJudge, llmCallLimiter);
+		}),
+	);
+
+	const totalMs = Date.now() - panelStartTime;
+	const aggregated = aggregateJudgeResults(judgeResults, numJudges);
+
+	return {
+		...aggregated,
+		timing: {
+			totalMs,
+			perJudgeMs: judgeTimings,
+		},
+	};
+}
+
+/**
+ * Aggregate results from multiple judges into summary metrics.
+ */
+export function aggregateJudgeResults(
+	judgeResults: PairwiseEvaluationResult[],
+	numJudges: number,
+): JudgePanelResult {
+	const primaryPasses = judgeResults.filter((r) => r.primaryPass).length;
+	const majorityPass = primaryPasses >= getMajorityThreshold(numJudges);
+	const avgDiagnosticScore =
+		judgeResults.reduce((sum, r) => sum + r.diagnosticScore, 0) / numJudges;
+
+	return {
+		judgeResults,
+		primaryPasses,
+		majorityPass,
+		avgDiagnosticScore,
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/metrics.ts

@@ -0,0 +1,7 @@
+export const PAIRWISE_METRICS = {
+	PAIRWISE_DIAGNOSTIC: 'pairwise_diagnostic',
+	PAIRWISE_JUDGES_PASSED: 'pairwise_judges_passed',
+	PAIRWISE_PRIMARY: 'pairwise_primary',
+	PAIRWISE_TOTAL_PASSES: 'pairwise_total_passes',
+	PAIRWISE_TOTAL_VIOLATIONS: 'pairwise_total_violations',
+} as const;