first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/responder/index.ts

@@ -0,0 +1,196 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { HumanMessage } from '@langchain/core/messages';
+
+import type { ResponderEvalCriteria } from './responder-judge.prompt';
+import { buildResponderJudgePrompt } from './responder-judge.prompt';
+import { runWithOptionalLimiter, withTimeout } from '../../harness/evaluation-helpers';
+import type { EvaluationContext, Evaluator, Feedback } from '../../harness/harness-types';
+import { DEFAULTS } from '../../support/constants';
+
+const EVALUATOR_NAME = 'responder-judge';
+
+export interface ResponderEvaluatorOptions {
+	/** Number of judges to run in parallel (default: DEFAULTS.NUM_JUDGES) */
+	numJudges?: number;
+}
+
+interface ResponderJudgeDimension {
+	score: number;
+	comment: string;
+}
+
+interface ResponderJudgeResult {
+	relevance: ResponderJudgeDimension;
+	accuracy: ResponderJudgeDimension;
+	completeness: ResponderJudgeDimension;
+	clarity: ResponderJudgeDimension;
+	tone: ResponderJudgeDimension;
+	criteriaMatch: ResponderJudgeDimension;
+	forbiddenPhrases: ResponderJudgeDimension;
+	overallScore: number;
+	summary: string;
+}
+
+/**
+ * Context for responder evaluation, extends standard EvaluationContext
+ * with the responder output and per-example criteria.
+ */
+export interface ResponderEvaluationContext extends EvaluationContext {
+	/** The text output from the responder agent */
+	responderOutput: string;
+	/** Per-example evaluation criteria from the dataset */
+	responderEvals: ResponderEvalCriteria;
+	/** The actual workflow JSON for accuracy verification */
+	workflowJSON?: unknown;
+}
+
+function isResponderContext(ctx: EvaluationContext): ctx is ResponderEvaluationContext {
+	return (
+		'responderOutput' in ctx &&
+		typeof (ctx as ResponderEvaluationContext).responderOutput === 'string' &&
+		'responderEvals' in ctx &&
+		typeof (ctx as ResponderEvaluationContext).responderEvals === 'object'
+	);
+}
+
+function parseJudgeResponse(content: string): ResponderJudgeResult {
+	// Extract JSON from markdown code block if present
+	const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
+	const jsonStr = jsonMatch ? jsonMatch[1].trim() : content.trim();
+	try {
+		return JSON.parse(jsonStr) as ResponderJudgeResult;
+	} catch {
+		throw new Error(`Failed to parse judge response as JSON: ${jsonStr.slice(0, 100)}...`);
+	}
+}
+
+const DIMENSION_KEYS = [
+	'relevance',
+	'accuracy',
+	'completeness',
+	'clarity',
+	'tone',
+	'criteriaMatch',
+	'forbiddenPhrases',
+] as const;
+
+const fb = (metric: string, score: number, kind: Feedback['kind'], comment?: string): Feedback => ({
+	evaluator: EVALUATOR_NAME,
+	metric,
+	score,
+	kind,
+	...(comment ? { comment } : {}),
+});
+
+/** Run a single judge invocation and return the parsed result. */
+async function runSingleJudge(
+	llm: BaseChatModel,
+	ctx: ResponderEvaluationContext,
+	judgeIndex: number,
+): Promise<ResponderJudgeResult> {
+	const judgePrompt = buildResponderJudgePrompt({
+		userPrompt: ctx.prompt,
+		responderOutput: ctx.responderOutput,
+		evalCriteria: ctx.responderEvals,
+		workflowJSON: ctx.workflowJSON,
+	});
+
+	return await runWithOptionalLimiter(async () => {
+		const response = await withTimeout({
+			promise: llm.invoke([new HumanMessage(judgePrompt)], {
+				runName: `responder_judge_${judgeIndex + 1}`,
+			}),
+			timeoutMs: ctx.timeoutMs,
+			label: `responder-judge:evaluate:judge_${judgeIndex + 1}`,
+		});
+
+		const content =
+			typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
+
+		return parseJudgeResponse(content);
+	}, ctx.llmCallLimiter);
+}
+
+/** Aggregate results from multiple judges into feedback items. */
+function aggregateResults(results: ResponderJudgeResult[], numJudges: number): Feedback[] {
+	const feedback: Feedback[] = [];
+
+	// Per-dimension averaged metrics
+	for (const key of DIMENSION_KEYS) {
+		const avgScore =
+			results.reduce((sum, r) => {
+				const dimension = r[key];
+				return sum + (dimension?.score ?? 0);
+			}, 0) / numJudges;
+		const comments = results
+			.map((r, i) => {
+				const dimension = r[key];
+				return `[Judge ${i + 1}] ${dimension?.comment ?? 'No comment'}`;
+			})
+			.join(' | ');
+		feedback.push(fb(key, avgScore, 'metric', comments));
+	}
+
+	// Aggregated overall score
+	const avgOverall = results.reduce((sum, r) => sum + r.overallScore, 0) / numJudges;
+	feedback.push(
+		fb(
+			'overallScore',
+			avgOverall,
+			'score',
+			`${numJudges}/${numJudges} judges averaged ${avgOverall.toFixed(2)}`,
+		),
+	);
+
+	// Per-judge detail items
+	for (let i = 0; i < results.length; i++) {
+		const r = results[i];
+		feedback.push(fb(`judge${i + 1}`, r.overallScore, 'detail', `Judge ${i + 1}: ${r.summary}`));
+	}
+
+	return feedback;
+}
+
+/**
+ * Create a responder LLM-judge evaluator.
+ *
+ * Uses an LLM to evaluate responder output against per-example criteria
+ * from the dataset. The evaluator expects a ResponderEvaluationContext
+ * with `responderOutput` and `responderEvals` fields.
+ *
+ * When `numJudges > 1`, runs multiple judge calls in parallel and aggregates
+ * dimension scores (averaged) and per-judge detail feedback.
+ *
+ * @param llm - The LLM to use for judging
+ * @param options - Optional configuration (e.g. numJudges)
+ * @returns An evaluator that produces feedback for responder output
+ */
+export function createResponderEvaluator(
+	llm: BaseChatModel,
+	options?: ResponderEvaluatorOptions,
+): Evaluator<EvaluationContext> {
+	const numJudges = options?.numJudges ?? DEFAULTS.NUM_JUDGES;
+
+	return {
+		name: EVALUATOR_NAME,
+
+		async evaluate(_workflow, ctx: EvaluationContext): Promise<Feedback[]> {
+			if (!isResponderContext(ctx)) {
+				return [
+					fb(
+						'error',
+						0,
+						'score',
+						'Missing responderOutput or responderEvals in evaluation context',
+					),
+				];
+			}
+
+			const results = await Promise.all(
+				Array.from({ length: numJudges }, async (_, i) => await runSingleJudge(llm, ctx, i)),
+			);
+
+			return aggregateResults(results, numJudges);
+		},
+	};
+}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/responder/responder-judge.prompt.ts

@@ -0,0 +1,131 @@
+import { prompt } from '@/prompts/builder';
+
+/**
+ * Responder evaluation types that map to different evaluation strategies.
+ *
+ * Currently all responses happen after a full workflow generation.
+ * Plan mode types can be added later when that feature is implemented.
+ */
+export type ResponderEvalType = 'workflow_summary' | 'datatable_instructions' | 'general_response';
+
+export interface ResponderEvalCriteria {
+	type: ResponderEvalType;
+	criteria: string;
+}
+
+const FORBIDDEN_PHRASES = [
+	'activate workflow',
+	'activate the workflow',
+	'click the activate button',
+];
+
+function buildForbiddenPhrasesSection(): string {
+	return FORBIDDEN_PHRASES.map((p) => `- "${p}"`).join('\n');
+}
+
+function buildTypeSpecificGuidance(evalType: ResponderEvalType): string {
+	switch (evalType) {
+		case 'workflow_summary':
+			return `
+				Additionally evaluate:
+				- Does the response accurately describe the workflow that was built?
+				- Are all key nodes and their purposes mentioned?
+				- Is the explanation of the workflow flow logical and complete?
+				- Does it explain how the workflow addresses the user request?
+				- Are setup instructions (credentials, placeholders) clearly provided?
+			`;
+
+		case 'datatable_instructions':
+			return `
+				Additionally evaluate:
+				- Are the data table creation instructions clear and actionable?
+				- Do the column names/types match what the workflow expects?
+				- Is the user told exactly what to create manually?
+			`;
+
+		case 'general_response':
+			return '';
+	}
+}
+
+function buildWorkflowSummary(workflowJSON: unknown): string {
+	if (!workflowJSON || typeof workflowJSON !== 'object') {
+		return 'No workflow data available';
+	}
+
+	const workflow = workflowJSON as { nodes?: Array<{ name?: string; type?: string }> };
+	if (!Array.isArray(workflow.nodes) || workflow.nodes.length === 0) {
+		return 'Empty workflow (no nodes)';
+	}
+
+	const nodeList = workflow.nodes
+		.map((node: { name?: string; type?: string }) => {
+			const name = node.name ?? 'unnamed';
+			const type = node.type ?? 'unknown';
+			return `- ${name} (${type})`;
+		})
+		.join('\n');
+
+	return `Workflow contains ${workflow.nodes.length} nodes:\n${nodeList}`;
+}
+
+/**
+ * Build the LLM judge prompt for evaluating a responder output.
+ */
+export function buildResponderJudgePrompt(args: {
+	userPrompt: string;
+	responderOutput: string;
+	evalCriteria: ResponderEvalCriteria;
+	workflowJSON?: unknown;
+}): string {
+	const { userPrompt, responderOutput, evalCriteria, workflowJSON } = args;
+	const typeGuidance = buildTypeSpecificGuidance(evalCriteria.type);
+	const hasWorkflow = workflowJSON !== undefined;
+
+	return prompt()
+		.section(
+			'role',
+			'You are an expert evaluator assessing the quality of an AI assistant response in a workflow automation context.',
+		)
+		.section(
+			'task',
+			`
+				Evaluate the responder output against the provided criteria.
+				Score each dimension from 0.0 to 1.0.
+
+				Return your evaluation as JSON with this exact structure:
+				\`\`\`json
+				{
+				  "relevance": { "score": 0.0, "comment": "..." },
+				  "accuracy": { "score": 0.0, "comment": "..." },',
+				  "completeness": { "score": 0.0, "comment": "..." },
+				  "clarity": { "score": 0.0, "comment": "..." },
+				  "tone": { "score": 0.0, "comment": "..." },',
+				  "criteriaMatch": { "score": 0.0, "comment": "..." },
+				  "forbiddenPhrases": { "score": 0.0, "comment": "..." },
+				  "overallScore": 0.0,',
+				  "summary": "..."
+				}
+				\`\`\`
+				`,
+		)
+		.section(
+			'dimensions',
+			`
+				**relevance** (0-1): Does the response address the user request?'
+				**accuracy** (0-1): Is the information factually correct? If a workflow is provided, verify that the responder's claims about the workflow (nodes, integrations, actions) match what was actually built."
+				**completeness** (0-1): Does it cover everything needed?
+				**clarity** (0-1): Is the response well-structured and easy to understand?
+				**tone** (0-1): Is the tone professional and helpful?',
+				**criteriaMatch** (0-1): Does it satisfy the specific evaluation criteria below?
+				**forbiddenPhrases** (0-1): 1.0 if no forbidden phrases are present, 0.0 if any are found.
+				`,
+		)
+		.section('forbiddenPhrases', buildForbiddenPhrasesSection())
+		.section('userPrompt', userPrompt)
+		.section('responderOutput', responderOutput)
+		.sectionIf(hasWorkflow, 'actualWorkflow', () => buildWorkflowSummary(workflowJSON))
+		.section('evaluationCriteria', evalCriteria.criteria)
+		.sectionIf(typeGuidance.length > 0, 'typeSpecificGuidance', typeGuidance)
+		.build();
+}