first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/responder/index.ts

@@ -0,0 +1,196 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { HumanMessage } from '@langchain/core/messages';
+
+import type { ResponderEvalCriteria } from './responder-judge.prompt';
+import { buildResponderJudgePrompt } from './responder-judge.prompt';
+import { runWithOptionalLimiter, withTimeout } from '../../harness/evaluation-helpers';
+import type { EvaluationContext, Evaluator, Feedback } from '../../harness/harness-types';
+import { DEFAULTS } from '../../support/constants';
+
+const EVALUATOR_NAME = 'responder-judge';
+
+export interface ResponderEvaluatorOptions {
+	/** Number of judges to run in parallel (default: DEFAULTS.NUM_JUDGES) */
+	numJudges?: number;
+}
+
+interface ResponderJudgeDimension {
+	score: number;
+	comment: string;
+}
+
+interface ResponderJudgeResult {
+	relevance: ResponderJudgeDimension;
+	accuracy: ResponderJudgeDimension;
+	completeness: ResponderJudgeDimension;
+	clarity: ResponderJudgeDimension;
+	tone: ResponderJudgeDimension;
+	criteriaMatch: ResponderJudgeDimension;
+	forbiddenPhrases: ResponderJudgeDimension;
+	overallScore: number;
+	summary: string;
+}
+
+/**
+ * Context for responder evaluation, extends standard EvaluationContext
+ * with the responder output and per-example criteria.
+ */
+export interface ResponderEvaluationContext extends EvaluationContext {
+	/** The text output from the responder agent */
+	responderOutput: string;
+	/** Per-example evaluation criteria from the dataset */
+	responderEvals: ResponderEvalCriteria;
+	/** The actual workflow JSON for accuracy verification */
+	workflowJSON?: unknown;
+}
+
+function isResponderContext(ctx: EvaluationContext): ctx is ResponderEvaluationContext {
+	return (
+		'responderOutput' in ctx &&
+		typeof (ctx as ResponderEvaluationContext).responderOutput === 'string' &&
+		'responderEvals' in ctx &&
+		typeof (ctx as ResponderEvaluationContext).responderEvals === 'object'
+	);
+}
+
+function parseJudgeResponse(content: string): ResponderJudgeResult {
+	// Extract JSON from markdown code block if present
+	const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
+	const jsonStr = jsonMatch ? jsonMatch[1].trim() : content.trim();
+	try {
+		return JSON.parse(jsonStr) as ResponderJudgeResult;
+	} catch {
+		throw new Error(`Failed to parse judge response as JSON: ${jsonStr.slice(0, 100)}...`);
+	}
+}
+
+const DIMENSION_KEYS = [
+	'relevance',
+	'accuracy',
+	'completeness',
+	'clarity',
+	'tone',
+	'criteriaMatch',
+	'forbiddenPhrases',
+] as const;
+
+const fb = (metric: string, score: number, kind: Feedback['kind'], comment?: string): Feedback => ({
+	evaluator: EVALUATOR_NAME,
+	metric,
+	score,
+	kind,
+	...(comment ? { comment } : {}),
+});
+
+/** Run a single judge invocation and return the parsed result. */
+async function runSingleJudge(
+	llm: BaseChatModel,
+	ctx: ResponderEvaluationContext,
+	judgeIndex: number,
+): Promise<ResponderJudgeResult> {
+	const judgePrompt = buildResponderJudgePrompt({
+		userPrompt: ctx.prompt,
+		responderOutput: ctx.responderOutput,
+		evalCriteria: ctx.responderEvals,
+		workflowJSON: ctx.workflowJSON,
+	});
+
+	return await runWithOptionalLimiter(async () => {
+		const response = await withTimeout({
+			promise: llm.invoke([new HumanMessage(judgePrompt)], {
+				runName: `responder_judge_${judgeIndex + 1}`,
+			}),
+			timeoutMs: ctx.timeoutMs,
+			label: `responder-judge:evaluate:judge_${judgeIndex + 1}`,
+		});
+
+		const content =
+			typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
+
+		return parseJudgeResponse(content);
+	}, ctx.llmCallLimiter);
+}
+
+/** Aggregate results from multiple judges into feedback items. */
+function aggregateResults(results: ResponderJudgeResult[], numJudges: number): Feedback[] {
+	const feedback: Feedback[] = [];
+
+	// Per-dimension averaged metrics
+	for (const key of DIMENSION_KEYS) {
+		const avgScore =
+			results.reduce((sum, r) => {
+				const dimension = r[key];
+				return sum + (dimension?.score ?? 0);
+			}, 0) / numJudges;
+		const comments = results
+			.map((r, i) => {
+				const dimension = r[key];
+				return `[Judge ${i + 1}] ${dimension?.comment ?? 'No comment'}`;
+			})
+			.join(' | ');
+		feedback.push(fb(key, avgScore, 'metric', comments));
+	}
+
+	// Aggregated overall score
+	const avgOverall = results.reduce((sum, r) => sum + r.overallScore, 0) / numJudges;
+	feedback.push(
+		fb(
+			'overallScore',
+			avgOverall,
+			'score',
+			`${numJudges}/${numJudges} judges averaged ${avgOverall.toFixed(2)}`,
+		),
+	);
+
+	// Per-judge detail items
+	for (let i = 0; i < results.length; i++) {
+		const r = results[i];
+		feedback.push(fb(`judge${i + 1}`, r.overallScore, 'detail', `Judge ${i + 1}: ${r.summary}`));
+	}
+
+	return feedback;
+}
+
+/**
+ * Create a responder LLM-judge evaluator.
+ *
+ * Uses an LLM to evaluate responder output against per-example criteria
+ * from the dataset. The evaluator expects a ResponderEvaluationContext
+ * with `responderOutput` and `responderEvals` fields.
+ *
+ * When `numJudges > 1`, runs multiple judge calls in parallel and aggregates
+ * dimension scores (averaged) and per-judge detail feedback.
+ *
+ * @param llm - The LLM to use for judging
+ * @param options - Optional configuration (e.g. numJudges)
+ * @returns An evaluator that produces feedback for responder output
+ */
+export function createResponderEvaluator(
+	llm: BaseChatModel,
+	options?: ResponderEvaluatorOptions,
+): Evaluator<EvaluationContext> {
+	const numJudges = options?.numJudges ?? DEFAULTS.NUM_JUDGES;
+
+	return {
+		name: EVALUATOR_NAME,
+
+		async evaluate(_workflow, ctx: EvaluationContext): Promise<Feedback[]> {
+			if (!isResponderContext(ctx)) {
+				return [
+					fb(
+						'error',
+						0,
+						'score',
+						'Missing responderOutput or responderEvals in evaluation context',
+					),
+				];
+			}
+
+			const results = await Promise.all(
+				Array.from({ length: numJudges }, async (_, i) => await runSingleJudge(llm, ctx, i)),
+			);
+
+			return aggregateResults(results, numJudges);
+		},
+	};
+}