n8n/packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/llm-judge/evaluation.ts

import { z } from 'zod';

import type { SimpleWorkflow } from '../../../src/types/workflow';

// Violation schema
const violationSchema = z.object({
	type: z.enum(['critical', 'major', 'minor']),
	description: z.string(),
	pointsDeducted: z.number().min(0),
});

// Category score schema
const categoryScoreSchema = z.object({
	violations: z.array(violationSchema),
	score: z.number().min(0).max(1),
});

// Structural similarity schema (with applicable flag)
const structuralSimilaritySchema = z.object({
	violations: z.array(violationSchema),
	score: z.number().min(0).max(1),
	applicable: z
		.boolean()
		.describe('Whether this category was evaluated (based on reference workflow availability)'),
});

const efficiencyScoreSchema = categoryScoreSchema.extend({
	redundancyScore: z.number().min(0).max(1).describe('Score for avoiding redundant operations'),
	pathOptimization: z.number().min(0).max(1).describe('Score for optimal execution paths'),
	nodeCountEfficiency: z.number().min(0).max(1).describe('Score for using minimal nodes'),
});

const maintainabilityScoreSchema = categoryScoreSchema.extend({
	nodeNamingQuality: z.number().min(0).max(1).describe('Score for descriptive node naming'),
	workflowOrganization: z.number().min(0).max(1).describe('Score for logical workflow structure'),
	modularity: z.number().min(0).max(1).describe('Score for reusable and modular components'),
});

const bestPracticesScoreSchema = categoryScoreSchema.extend({
	techniques: z
		.array(z.string())
		.describe(
			'Workflow techniques identified for this evaluation (e.g., chatbot, content-generation)',
		)
		.optional(),
});

// Main evaluation result schema
export const evaluationResultSchema = z.object({
	overallScore: z
		.number()
		.min(0)
		.max(1)
		.describe('Weighted average score across all categories (0-1)'),
	functionality: categoryScoreSchema,
	connections: categoryScoreSchema,
	expressions: categoryScoreSchema,
	nodeConfiguration: categoryScoreSchema,
	structuralSimilarity: structuralSimilaritySchema,
	efficiency: efficiencyScoreSchema,
	dataFlow: categoryScoreSchema,
	maintainability: maintainabilityScoreSchema,
	bestPractices: bestPracticesScoreSchema,
	summary: z.string().describe('2-3 sentences summarizing main strengths and weaknesses'),
	criticalIssues: z
		.array(z.string())
		.describe('List of issues that would prevent the workflow from functioning')
		.optional(),
});

// Type exports
export type Violation = z.infer<typeof violationSchema>;
export type CategoryScore = z.infer<typeof categoryScoreSchema>;
export type EfficiencyScore = z.infer<typeof efficiencyScoreSchema>;
export type MaintainabilityScore = z.infer<typeof maintainabilityScoreSchema>;
export type BestPracticesScore = z.infer<typeof bestPracticesScoreSchema>;
export type EvaluationResult = z.infer<typeof evaluationResultSchema>;

// Test case schema for evaluation
export const testCaseSchema = z.object({
	id: z.string(),
	name: z.string(),
	prompt: z.string(),
	referenceWorkflows: z.array(z.custom<SimpleWorkflow>()).optional(),
});

export type TestCase = z.infer<typeof testCaseSchema>;

// Evaluation input schema
export const evaluationInputSchema = z.object({
	userPrompt: z.string(),
	generatedWorkflow: z.custom<SimpleWorkflow>(),
	referenceWorkflows: z.array(z.custom<SimpleWorkflow>()).optional(),
	preset: z.enum(['strict', 'standard', 'lenient']).optional(),
});

export type EvaluationInput = z.infer<typeof evaluationInputSchema>;