alighasami
3d5eaf9445
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
43 lines
1.4 KiB
TypeScript
43 lines
1.4 KiB
TypeScript
/**
|
|
* Execution evaluator for the v2 evaluation harness.
|
|
*
|
|
* Executes generated workflows with pin data to verify they run without errors.
|
|
* Service nodes use pin data (skipping real API calls); utility nodes execute
|
|
* using their real compiled dist implementations.
|
|
*/
|
|
|
|
import type { SimpleWorkflow } from '@/types/workflow';
|
|
|
|
import type { EvaluationContext, Evaluator, Feedback } from '../../harness/harness-types';
|
|
import { executeWorkflowWithPinData } from '../../support/workflow-executor';
|
|
|
|
/**
|
|
* Create an execution evaluator that runs the workflow with pin data.
|
|
*
|
|
* Node implementations are loaded lazily from the compiled dist/ files in
|
|
* nodes-base and nodes-langchain — no DI or package-loading setup required.
|
|
*/
|
|
export function createExecutionEvaluator(): Evaluator<EvaluationContext> {
|
|
return {
|
|
name: 'execution',
|
|
|
|
async evaluate(workflow: SimpleWorkflow, ctx: EvaluationContext): Promise<Feedback[]> {
|
|
const pinData = ctx.pinData ?? {};
|
|
|
|
const result = await executeWorkflowWithPinData(workflow, pinData);
|
|
|
|
return [
|
|
{
|
|
evaluator: 'execution',
|
|
metric: 'executionSuccess',
|
|
score: result.success ? 1 : 0,
|
|
kind: 'score',
|
|
comment: result.success
|
|
? `Workflow executed successfully (${result.executedNodes.length} nodes)`
|
|
: `Execution failed: ${result.error}${result.errorNode ? ` (at node: ${result.errorNode})` : ''}`,
|
|
},
|
|
];
|
|
},
|
|
};
|
|
}
|