first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled

This commit is contained in:
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
@@ -0,0 +1,488 @@
import { mock } from 'jest-mock-extended';
import {
type IDataTableProjectService,
NodeOperationError,
type AssignmentCollectionValue,
type IExecuteFunctions,
type INodeTypes,
type NodeParameterValueType,
} from 'n8n-workflow';
import { GoogleSheet } from '../../Google/Sheet/v2/helpers/GoogleSheet';
import { Evaluation } from '../Evaluation/Evaluation.node.ee';
describe('Test Evaluation', () => {
const sheetName = 'Sheet5';
const spreadsheetId = '1oqFpPgEPTGDw7BPkp1SfPXq3Cb3Hyr1SROtf-Ec4zvA';
const mockDataTable = mock<IDataTableProjectService>({
getColumns: jest.fn(),
addColumn: jest.fn(),
updateRows: jest.fn(),
});
const mockExecuteFunctions = mock<IExecuteFunctions>({
helpers: { getDataTableProxy: jest.fn().mockResolvedValue(mockDataTable) },
});
beforeEach(() => {
(mockExecuteFunctions.getInputData as jest.Mock).mockReturnValue([{ json: {} }]);
(mockExecuteFunctions.getNode as jest.Mock).mockReturnValue({ typeVersion: 4.6 });
(mockExecuteFunctions.getParentNodes as jest.Mock).mockReturnValue([
{ type: 'n8n-nodes-base.evaluationTrigger', name: 'Evaluation' },
]);
(mockExecuteFunctions.evaluateExpression as jest.Mock).mockReturnValue({
row_number: 23,
foo: 1,
bar: 2,
_rowsLeft: 2,
});
});
afterEach(() => jest.clearAllMocks());
describe('Test Evaluation Node for Set Output', () => {
describe('Data tables', () => {
test('should have data table methods defined', async () => {
const evaluationNode = new Evaluation();
expect(evaluationNode.methods.listSearch.dataTableSearch).toBeDefined();
expect(evaluationNode.methods.loadOptions.getConditionsForColumn).toBeDefined();
expect(evaluationNode.methods.loadOptions.getDataTableColumns).toBeDefined();
});
test('should throw error if output values is empty', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
'outputs.values': [],
dataTableId: 'mockDataTableId',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
await expect(new Evaluation().execute.call(mockExecuteFunctions)).rejects.toThrow(
'No outputs to set',
);
expect(mockDataTable.getColumns).not.toHaveBeenCalled();
expect(mockDataTable.addColumn).not.toHaveBeenCalled();
expect(mockDataTable.updateRows).not.toHaveBeenCalled();
});
test('should return empty when there is no parent evaluation trigger', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
'outputs.values': [{ outputName: 'bob', outputValue: 'clam' }],
dataTableId: 'mockDataTableId',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
mockExecuteFunctions.getParentNodes.mockReturnValue([]);
const result = await new Evaluation().execute.call(mockExecuteFunctions);
expect(result).toEqual([[{ json: {} }]]);
expect(mockDataTable.getColumns).not.toHaveBeenCalled();
expect(mockDataTable.addColumn).not.toHaveBeenCalled();
expect(mockDataTable.updateRows).not.toHaveBeenCalled();
});
test('should update rows and return input data with existing columns', async () => {
(mockExecuteFunctions.evaluateExpression as jest.Mock).mockReturnValue({
row_id: 23,
row_number: 23,
foo: 1,
bar: 2,
_rowsLeft: 2,
});
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
'outputs.values': [{ outputName: 'foo', outputValue: 'clam' }],
dataTableId: 'mockDataTableId',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
mockDataTable.getColumns.mockResolvedValue([
{
id: '1',
index: 0,
name: 'foo',
type: 'string',
dataTableId: 'mockDataTableId',
},
]);
await new Evaluation().execute.call(mockExecuteFunctions);
expect(mockDataTable.getColumns).toHaveBeenCalled();
expect(mockDataTable.addColumn).not.toHaveBeenCalled();
expect(mockDataTable.updateRows).toHaveBeenCalledWith({
filter: {
type: 'and',
filters: [{ columnName: 'id', condition: 'eq', value: 23 }],
},
data: { foo: 'clam' },
});
});
test('should update rows and return input data with new columns', async () => {
(mockExecuteFunctions.evaluateExpression as jest.Mock).mockReturnValue({
row_id: 23,
row_number: 23,
foo: 1,
bar: 2,
_rowsLeft: 2,
});
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
'outputs.values': [
{ outputName: 'foo', outputValue: 'clam' },
{ outputName: 'bar', outputValue: 'baz' },
],
dataTableId: 'mockDataTableId',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
mockDataTable.getColumns.mockResolvedValue([
{
id: '1',
index: 0,
name: 'foo',
type: 'string',
dataTableId: 'mockDataTableId',
},
]);
await new Evaluation().execute.call(mockExecuteFunctions);
expect(mockDataTable.getColumns).toHaveBeenCalled();
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'bar',
type: 'string',
});
expect(mockDataTable.updateRows).toHaveBeenCalledWith({
filter: {
type: 'and',
filters: [{ columnName: 'id', condition: 'eq', value: 23 }],
},
data: { foo: 'clam', bar: 'baz' },
});
});
});
describe('Google Sheets', () => {
jest.spyOn(GoogleSheet.prototype, 'spreadsheetGetSheet').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
jest.spyOn(GoogleSheet.prototype, 'updateRows').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
jest.spyOn(GoogleSheet.prototype, 'batchUpdate').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
test('credential test for googleApi should be in methods', async () => {
const evaluationNode = new Evaluation();
expect(evaluationNode.methods.credentialTest.googleApiCredentialTest).toBeDefined();
});
test('should throw error if output values is empty', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
'outputs.values': [],
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
await expect(new Evaluation().execute.call(mockExecuteFunctions)).rejects.toThrow(
'No outputs to set',
);
expect(GoogleSheet.prototype.updateRows).not.toBeCalled();
expect(GoogleSheet.prototype.batchUpdate).not.toBeCalled();
});
test('should update rows and return input data for existing headers', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
'outputs.values': [{ outputName: 'foo', outputValue: 'clam' }],
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
await new Evaluation().execute.call(mockExecuteFunctions);
expect(GoogleSheet.prototype.updateRows).toHaveBeenCalledWith(
sheetName,
[['foo', 'bar']],
'RAW',
1,
);
expect(GoogleSheet.prototype.batchUpdate).toHaveBeenCalledWith(
[
{
range: 'Sheet5!A23',
values: [['clam']],
},
],
'RAW',
);
});
test('should return empty when there is no parent evaluation trigger', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
'outputs.values': [{ outputName: 'bob', outputValue: 'clam' }],
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
mockExecuteFunctions.getParentNodes.mockReturnValue([]);
const result = await new Evaluation().execute.call(mockExecuteFunctions);
expect(result).toEqual([[{ json: {} }]]);
expect(GoogleSheet.prototype.updateRows).not.toBeCalled();
expect(GoogleSheet.prototype.batchUpdate).not.toBeCalled();
});
test('should update rows and return input data for new headers', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
'outputs.values': [{ outputName: 'bob', outputValue: 'clam' }],
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
operation: 'setOutputs',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
await new Evaluation().execute.call(mockExecuteFunctions);
expect(GoogleSheet.prototype.updateRows).toHaveBeenCalledWith(
sheetName,
[['foo', 'bar', 'bob']],
'RAW',
1,
);
expect(GoogleSheet.prototype.batchUpdate).toHaveBeenCalledWith(
[
{
range: 'Sheet5!C23',
values: [['clam']],
},
],
'RAW',
);
});
});
});
describe('Test Evaluation Node for Set Metrics', () => {
const nodeTypes = mock<INodeTypes>();
const evaluationMetricsNode = new Evaluation();
let mockExecuteFunction: IExecuteFunctions;
function getMockExecuteFunction(metrics: AssignmentCollectionValue['assignments']) {
return {
getInputData: jest.fn().mockReturnValue([{}]),
getNodeParameter: jest.fn((param: string, _: number) => {
if (param === 'metrics') {
return { assignments: metrics };
}
if (param === 'operation') {
return 'setMetrics';
}
if (param === 'metric') {
return 'customMetrics';
}
return param;
}),
getNode: jest.fn().mockReturnValue({
typeVersion: 1,
}),
} as unknown as IExecuteFunctions;
}
beforeAll(() => {
mockExecuteFunction = getMockExecuteFunction([
{
id: '1',
name: 'Accuracy',
value: 0.95,
type: 'number',
},
{
id: '2',
name: 'Latency',
value: 100,
type: 'number',
},
]);
nodeTypes.getByName.mockReturnValue(evaluationMetricsNode);
jest.clearAllMocks();
});
describe('execute', () => {
it('should output the defined metrics', async () => {
const result = await evaluationMetricsNode.execute.call(mockExecuteFunction);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(1);
const outputItem = result[0][0].json;
expect(outputItem).toEqual({
Accuracy: 0.95,
Latency: 100,
});
});
it('should handle no metrics defined', async () => {
mockExecuteFunction = getMockExecuteFunction([]);
const result = await evaluationMetricsNode.execute.call(mockExecuteFunction);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(1);
expect(result[0][0].json).toEqual({});
});
it('should convert string values to numbers', async () => {
const mockExecuteWithStringValues = getMockExecuteFunction([
{
id: '1',
name: 'Accuracy',
value: '0.95',
type: 'number',
},
{
id: '2',
name: 'Latency',
value: '100',
type: 'number',
},
]);
const result = await evaluationMetricsNode.execute.call(mockExecuteWithStringValues);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(1);
const outputItem = result[0][0].json;
expect(outputItem).toEqual({
Accuracy: 0.95,
Latency: 100,
});
});
it('should throw error for non-numeric string values', async () => {
const mockExecuteWithInvalidValue = getMockExecuteFunction([
{
id: '1',
name: 'Accuracy',
value: 'not-a-number',
type: 'number',
},
]);
await expect(
evaluationMetricsNode.execute.call(mockExecuteWithInvalidValue),
).rejects.toThrow(NodeOperationError);
});
});
});
describe('Test Evaluation Node for Check If Evaluating', () => {
beforeEach(() => {
(mockExecuteFunctions.getInputData as jest.Mock).mockReturnValue([{ json: {} }]);
(mockExecuteFunctions.getNode as jest.Mock).mockReturnValue({ typeVersion: 4.6 });
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
operation: 'checkIfEvaluating',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
});
afterEach(() => jest.clearAllMocks());
test('should return output in normal branch if normal execution', async () => {
(mockExecuteFunctions.getParentNodes as jest.Mock).mockReturnValue([]);
const result = await new Evaluation().execute.call(mockExecuteFunctions);
expect(result).toEqual([[], [{ json: {} }]]);
});
test('should return output in evaluation branch if evaluation execution', async () => {
(mockExecuteFunctions.getParentNodes as jest.Mock).mockReturnValue([
{ type: 'n8n-nodes-base.evaluationTrigger', name: 'Evaluation' },
]);
const result = await new Evaluation().execute.call(mockExecuteFunctions);
expect(result).toEqual([[{ json: {} }], []]);
});
});
});
@@ -0,0 +1,736 @@
import { mock, mockDeep } from 'jest-mock-extended';
import type { IExecuteFunctions, NodeParameterValueType } from 'n8n-workflow';
import { GoogleSheet } from '../../Google/Sheet/v2/helpers/GoogleSheet';
import { EvaluationTrigger } from '../EvaluationTrigger/EvaluationTrigger.node.ee';
import * as utils from '../utils/evaluationTriggerUtils';
describe('Evaluation Trigger Node', () => {
const sheetName = 'Sheet5';
const spreadsheetId = '1oqFpPgEPTGDw7BPkp1SfPXq3Cb3Hyr1SROtf-Ec4zvA';
let mockExecuteFunctions = mock<IExecuteFunctions>({
getInputData: jest.fn().mockReturnValue([{ json: {} }]),
getNode: jest.fn().mockReturnValue({ typeVersion: 4.6 }),
});
let mockDataTable: { getManyRowsAndCount: jest.Mock; getColumns: jest.Mock };
describe('execute', () => {
describe('without filters', () => {
beforeEach(() => {
jest.resetAllMocks();
mockExecuteFunctions = mock<IExecuteFunctions>({
getInputData: jest.fn().mockReturnValue([{ json: {} }]),
getNode: jest.fn().mockReturnValue({ typeVersion: 4.6 }),
});
jest.spyOn(GoogleSheet.prototype, 'spreadsheetGetSheet').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
// Mocks getResults() and getRowsLeft()
jest.spyOn(GoogleSheet.prototype, 'getData').mockImplementation(async (range: string) => {
if (range === `${sheetName}!1:1`) {
return [['Header1', 'Header2']];
} else if (range === `${sheetName}!2:1000`) {
return [
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
];
} else if (range === `${sheetName}!2:2`) {
// getRowsLeft with limit
return [];
} else if (range === sheetName) {
return [
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
];
} else {
return [];
}
});
});
test('credential test for googleApi should be in methods', async () => {
const evaluationTrigger = new EvaluationTrigger();
expect(evaluationTrigger.methods.credentialTest.googleApiCredentialTest).toBeDefined();
});
test('should return a single row from google sheet', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
options: {},
'filtersUI.values': [],
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().execute.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
_rowsLeft: 2,
},
pairedItem: {
item: 0,
},
},
],
]);
});
test('should return the next row from google sheet', async () => {
mockExecuteFunctions.getInputData.mockReturnValue([
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
_rowsLeft: 1,
},
pairedItem: {
item: 0,
input: undefined,
},
},
]);
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
options: {},
'filtersUI.values': [],
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().execute.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: {
row_number: 3,
Header1: 'Value3',
Header2: 'Value4',
_rowsLeft: 0,
},
pairedItem: {
item: 0,
},
},
],
]);
});
test('should return the first row from google sheet if no rows left', async () => {
mockExecuteFunctions.getInputData.mockReturnValue([
{
json: {
row_number: 3,
Header1: 'Value3',
Header2: 'Value4',
_rowsLeft: 0,
},
pairedItem: {
item: 0,
input: undefined,
},
},
]);
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
options: {},
'filtersUI.values': [],
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().execute.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
_rowsLeft: 2,
},
pairedItem: {
item: 0,
},
},
],
]);
});
test('should return a single row from google sheet with limit', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
options: {},
'filtersUI.values': [],
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
limitRows: true,
maxRows: 1,
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().execute.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
_rowsLeft: 0,
},
pairedItem: {
item: 0,
},
},
],
]);
});
});
describe('with filters', () => {
beforeEach(() => {
jest.resetAllMocks();
mockExecuteFunctions = mock<IExecuteFunctions>({
getInputData: jest.fn().mockReturnValue([{ json: {} }]),
getNode: jest.fn().mockReturnValue({ typeVersion: 4.6 }),
});
jest.spyOn(GoogleSheet.prototype, 'spreadsheetGetSheet').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
});
test('should return a single row from google sheet using filter', async () => {
jest
.spyOn(GoogleSheet.prototype, 'getData')
.mockResolvedValueOnce([
// operationResult
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
])
.mockResolvedValueOnce([
// rowsLeft
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
]);
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
limitRows: true,
maxRows: 2,
'filtersUI.values': [{ lookupColumn: 'Header1', lookupValue: 'Value1' }],
options: {},
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
jest.spyOn(utils, 'getRowsLeft').mockResolvedValue(0);
const evaluationTrigger = new EvaluationTrigger();
const result = await evaluationTrigger.execute.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
_rowsLeft: 0,
},
pairedItem: {
item: 0,
},
},
],
]);
});
});
describe('Data tables with filters', () => {
beforeEach(() => {
jest.resetAllMocks();
mockDataTable = {
getManyRowsAndCount: jest.fn(),
getColumns: jest.fn().mockResolvedValue([{ name: 'processed', type: 'number' }]),
};
mockExecuteFunctions = mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ typeVersion: 4.7 }),
helpers: {
getDataTableProxy: jest.fn().mockResolvedValue(mockDataTable),
},
});
});
test('should process rows sequentially with filters when dataset changes', async () => {
// Simulate the user's scenario: 5 rows with processed=1, updating to processed=2 after each execution
// With each execution, one row is processed and thus no longer matches the filter
mockDataTable.getManyRowsAndCount
.mockResolvedValueOnce({
data: [{ id: 1, processed: 1 }],
count: 5,
})
.mockResolvedValueOnce({
data: [{ id: 2, processed: 1 }],
count: 4,
})
.mockResolvedValueOnce({
data: [{ id: 3, processed: 1 }],
count: 3,
});
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
limitRows: false,
dataTableId: 'mockDataTableId',
'filters.conditions': [
{
keyName: 'processed',
condition: 'eq',
keyValue: '1',
},
],
matchType: 'anyCondition',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const evaluationTrigger = new EvaluationTrigger();
// First execution - no previous data
mockExecuteFunctions.getInputData.mockReturnValue([{ json: {} }]);
const result1 = await evaluationTrigger.execute.call(mockExecuteFunctions);
expect(result1[0][0].json.row_id).toBe(1);
expect(result1[0][0].json.row_number).toBe(0);
expect(result1[0][0].json._rowsLeft).toBe(4);
// Verify first call used user filter only (no id filter yet)
expect(mockDataTable.getManyRowsAndCount).toHaveBeenNthCalledWith(1, {
skip: 0,
take: 1,
filter: {
type: 'or',
filters: [
{
columnName: 'processed',
condition: 'eq',
value: '1',
},
],
},
});
// Second execution - previous row was id=1
mockExecuteFunctions.getInputData.mockReturnValue(result1[0]);
const result2 = await evaluationTrigger.execute.call(mockExecuteFunctions);
expect(result2[0][0].json.row_id).toBe(2);
expect(result2[0][0].json.row_number).toBe(1);
// Verify second call includes id > 1 filter
expect(mockDataTable.getManyRowsAndCount).toHaveBeenNthCalledWith(2, {
skip: 0,
take: 1,
filter: {
type: 'and',
filters: [
{
columnName: 'processed',
condition: 'eq',
value: '1',
},
{
columnName: 'id',
condition: 'gt',
value: 1,
},
],
},
});
// Third execution - previous row was id=2
mockExecuteFunctions.getInputData.mockReturnValue(result2[0]);
const result3 = await evaluationTrigger.execute.call(mockExecuteFunctions);
expect(result3[0][0].json.row_id).toBe(3);
expect(result3[0][0].json.row_number).toBe(2);
// Verify third call includes id > 2 filter
expect(mockDataTable.getManyRowsAndCount).toHaveBeenNthCalledWith(3, {
skip: 0,
take: 1,
filter: {
type: 'and',
filters: [
{
columnName: 'processed',
condition: 'eq',
value: '1',
},
{
columnName: 'id',
condition: 'gt',
value: 2,
},
],
},
});
});
});
});
describe('customOperations.dataset.getRows', () => {
describe('Data tables', () => {
beforeEach(() => {
jest.resetAllMocks();
mockDataTable = {
getManyRowsAndCount: jest.fn().mockResolvedValue({
data: [
{ id: 1, field1: 'value1', field2: 'value2' },
{ id: 2, field1: 'value3', field2: 'value4' },
],
}),
getColumns: jest.fn().mockResolvedValue([
{ name: 'field1', type: 'string' },
{ name: 'field2', type: 'string' },
]),
};
mockExecuteFunctions = mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ typeVersion: 4.7 }),
helpers: {
getDataTableProxy: jest.fn().mockResolvedValue(mockDataTable),
},
});
});
test('should return the rows with limits applied, without filters', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
limitRows: true,
maxRows: 2,
dataTableId: 'mockDataTableId',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().customOperations.dataset.getRows.call(
mockExecuteFunctions,
);
expect(mockDataTable.getManyRowsAndCount).toHaveBeenCalledWith({
skip: 0,
take: 2,
filter: { filters: [], type: 'or' },
});
expect(result).toEqual([
[
{
json: {
id: 1,
row_id: 1,
row_number: 0,
field1: 'value1',
field2: 'value2',
},
pairedItem: {
item: 0,
},
},
{
json: {
id: 2,
row_id: 2,
row_number: 1,
field1: 'value3',
field2: 'value4',
},
pairedItem: {
item: 0,
},
},
],
]);
});
test('should return the rows with limits applied, with filters', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'dataTable',
limitRows: true,
maxRows: 2,
dataTableId: 'mockDataTableId',
'filters.conditions': [
{
keyName: 'field1',
condition: 'like',
keyValue: '1',
},
{
keyName: 'field2',
condition: 'eq',
keyValue: 'value4',
},
],
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
await new EvaluationTrigger().customOperations.dataset.getRows.call(mockExecuteFunctions);
expect(mockDataTable.getManyRowsAndCount).toHaveBeenCalledWith({
skip: 0,
take: 2,
filter: {
filters: [
{
columnName: 'field1',
condition: 'like',
value: '1',
},
{
columnName: 'field2',
condition: 'eq',
value: 'value4',
},
],
type: 'or',
},
});
});
});
describe('Google Sheets', () => {
beforeEach(() => {
jest.resetAllMocks();
mockExecuteFunctions = mock<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ typeVersion: 4.6 }),
});
jest.spyOn(GoogleSheet.prototype, 'spreadsheetGetSheet').mockImplementation(async () => {
return { sheetId: 1, title: sheetName };
});
// Mocks getResults() and getRowsLeft()
jest.spyOn(GoogleSheet.prototype, 'getData').mockImplementation(async (range: string) => {
if (range === `${sheetName}!1:1`) {
return [['Header1', 'Header2']];
} else if (range === `${sheetName}!2:1000`) {
return [
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
];
} else if (range === `${sheetName}!2:2`) {
// getRowsLeft with limit
return [];
} else if (range === sheetName) {
return [
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
];
} else {
return [];
}
});
});
test('should return the sheet with limits applied, without filters', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
options: {},
'filtersUI.values': [],
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
limitRows: true,
maxRows: 2,
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
const result = await new EvaluationTrigger().customOperations.dataset.getRows.call(
mockExecuteFunctions,
);
expect(result).toEqual([
[
{
json: {
row_number: 2,
Header1: 'Value1',
Header2: 'Value2',
},
pairedItem: {
item: 0,
},
},
{
json: {
row_number: 3,
Header1: 'Value3',
Header2: 'Value4',
},
pairedItem: {
item: 0,
},
},
],
]);
});
test('should return all relevant rows from google sheet using filters', async () => {
mockExecuteFunctions.getInputData.mockReturnValue([{ json: {} }]);
jest
.spyOn(GoogleSheet.prototype, 'getData')
.mockResolvedValueOnce([
// operationResult
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
['Value1', 'Value4'],
])
.mockResolvedValueOnce([
// rowsLeft
['Header1', 'Header2'],
['Value1', 'Value2'],
['Value3', 'Value4'],
['Value1', 'Value4'],
]);
mockExecuteFunctions.getNodeParameter.mockImplementation(
(key: string, _: number, fallbackValue?: string | number | boolean | object) => {
const mockParams: { [key: string]: unknown } = {
source: 'googleSheets',
'filtersUI.values': [{ lookupColumn: 'Header1', lookupValue: 'Value1' }],
options: {},
combineFilters: 'AND',
documentId: {
mode: 'id',
value: spreadsheetId,
},
sheetName,
sheetMode: 'id',
};
return (mockParams[key] ?? fallbackValue) as NodeParameterValueType;
},
);
jest.spyOn(utils, 'getRowsLeft').mockResolvedValue(0);
const evaluationTrigger = new EvaluationTrigger();
const result =
await evaluationTrigger.customOperations.dataset.getRows.call(mockExecuteFunctions);
expect(result).toEqual([
[
{
json: { row_number: 2, Header1: 'Value1', Header2: 'Value2' },
pairedItem: {
item: 0,
},
},
{
json: { row_number: 4, Header1: 'Value1', Header2: 'Value4' },
pairedItem: {
item: 0,
},
},
],
]);
});
});
});
});
@@ -0,0 +1,102 @@
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { readSheet } from '../../Google/Sheet/v2/actions/utils/readOperation';
import { GoogleSheet } from '../../Google/Sheet/v2/helpers/GoogleSheet';
import { getFilteredResults } from '../utils/evaluationTriggerUtils';
jest.mock('../../Google/Sheet/v2/actions/utils/readOperation', () => ({
readSheet: jest.fn(),
}));
describe('getFilteredResults', () => {
let mockThis: IExecuteFunctions;
let mockGoogleSheet: GoogleSheet;
beforeEach(() => {
// Mock the `this` context
mockThis = {
getNode: jest.fn().mockReturnValue({ typeVersion: 1 }),
} as unknown as IExecuteFunctions;
// Mock the GoogleSheet instance
mockGoogleSheet = new GoogleSheet('mockSpreadsheetId', mockThis);
// Reset mocks before each test
jest.clearAllMocks();
});
it('should return filtered results based on endingRow', async () => {
// Arrange
const mockOperationResult: INodeExecutionData[] = [];
const mockResult = { title: 'Sheet1', sheetId: 1 };
const startingRow = 1;
const endingRow = 3;
(readSheet as jest.Mock).mockResolvedValue([
{ json: { row_number: 1, data: 'Row 1' } },
{ json: { row_number: 2, data: 'Row 2' } },
{ json: { row_number: 3, data: 'Row 3' } },
{ json: { row_number: 4, data: 'Row 4' } },
]);
// Act
const result = await getFilteredResults.call(
mockThis,
mockOperationResult,
mockGoogleSheet,
mockResult,
startingRow,
endingRow,
);
// Assert
expect(readSheet).toHaveBeenCalledWith(
mockGoogleSheet,
'Sheet1',
0,
mockOperationResult,
1,
[],
undefined,
{
rangeDefinition: 'specifyRange',
headerRow: 1,
firstDataRow: startingRow,
includeHeadersWithEmptyCells: true,
},
);
expect(result).toEqual([
{ json: { row_number: 1, data: 'Row 1' } },
{ json: { row_number: 2, data: 'Row 2' } },
{ json: { row_number: 3, data: 'Row 3' } },
]);
});
it('should return an empty array if no rows match the filter', async () => {
// Arrange
const mockOperationResult: INodeExecutionData[] = [];
const mockResult = { title: 'Sheet1', sheetId: 1 };
const startingRow = 1;
const endingRow = 0;
(readSheet as jest.Mock).mockResolvedValue([
{ json: { row_number: 1, data: 'Row 1' } },
{ json: { row_number: 2, data: 'Row 2' } },
]);
// Act
const result = await getFilteredResults.call(
mockThis,
mockOperationResult,
mockGoogleSheet,
mockResult,
startingRow,
endingRow,
);
// Assert
expect(readSheet).toHaveBeenCalled();
expect(result).toEqual([]);
});
});
@@ -0,0 +1,497 @@
import type { IExecuteFunctions } from 'n8n-workflow';
import { UserError } from 'n8n-workflow';
import { setInputs, setOutputs } from '../utils/evaluationUtils';
jest.mock('../utils/evaluationTriggerUtils', () => ({
getGoogleSheet: jest.fn(),
getSheet: jest.fn(),
}));
import { getGoogleSheet, getSheet } from '../utils/evaluationTriggerUtils';
import { mockDeep } from 'jest-mock-extended';
describe('setInputs', () => {
const mockThis = (options: Partial<any> = {}) =>
mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ name: 'EvalNode' }),
getParentNodes: jest
.fn()
.mockReturnValue([{ name: 'EvalTrigger', type: 'n8n-nodes-base.evaluationTrigger' }]),
evaluateExpression: jest.fn().mockReturnValue(true),
getNodeParameter: jest.fn().mockReturnValue([
{ inputName: 'foo', inputValue: 'bar' },
{ inputName: 'baz', inputValue: 'qux' },
]),
getInputData: jest.fn().mockReturnValue([{ json: { test: 1 } }]),
addExecutionHints: jest.fn(),
getMode: jest.fn().mockReturnValue('evaluation'),
...options,
});
it('should return input data with evaluationData when inputs are provided', () => {
const context = mockThis();
const result = setInputs.call(context);
expect(result).toHaveLength(1);
expect(result[0][0].evaluationData).toEqual({ foo: 'bar', baz: 'qux' });
});
it('should throw UserError if no input fields are provided', () => {
const context = mockThis({
getNodeParameter: jest.fn().mockReturnValue([]),
});
expect(() => setInputs.call(context)).toThrow(UserError);
});
it('should add execution hints and return input data if not started from evaluation trigger', () => {
const context = mockThis({
getParentNodes: jest.fn().mockReturnValue([]),
getInputData: jest.fn().mockReturnValue([{ json: { test: 2 } }]),
});
const result = setInputs.call(context);
expect(context.addExecutionHints).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('No inputs were set'),
}),
);
expect(result).toEqual([[{ json: { test: 2 } }]]);
});
it('should add execution hints and return input data if evalTriggerOutput is falsy', () => {
const context = mockThis({
evaluateExpression: jest.fn().mockReturnValue(undefined),
getInputData: jest.fn().mockReturnValue([{ json: { test: 3 } }]),
});
const result = setInputs.call(context);
expect(context.addExecutionHints).toHaveBeenCalled();
expect(result).toEqual([[{ json: { test: 3 } }]]);
});
});
describe('setOutputs', () => {
describe('common', () => {
const mockThis = (options: Partial<IExecuteFunctions> = {}) =>
mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ name: 'EvalNode' }),
getParentNodes: jest
.fn()
.mockReturnValue([{ name: 'EvalTrigger', type: 'n8n-nodes-base.evaluationTrigger' }]),
evaluateExpression: jest.fn().mockImplementation((expr: string) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json')) return { row_id: 1, inputField: 'inputValue' };
return true;
}),
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [{ outputName: 'result', outputValue: 'success' }];
}
}),
getInputData: jest.fn().mockReturnValue([{ json: { test: 1 } }]),
addExecutionHints: jest.fn(),
getMode: jest.fn().mockReturnValue('evaluation'),
...options,
});
it('should throw UserError if no output fields are provided', async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockReturnValue([]),
});
await expect(setOutputs.call(context)).rejects.toThrow(UserError);
await expect(setOutputs.call(context)).rejects.toThrow('No outputs to set');
});
it('should add execution hints and return input data if not started from evaluation trigger', async () => {
const context = mockThis({
getParentNodes: jest.fn().mockReturnValue([]),
getInputData: jest.fn().mockReturnValue([{ json: { test: 2 } }]),
});
const result = await setOutputs.call(context);
expect(context.addExecutionHints).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining("execution didn't start from an evaluation trigger"),
}),
);
expect(result).toEqual([[{ json: { test: 2 } }]]);
expect(getGoogleSheet).not.toHaveBeenCalled();
});
it('should add execution hints and return input data if evalTriggerOutput is falsy', async () => {
const context = mockThis({
evaluateExpression: jest.fn().mockImplementation((expr: string) => {
if (expr.includes('isExecuted')) return false;
return true;
}),
getInputData: jest.fn().mockReturnValue([{ json: { test: 3 } }]),
});
const result = await setOutputs.call(context);
expect(context.addExecutionHints).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining("execution didn't start from an evaluation trigger"),
}),
);
expect(result).toEqual([[{ json: { test: 3 } }]]);
});
});
describe('Data tables', () => {
const outputValues = [
{ outputName: 'result', outputValue: 'success' },
{ outputName: 'score', outputValue: 95 },
{ outputName: 'active', outputValue: true },
{ outputName: 'timestamp', outputValue: new Date('2025-09-18T12:34:56Z') },
{ outputName: 'data', outputValue: { key: 'value' } },
];
const mockDataTable = {
updateRows: jest.fn(),
getColumns: jest.fn().mockReturnValue(outputValues.map((o) => ({ name: o.outputName }))),
addColumn: jest.fn(),
};
const mockThis = (options: Partial<IExecuteFunctions> = {}) =>
mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ name: 'EvalNode' }),
getParentNodes: jest
.fn()
.mockReturnValue([{ name: 'EvalTrigger', type: 'n8n-nodes-base.evaluationTrigger' }]),
evaluateExpression: jest.fn().mockImplementation((expr: string) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json')) return { row_id: 1, inputField: 'inputValue' };
return true;
}),
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return outputValues;
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
getInputData: jest.fn().mockReturnValue([{ json: { test: 1 } }]),
addExecutionHints: jest.fn(),
getMode: jest.fn().mockReturnValue('evaluation'),
helpers: {
getDataTableProxy: jest.fn().mockResolvedValue(mockDataTable),
},
...options,
});
beforeEach(() => {
jest.clearAllMocks();
});
it('should set outputs to Data table and return evaluation data', async () => {
const context = mockThis();
const result = await setOutputs.call(context);
expect(mockDataTable.getColumns).toHaveBeenCalled();
expect(mockDataTable.addColumn).not.toHaveBeenCalled();
expect(mockDataTable.updateRows).toHaveBeenCalledWith({
filter: {
type: 'and',
filters: [
{
columnName: 'id',
condition: 'eq',
value: 1,
},
],
},
data: {
result: 'success',
score: 95,
active: true,
timestamp: new Date('2025-09-18T12:34:56'),
data: '{"key":"value"}',
},
});
expect(result).toHaveLength(1);
expect(result[0][0].evaluationData).toEqual({
result: 'success',
score: 95,
active: true,
timestamp: new Date('2025-09-18T12:34:56'),
data: { key: 'value' },
});
});
it('should set outputs to Data table, correct subsequent row', async () => {
const context = mockThis({
evaluateExpression: jest.fn().mockImplementation((expr: string) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json')) return { row_id: 3, inputField: 'inputValue' };
return true;
}),
});
await setOutputs.call(context);
expect(mockDataTable.updateRows).toHaveBeenCalledWith(
expect.objectContaining({
filter: {
type: 'and',
filters: [
{
columnName: 'id',
condition: 'eq',
value: 3,
},
],
},
}),
);
});
it("should create columns if they don't exist, string", async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [...outputValues, { outputName: 'new_column', outputValue: 'new_value' }];
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
});
const result = await setOutputs.call(context);
expect(mockDataTable.getColumns).toHaveBeenCalled();
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'new_column',
type: 'string',
});
expect(mockDataTable.updateRows).toHaveBeenCalledWith({
filter: {
type: 'and',
filters: [
{
columnName: 'id',
condition: 'eq',
value: 1,
},
],
},
data: {
result: 'success',
score: 95,
active: true,
timestamp: new Date('2025-09-18T12:34:56'),
data: '{"key":"value"}',
new_column: 'new_value',
},
});
expect(result).toHaveLength(1);
expect(result[0][0].evaluationData).toEqual({
result: 'success',
score: 95,
active: true,
timestamp: new Date('2025-09-18T12:34:56'),
data: { key: 'value' },
new_column: 'new_value',
});
});
it("should create columns if they don't exist, number", async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [...outputValues, { outputName: 'new_column', outputValue: 123.45 }];
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
});
await setOutputs.call(context);
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'new_column',
type: 'number',
});
});
it("should create columns if they don't exist, boolean", async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [...outputValues, { outputName: 'new_column', outputValue: true }];
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
});
await setOutputs.call(context);
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'new_column',
type: 'boolean',
});
});
it("should create columns if they don't exist, date", async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [...outputValues, { outputName: 'new_column', outputValue: new Date() }];
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
});
await setOutputs.call(context);
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'new_column',
type: 'date',
});
});
it("should create columns if they don't exist, null", async () => {
const context = mockThis({
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [...outputValues, { outputName: 'new_column', outputValue: null }];
} else if (param === 'source') {
return 'dataTable';
} else if (param === 'dataTableId') {
return 'mockDataTableId';
}
}),
});
await setOutputs.call(context);
expect(mockDataTable.addColumn).toHaveBeenCalledWith({
name: 'new_column',
type: 'string',
});
});
});
describe('Google Sheets', () => {
const mockGoogleSheetInstance = {
updateRows: jest.fn(),
prepareDataForUpdatingByRowNumber: jest.fn().mockReturnValue({
updateData: [{ range: 'Sheet1!A2:C2', values: [['foo', 'bar']] }],
}),
batchUpdate: jest.fn(),
};
const mockSheet = {
title: 'Sheet1',
};
const mockThis = (options: Partial<any> = {}) =>
mockDeep<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ name: 'EvalNode' }),
getParentNodes: jest
.fn()
.mockReturnValue([{ name: 'EvalTrigger', type: 'n8n-nodes-base.evaluationTrigger' }]),
evaluateExpression: jest.fn().mockImplementation((expr) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json')) return { row_number: 2, inputField: 'inputValue' };
return true;
}),
getNodeParameter: jest.fn().mockImplementation((param: string) => {
if (param === 'outputs.values') {
return [
{ outputName: 'result', outputValue: 'success' },
{ outputName: 'score', outputValue: '95' },
];
} else if (param === 'source') {
return 'googleSheets';
}
}),
getInputData: jest.fn().mockReturnValue([{ json: { test: 1 } }]),
addExecutionHints: jest.fn(),
getMode: jest.fn().mockReturnValue('evaluation'),
...options,
});
beforeEach(() => {
jest.clearAllMocks();
(getGoogleSheet as jest.Mock).mockReturnValue(mockGoogleSheetInstance);
(getSheet as jest.Mock).mockResolvedValue(mockSheet);
});
it('should set outputs to Google Sheet and return evaluation data', async () => {
const context = mockThis();
const result = await setOutputs.call(context);
expect(getGoogleSheet).toHaveBeenCalled();
expect(getSheet).toHaveBeenCalledWith(mockGoogleSheetInstance);
expect(mockGoogleSheetInstance.updateRows).toHaveBeenCalledWith(
'Sheet1',
[['inputField', 'result', 'score']],
'RAW',
1,
);
expect(mockGoogleSheetInstance.prepareDataForUpdatingByRowNumber).toHaveBeenCalledWith(
[{ row_number: 2, result: 'success', score: '95' }],
'Sheet1!A:Z',
[['inputField', 'result', 'score']],
);
expect(mockGoogleSheetInstance.batchUpdate).toHaveBeenCalledWith(
[{ range: 'Sheet1!A2:C2', values: [['foo', 'bar']] }],
'RAW',
);
expect(result).toHaveLength(1);
expect(result[0][0].evaluationData).toEqual({ result: 'success', score: '95' });
});
it('should handle row_number as string "row_number" by using 1', async () => {
const context = mockThis({
evaluateExpression: jest.fn().mockImplementation((expr) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json'))
return { row_number: 'row_number', inputField: 'inputValue' };
return true;
}),
});
const result = await setOutputs.call(context);
expect(mockGoogleSheetInstance.prepareDataForUpdatingByRowNumber).toHaveBeenCalledWith(
[{ row_number: 1, result: 'success', score: '95' }],
'Sheet1!A:Z',
[['inputField', 'result', 'score']],
);
expect(result).toHaveLength(1);
});
it('should add new column names that are not in existing columns', async () => {
const context = mockThis({
evaluateExpression: jest.fn().mockImplementation((expr) => {
if (expr.includes('isExecuted')) return true;
if (expr.includes('first().json')) return { row_number: 2, existingCol: 'value' };
return true;
}),
getNodeParameter: jest.fn().mockImplementation((param) => {
if (param === 'outputs.values') {
return [{ outputName: 'newCol', outputValue: 'newValue' }];
} else if (param === 'source') {
return 'googleSheets';
}
}),
});
const result = await setOutputs.call(context);
expect(mockGoogleSheetInstance.updateRows).toHaveBeenCalledWith(
'Sheet1',
[['existingCol', 'newCol']],
'RAW',
1,
);
expect(result).toHaveLength(1);
});
});
});
@@ -0,0 +1,63 @@
/* eslint-disable n8n-nodes-base/node-param-display-name-miscased */
import { type ILoadOptionsFunctions } from 'n8n-workflow';
import { getSheetHeaderRow } from '../../Google/Sheet/v2/methods/loadOptions';
import { getSheetHeaderRowWithGeneratedColumnNames } from '../methods/loadOptions';
jest.mock('../../Google/Sheet/v2/methods/loadOptions', () => ({
getSheetHeaderRow: jest.fn(),
}));
describe('getSheetHeaderRowWithGeneratedColumnNames', () => {
let mockThis: ILoadOptionsFunctions;
beforeEach(() => {
mockThis = {
getNodeParameter: jest.fn(),
getCredentials: jest.fn(),
} as unknown as ILoadOptionsFunctions;
jest.clearAllMocks();
});
it('should return column names as-is if they are not empty', async () => {
(getSheetHeaderRow as jest.Mock).mockResolvedValue([
{ name: 'Column1', value: 'Column1' },
{ name: 'Column2', value: 'Column2' },
]);
const result = await getSheetHeaderRowWithGeneratedColumnNames.call(mockThis);
expect(getSheetHeaderRow).toHaveBeenCalled();
expect(result).toEqual([
{ name: 'Column1', value: 'Column1' },
{ name: 'Column2', value: 'Column2' },
]);
});
it('should generate column names for empty values', async () => {
(getSheetHeaderRow as jest.Mock).mockResolvedValue([
{ name: '', value: '' },
{ name: 'Column2', value: 'Column2' },
{ name: '', value: '' },
]);
const result = await getSheetHeaderRowWithGeneratedColumnNames.call(mockThis);
expect(getSheetHeaderRow).toHaveBeenCalled();
expect(result).toEqual([
{ name: 'col_1', value: 'col_1' },
{ name: 'Column2', value: 'Column2' },
{ name: 'col_3', value: 'col_3' },
]);
});
it('should handle an empty header row gracefully', async () => {
(getSheetHeaderRow as jest.Mock).mockResolvedValue([]);
const result = await getSheetHeaderRowWithGeneratedColumnNames.call(mockThis);
expect(getSheetHeaderRow).toHaveBeenCalled();
expect(result).toEqual([]);
});
});
@@ -0,0 +1,791 @@
import { mock } from 'jest-mock-extended';
import { NodeOperationError } from 'n8n-workflow';
import type { IExecuteFunctions, INode, AssignmentCollectionValue } from 'n8n-workflow';
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import type { Runnable } from '@langchain/core/runnables';
import { metricHandlers } from '../utils/metricHandlers';
// Mock the validateEntry function
jest.mock('../../Set/v2/helpers/utils', () => ({
validateEntry: jest.fn((name: string, _type: string, value: any) => ({
name,
value,
})),
}));
describe('metricHandlers', () => {
let mockExecuteFunctions: jest.Mocked<IExecuteFunctions>;
let mockNode: INode;
beforeEach(() => {
mockExecuteFunctions = mock<IExecuteFunctions>();
mockNode = {
id: 'test-node',
name: 'Test Node',
type: 'n8n-nodes-base.evaluation',
typeVersion: 1,
position: [0, 0],
parameters: {},
};
mockExecuteFunctions.getNode.mockReturnValue(mockNode);
});
afterEach(() => {
jest.clearAllMocks();
});
describe('customMetrics', () => {
it('should process valid custom metrics', async () => {
const metricsData: AssignmentCollectionValue = {
assignments: [
{ id: '1', name: 'Metric1', value: 5, type: 'number' },
{ id: '2', name: 'Metric2', value: '10', type: 'number' },
{ id: '3', name: 'Metric3', value: 7.5, type: 'number' },
],
};
mockExecuteFunctions.getNodeParameter.mockReturnValue(metricsData);
const result = await metricHandlers.customMetrics.call(mockExecuteFunctions, 0);
expect(result).toEqual({
Metric1: 5,
Metric2: 10,
Metric3: 7.5,
});
});
it('should throw error for non-numeric values', async () => {
const metricsData: AssignmentCollectionValue = {
assignments: [{ id: '1', name: 'Metric1', value: 'not-a-number', type: 'number' }],
};
mockExecuteFunctions.getNodeParameter.mockReturnValue(metricsData);
await expect(metricHandlers.customMetrics.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for missing metric name', async () => {
const metricsData: AssignmentCollectionValue = {
assignments: [{ id: '1', name: '', value: 5, type: 'number' }],
};
mockExecuteFunctions.getNodeParameter.mockReturnValue(metricsData);
await expect(metricHandlers.customMetrics.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should handle empty assignments array', async () => {
const metricsData: AssignmentCollectionValue = {
assignments: [],
};
mockExecuteFunctions.getNodeParameter.mockReturnValue(metricsData);
const result = await metricHandlers.customMetrics.call(mockExecuteFunctions, 0);
expect(result).toEqual({});
});
it('should handle undefined assignments', async () => {
const metricsData: AssignmentCollectionValue = { assignments: [] };
mockExecuteFunctions.getNodeParameter.mockReturnValue(metricsData);
const result = await metricHandlers.customMetrics.call(mockExecuteFunctions, 0);
expect(result).toEqual({});
});
});
describe('toolsUsed', () => {
it('should return correct tool usage metrics', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps = [
{ action: { tool: 'calculator' } },
{ action: { tool: 'calculator' } },
{ action: { tool: 'search' } },
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
expect(result).toEqual({
'Tools Used': 1,
});
});
it('should return 0 for unused tools', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps = [{ action: { tool: 'calculator' } }];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
expect(result).toEqual({
'Tools Used': 0.5,
});
});
it('should handle tool names with spaces and special characters', async () => {
const expectedTools = 'Get Events, Send Email, Search Database';
const intermediateSteps = [
{ action: { tool: 'Get_Events' } },
{ action: { tool: 'Send_Email' } },
{ action: { tool: 'Search_Database' } },
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
expect(result).toEqual({
'Tools Used': 1,
});
});
it('should work case-insensitively', async () => {
const expectedTools = 'Get Events, send email, SEARCH DATABASE';
const intermediateSteps = [
{ action: { tool: 'get_events' } },
{ action: { tool: 'SEND_EMAIL' } },
{ action: { tool: 'Search_Database' } },
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
expect(result).toEqual({
'Tools Used': 1,
});
});
it('should handle mixed case and format variations', async () => {
const expectedTools = 'calculator tool, Search Engine, data-processor';
const intermediateSteps = [
{ action: { tool: 'Calculator_Tool' } },
{ action: { tool: 'search_engine' } },
// data-processor is not used, so partial match
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
// 2 out of 3 tools used = 2/3 ≈ 0.6667
expect(result).toEqual({
'Tools Used': 2 / 3,
});
});
it('should throw error for missing expected tools', async () => {
const expectedTools = '';
const intermediateSteps: any[] = [];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for undefined expected tools', async () => {
const expectedTools = undefined;
const intermediateSteps: any[] = [];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
describe('intermediate steps validation', () => {
it('should throw error for missing intermediate steps parameter', async () => {
const expectedTools = 'calculator';
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return undefined;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
new NodeOperationError(mockNode, 'Intermediate steps missing', {
description:
"Make sure to enable returning intermediate steps in your agent node's options, then map them in here",
}),
);
});
it('should throw error for empty object intermediate steps', async () => {
const expectedTools = 'calculator';
const intermediateSteps = {};
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for string intermediate steps', async () => {
const expectedTools = 'calculator';
const intermediateSteps = 'not an array';
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for null intermediate steps', async () => {
const expectedTools = 'calculator';
const intermediateSteps = null;
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
return undefined;
});
await expect(metricHandlers.toolsUsed.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should handle empty array intermediate steps gracefully', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps: any[] = [];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
expect(result).toEqual({
'Tools Used': 0,
});
});
it('should handle malformed intermediate steps objects', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps = [
{ action: { tool: 'calculator' } }, // valid
{ action: {} }, // missing tool property
{ notAction: { tool: 'search' } }, // wrong structure
{}, // completely empty
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
// Only 'calculator' should match (1 out of 2 expected tools)
expect(result).toEqual({
'Tools Used': 0.5,
});
});
it('should handle intermediate steps with null/undefined tool names', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps = [
{ action: { tool: 'calculator' } }, // valid
{ action: { tool: null } }, // null tool
{ action: { tool: undefined } }, // undefined tool
{ action: { tool: '' } }, // empty string tool
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
// Only 'calculator' should match (1 out of 2 expected tools)
expect(result).toEqual({
'Tools Used': 0.5,
});
});
it('should handle intermediate steps with non-string tool names', async () => {
const expectedTools = 'calculator, search';
const intermediateSteps = [
{ action: { tool: 'calculator' } }, // valid
{ action: { tool: 123 } }, // number
{ action: { tool: { name: 'search' } } }, // object
{ action: { tool: ['search'] } }, // array
];
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedTools') return expectedTools;
if (paramName === 'intermediateSteps') return intermediateSteps;
if (paramName === 'options.metricName') return 'Tools Used';
return undefined;
});
// This should not throw an error, but might have unexpected behavior
// depending on how the comparison works
const result = await metricHandlers.toolsUsed.call(mockExecuteFunctions, 0);
// Only 'calculator' should match reliably (1 out of 2 expected tools)
expect(result).toEqual({
'Tools Used': 0.5,
});
});
});
});
describe('categorization', () => {
it('should return 1 for exact match', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'expected answer';
if (paramName === 'actualAnswer') return 'expected answer';
if (paramName === 'options.metricName') return 'Categorization';
return undefined;
});
const result = await metricHandlers.categorization.call(mockExecuteFunctions, 0);
expect(result).toEqual({ Categorization: 1 });
});
it('should return 0 for non-match', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'expected answer';
if (paramName === 'actualAnswer') return 'different answer';
if (paramName === 'options.metricName') return 'Categorization';
return undefined;
});
const result = await metricHandlers.categorization.call(mockExecuteFunctions, 0);
expect(result).toEqual({ Categorization: 0 });
});
it('should use custom metric name', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'expected answer';
if (paramName === 'actualAnswer') return 'expected answer';
if (paramName === 'options.metricName') return 'Custom Categorization';
return undefined;
});
const result = await metricHandlers.categorization.call(mockExecuteFunctions, 0);
expect(result).toEqual({ 'Custom Categorization': 1 });
});
it('should handle whitespace trimming', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return ' expected answer ';
if (paramName === 'actualAnswer') return 'expected answer';
if (paramName === 'options.metricName') return 'Categorization';
return undefined;
});
const result = await metricHandlers.categorization.call(mockExecuteFunctions, 0);
expect(result).toEqual({ Categorization: 1 });
});
it('should throw error for missing expected answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return '';
if (paramName === 'actualAnswer') return 'actual answer';
return undefined;
});
await expect(metricHandlers.categorization.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for missing actual answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'expected answer';
if (paramName === 'actualAnswer') return '';
return undefined;
});
await expect(metricHandlers.categorization.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
});
describe('stringSimilarity', () => {
it('should return inverted similarity score', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'hello';
if (paramName === 'actualAnswer') return 'helo';
if (paramName === 'options.metricName') return 'String similarity';
return undefined;
});
const result = await metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0);
// Edit distance is 1, longer string length is 5, so similarity = 1 - (1/5) = 0.8
expect(result).toEqual({ 'String similarity': 0.8 });
});
it('should return 1 for identical strings', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'hello';
if (paramName === 'actualAnswer') return 'hello';
if (paramName === 'options.metricName') return 'String similarity';
return undefined;
});
const result = await metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0);
expect(result).toEqual({ 'String similarity': 1 });
});
it('should handle whitespace trimming', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return ' hello ';
if (paramName === 'actualAnswer') return 'hello';
if (paramName === 'options.metricName') return 'String similarity';
return undefined;
});
const result = await metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0);
expect(result).toEqual({ 'String similarity': 1 });
});
it('should return low similarity for very different strings', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'hello';
if (paramName === 'actualAnswer') return 'world';
if (paramName === 'options.metricName') return 'String similarity';
return undefined;
});
const result = await metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0);
// Edit distance is 4, longer string length is 5, so similarity = 1 - (4/5) = 0.2
expect(result['String similarity']).toBeCloseTo(0.2, 2);
});
it('should handle different string lengths', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'hello';
if (paramName === 'actualAnswer') return 'hello world';
if (paramName === 'options.metricName') return 'String similarity';
return undefined;
});
const result = await metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0);
// Edit distance is 6, longer string length is 11, so similarity = 1 - (6/11) ≈ 0.45
expect(result['String similarity']).toBeCloseTo(0.45, 2);
});
it('should throw error for missing expected answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return '';
if (paramName === 'actualAnswer') return 'actual answer';
return undefined;
});
await expect(metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for missing actual answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'expected answer';
if (paramName === 'actualAnswer') return '';
return undefined;
});
await expect(metricHandlers.stringSimilarity.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
});
describe('helpfulness', () => {
let mockLLM: jest.Mocked<BaseLanguageModel>;
beforeEach(() => {
mockLLM = mock<BaseLanguageModel>();
mockExecuteFunctions.getInputConnectionData.mockResolvedValue(mockLLM);
});
it('should return helpfulness score from LLM', async () => {
const mockResponse = {
extended_reasoning: 'The response is very helpful...',
reasoning_summary: 'Response directly addresses the query',
score: 4,
};
// Mock the LLM with withStructuredOutput
const mockLLMWithStructuredOutput = mock<Runnable>();
mockLLMWithStructuredOutput.invoke.mockResolvedValue(mockResponse);
mockLLM.withStructuredOutput = jest.fn().mockReturnValue(mockLLMWithStructuredOutput);
// Mock ChatPromptTemplate.fromMessages to return a chain that can be piped
const mockChatPromptTemplate = mock<ChatPromptTemplate>();
mockChatPromptTemplate.pipe.mockReturnValue(mockLLMWithStructuredOutput);
// Mock the static method
jest.spyOn(ChatPromptTemplate, 'fromMessages').mockReturnValue(mockChatPromptTemplate);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'userQuery') return 'What is the capital of France?';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
if (paramName === 'prompt') return 'You are an AI assistant...';
if (paramName === 'options.inputPrompt')
return 'Query: {user_query}\\nResponse: {actual_answer}';
if (paramName === 'options.metricName') return 'Helpfulness';
return undefined;
});
const result = await metricHandlers.helpfulness.call(mockExecuteFunctions, 0);
expect(result).toEqual({ Helpfulness: 4 });
});
it('should throw error for missing user query', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'userQuery') return '';
if (paramName === 'actualAnswer') return 'Some response';
return undefined;
});
await expect(metricHandlers.helpfulness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for missing actual answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'userQuery') return 'Some query';
if (paramName === 'actualAnswer') return '';
return undefined;
});
await expect(metricHandlers.helpfulness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error when no LLM is connected', async () => {
mockExecuteFunctions.getInputConnectionData.mockResolvedValue(null);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'userQuery') return 'What is the capital of France?';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
return undefined;
});
await expect(metricHandlers.helpfulness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should handle LLM errors gracefully', async () => {
const mockError = new Error('LLM processing failed');
const mockFinalChain = mock<Runnable>();
mockFinalChain.invoke.mockRejectedValue(mockError);
const mockMiddleChain = mock<Runnable>();
mockMiddleChain.pipe.mockReturnValue(mockFinalChain);
const mockChatPromptTemplate = mock<ChatPromptTemplate>();
mockChatPromptTemplate.pipe.mockReturnValue(mockMiddleChain);
jest.spyOn(ChatPromptTemplate, 'fromMessages').mockReturnValue(mockChatPromptTemplate);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'userQuery') return 'What is the capital of France?';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
if (paramName === 'prompt') return 'You are an AI assistant...';
if (paramName === 'options.inputPrompt')
return 'Query: {user_query}\\nResponse: {actual_answer}';
if (paramName === 'options.metricName') return 'Helpfulness';
return undefined;
});
await expect(metricHandlers.helpfulness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
});
describe('correctness', () => {
let mockLLM: jest.Mocked<BaseLanguageModel>;
beforeEach(() => {
mockLLM = mock<BaseLanguageModel>();
mockExecuteFunctions.getInputConnectionData.mockResolvedValue(mockLLM);
});
it('should return correctness score from LLM', async () => {
const mockResponse = {
extended_reasoning: 'The response is factually correct...',
reasoning_summary: 'Response matches expected answer',
score: 5,
};
// Mock the LLM with withStructuredOutput
const mockLLMWithStructuredOutput = mock<Runnable>();
mockLLMWithStructuredOutput.invoke.mockResolvedValue(mockResponse);
mockLLM.withStructuredOutput = jest.fn().mockReturnValue(mockLLMWithStructuredOutput);
const mockChatPromptTemplate = mock<ChatPromptTemplate>();
mockChatPromptTemplate.pipe.mockReturnValue(mockLLMWithStructuredOutput);
jest.spyOn(ChatPromptTemplate, 'fromMessages').mockReturnValue(mockChatPromptTemplate);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'Paris';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
if (paramName === 'prompt') return 'You are an AI assistant...';
if (paramName === 'options.inputPrompt')
return 'Expected: {expected_answer}\\nActual: {actual_answer}';
if (paramName === 'options.metricName') return 'Correctness';
return undefined;
});
const result = await metricHandlers.correctness.call(mockExecuteFunctions, 0);
expect(result).toEqual({ Correctness: 5 });
});
it('should throw error for missing expected answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return '';
if (paramName === 'actualAnswer') return 'Some response';
return undefined;
});
await expect(metricHandlers.correctness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error for missing actual answer', async () => {
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'Expected answer';
if (paramName === 'actualAnswer') return '';
return undefined;
});
await expect(metricHandlers.correctness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should throw error when no LLM is connected', async () => {
mockExecuteFunctions.getInputConnectionData.mockResolvedValue(null);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'Paris';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
return undefined;
});
await expect(metricHandlers.correctness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
it('should handle LLM errors gracefully', async () => {
const mockError = new Error('LLM processing failed');
const mockFinalChain = mock<Runnable>();
mockFinalChain.invoke.mockRejectedValue(mockError);
const mockMiddleChain = mock<Runnable>();
mockMiddleChain.pipe.mockReturnValue(mockFinalChain);
const mockChatPromptTemplate = mock<ChatPromptTemplate>();
mockChatPromptTemplate.pipe.mockReturnValue(mockMiddleChain);
jest.spyOn(ChatPromptTemplate, 'fromMessages').mockReturnValue(mockChatPromptTemplate);
mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
if (paramName === 'expectedAnswer') return 'Paris';
if (paramName === 'actualAnswer') return 'Paris is the capital of France.';
if (paramName === 'prompt') return 'You are an AI assistant...';
if (paramName === 'options.inputPrompt')
return 'Expected: {expected_answer}\\nActual: {actual_answer}';
if (paramName === 'options.metricName') return 'Correctness';
return undefined;
});
await expect(metricHandlers.correctness.call(mockExecuteFunctions, 0)).rejects.toThrow(
NodeOperationError,
);
});
});
});