first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
import { defineConfig } from 'eslint/config';
|
||||
import { baseConfig } from '@n8n/eslint-config/base';
|
||||
|
||||
export default defineConfig(
|
||||
baseConfig,
|
||||
{
|
||||
rules: {
|
||||
'unicorn/filename-case': ['error', { case: 'kebabCase' }],
|
||||
complexity: 'error',
|
||||
|
||||
// TODO: Remove this
|
||||
'@typescript-eslint/naming-convention': 'warn',
|
||||
'@typescript-eslint/no-require-imports': 'warn',
|
||||
'@typescript-eslint/require-await': 'warn',
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['**/*.test.ts'],
|
||||
rules: {
|
||||
'n8n-local-rules/no-uncaught-json-parse': 'warn',
|
||||
'import-x/no-duplicates': 'warn',
|
||||
'@typescript-eslint/unbound-method': 'warn',
|
||||
'@typescript-eslint/no-unsafe-argument': 'warn',
|
||||
'@typescript-eslint/no-unsafe-member-access': 'warn',
|
||||
'@typescript-eslint/no-unsafe-assignment': 'warn',
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
/** @type {import('jest').Config} */
|
||||
module.exports = {
|
||||
...require('../../../jest.config'),
|
||||
testTimeout: 10_000,
|
||||
};
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"name": "@n8n/task-runner",
|
||||
"version": "2.11.0",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"start": "node dist/start.js",
|
||||
"dev": "pnpm build && pnpm start",
|
||||
"debug": "./scripts/debug.sh",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"build": "tsc -p ./tsconfig.build.json && tsc-alias -p tsconfig.build.json",
|
||||
"format": "biome format --write src",
|
||||
"format:check": "biome ci src",
|
||||
"test": "jest",
|
||||
"test:unit": "jest",
|
||||
"test:watch": "jest --watch",
|
||||
"lint": "eslint . --quiet",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\""
|
||||
},
|
||||
"main": "dist/start.js",
|
||||
"module": "src/start.ts",
|
||||
"types": "dist/index.d.ts",
|
||||
"files": [
|
||||
"dist/**/*"
|
||||
],
|
||||
"exports": {
|
||||
"./start": {
|
||||
"require": "./dist/start.js",
|
||||
"import": "./src/start.ts",
|
||||
"types": "./dist/start.d.ts"
|
||||
},
|
||||
".": {
|
||||
"require": "./dist/index.js",
|
||||
"import": "./src/index.ts",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@n8n/config": "workspace:*",
|
||||
"@n8n/di": "workspace:*",
|
||||
"@n8n/errors": "workspace:*",
|
||||
"@sentry/node": "catalog:sentry",
|
||||
"acorn": "8.14.0",
|
||||
"acorn-walk": "8.3.4",
|
||||
"lodash": "catalog:",
|
||||
"luxon": "catalog:",
|
||||
"n8n-core": "workspace:*",
|
||||
"n8n-workflow": "workspace:*",
|
||||
"nanoid": "catalog:",
|
||||
"ws": "^8.18.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@n8n/typescript-config": "workspace:*",
|
||||
"@types/lodash": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# For debugging only, start the runner with a manually fetched grant token. If no broker, wait until available.
|
||||
|
||||
for i in {1..30}; do
|
||||
GRANT_TOKEN=$(curl -s -X POST http://127.0.0.1:5679/runners/auth -H "Content-Type: application/json" -d '{"token":"test"}' | jq -r '.data.token')
|
||||
if [ -n "$GRANT_TOKEN" ] && [ "$GRANT_TOKEN" != "null" ]; then
|
||||
N8N_RUNNERS_GRANT_TOKEN="$GRANT_TOKEN" pnpm start
|
||||
exit 0
|
||||
fi
|
||||
[ $i -eq 1 ] && echo "Waiting for n8n task broker server at http://127.0.0.1:5679..."
|
||||
sleep 1
|
||||
done
|
||||
echo "Error: Could not connect to n8n task broker server after 30 seconds"
|
||||
exit 1
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { INodeTypeDescription } from 'n8n-workflow';
|
||||
|
||||
import { TaskRunnerNodeTypes } from '../node-types';
|
||||
|
||||
const SINGLE_VERSIONED = { name: 'single-versioned', version: 1 };
|
||||
|
||||
const SINGLE_UNVERSIONED = { name: 'single-unversioned' };
|
||||
|
||||
const MULTI_VERSIONED = { name: 'multi-versioned', version: [1, 2] };
|
||||
|
||||
const SPLIT_VERSIONED = [
|
||||
{ name: 'split-versioned', version: 1 },
|
||||
{ name: 'split-versioned', version: 2 },
|
||||
];
|
||||
|
||||
const TYPES: INodeTypeDescription[] = [
|
||||
SINGLE_VERSIONED,
|
||||
SINGLE_UNVERSIONED,
|
||||
MULTI_VERSIONED,
|
||||
...SPLIT_VERSIONED,
|
||||
] as INodeTypeDescription[];
|
||||
|
||||
describe('TaskRunnerNodeTypes', () => {
|
||||
describe('getByNameAndVersion', () => {
|
||||
let nodeTypes: TaskRunnerNodeTypes;
|
||||
|
||||
beforeEach(() => {
|
||||
nodeTypes = new TaskRunnerNodeTypes(TYPES);
|
||||
});
|
||||
|
||||
it('should return undefined if not found', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('unknown', 1)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return highest versioned node type if no version is given', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('split-versioned')).toEqual({
|
||||
description: SPLIT_VERSIONED[1],
|
||||
});
|
||||
});
|
||||
|
||||
it('should return specified version for split version', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('split-versioned', 1)).toEqual({
|
||||
description: SPLIT_VERSIONED[0],
|
||||
});
|
||||
});
|
||||
|
||||
it('should return undefined on unknown version', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('split-versioned', 3)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return specified version for multi version', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('multi-versioned', 1)).toEqual({
|
||||
description: MULTI_VERSIONED,
|
||||
});
|
||||
expect(nodeTypes.getByNameAndVersion('multi-versioned', 2)).toEqual({
|
||||
description: MULTI_VERSIONED,
|
||||
});
|
||||
});
|
||||
|
||||
it('should default to DEFAULT_NODETYPE_VERSION if no version specified', () => {
|
||||
expect(nodeTypes.getByNameAndVersion('single-unversioned', 1)).toEqual({
|
||||
description: SINGLE_UNVERSIONED,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('addNodeTypeDescriptions', () => {
|
||||
it('should add new node types', () => {
|
||||
const nodeTypes = new TaskRunnerNodeTypes(TYPES);
|
||||
|
||||
const nodeTypeDescriptions = [
|
||||
{ name: 'new-type', version: 1 },
|
||||
{ name: 'new-type', version: 2 },
|
||||
] as INodeTypeDescription[];
|
||||
|
||||
nodeTypes.addNodeTypeDescriptions(nodeTypeDescriptions);
|
||||
|
||||
expect(nodeTypes.getByNameAndVersion('new-type', 1)).toEqual({
|
||||
description: { name: 'new-type', version: 1 },
|
||||
});
|
||||
expect(nodeTypes.getByNameAndVersion('new-type', 2)).toEqual({
|
||||
description: { name: 'new-type', version: 2 },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('onlyUnknown', () => {
|
||||
it('should return only unknown node types', () => {
|
||||
const nodeTypes = new TaskRunnerNodeTypes(TYPES);
|
||||
|
||||
const candidate = { name: 'unknown', version: 1 };
|
||||
|
||||
expect(nodeTypes.onlyUnknown([candidate])).toEqual([candidate]);
|
||||
expect(nodeTypes.onlyUnknown([SINGLE_VERSIONED])).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,331 @@
|
||||
import type { ErrorEvent } from '@sentry/core';
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type { ErrorReporter } from 'n8n-core';
|
||||
|
||||
import { TaskRunnerSentry } from '../task-runner-sentry';
|
||||
|
||||
describe('TaskRunnerSentry', () => {
|
||||
const commonConfig = {
|
||||
n8nVersion: '1.0.0',
|
||||
environment: 'local',
|
||||
deploymentName: 'test',
|
||||
profilesSampleRate: 0,
|
||||
tracesSampleRate: 0,
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
describe('filterOutUserCodeErrors', () => {
|
||||
const sentry = new TaskRunnerSentry(
|
||||
{
|
||||
...commonConfig,
|
||||
dsn: 'https://sentry.io/123',
|
||||
},
|
||||
mock(),
|
||||
);
|
||||
|
||||
it('should filter out user code errors with node:vm frame', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'ReferenceError',
|
||||
value: 'fetch is not defined',
|
||||
stacktrace: {
|
||||
frames: [
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'JsTaskRunner.executeTask',
|
||||
},
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'JsTaskRunner.runForAllItems',
|
||||
},
|
||||
{
|
||||
filename: '<anonymous>',
|
||||
module: '<anonymous>',
|
||||
function: 'new Promise',
|
||||
},
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'result',
|
||||
},
|
||||
{
|
||||
filename: 'node:vm',
|
||||
module: 'node:vm',
|
||||
function: 'runInContext',
|
||||
},
|
||||
{
|
||||
filename: 'node:vm',
|
||||
module: 'node:vm',
|
||||
function: 'Script.runInContext',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
function: '?',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
function: 'VmCodeWrapper',
|
||||
},
|
||||
{
|
||||
filename: '<anonymous>',
|
||||
module: '<anonymous>',
|
||||
function: 'new Promise',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
},
|
||||
],
|
||||
},
|
||||
mechanism: { type: 'onunhandledrejection', handled: false },
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfd8',
|
||||
platform: 'node',
|
||||
contexts: {
|
||||
trace: { trace_id: '3c3614d33a6b47f09b85ec7d2710acea', span_id: 'ad00fdf6d6173aeb' },
|
||||
runtime: { name: 'node', version: 'v20.17.0' },
|
||||
},
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(true);
|
||||
});
|
||||
|
||||
it('should filter out user code errors with only evalmachine frames', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'Error',
|
||||
value: 'User code error',
|
||||
stacktrace: {
|
||||
frames: [
|
||||
{
|
||||
filename: '<anonymous>',
|
||||
module: '<anonymous>',
|
||||
function: 'new Promise',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
function: 'toDataUrl',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
function: 'loadImages',
|
||||
},
|
||||
],
|
||||
},
|
||||
mechanism: { type: 'onunhandledrejection', handled: false },
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfd9',
|
||||
platform: 'node',
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(true);
|
||||
});
|
||||
|
||||
it('should filter out user code errors with VmCodeWrapper frame', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'TypeError',
|
||||
value: 'Cannot read property of undefined',
|
||||
stacktrace: {
|
||||
frames: [
|
||||
{
|
||||
filename: '<anonymous>',
|
||||
module: '<anonymous>',
|
||||
function: 'new Promise',
|
||||
},
|
||||
{
|
||||
filename: 'evalmachine.<anonymous>',
|
||||
module: 'evalmachine.<anonymous>',
|
||||
function: 'VmCodeWrapper',
|
||||
},
|
||||
],
|
||||
},
|
||||
mechanism: { type: 'onunhandledrejection', handled: false },
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfda',
|
||||
platform: 'node',
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(true);
|
||||
});
|
||||
|
||||
it('should filter out EvalError from disallowed code generation', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'EvalError',
|
||||
value: 'Code generation from strings disallowed for this context',
|
||||
stacktrace: {
|
||||
frames: [
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'JsTaskRunner.executeTask',
|
||||
},
|
||||
],
|
||||
},
|
||||
mechanism: { type: 'generic', handled: true },
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfdd',
|
||||
platform: 'node',
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(true);
|
||||
});
|
||||
|
||||
it('should not filter out task runner errors', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'Error',
|
||||
value: 'Task runner internal error',
|
||||
stacktrace: {
|
||||
frames: [
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'JsTaskRunner.executeTask',
|
||||
},
|
||||
{
|
||||
filename: 'app:///dist/js-task-runner/js-task-runner.js',
|
||||
module: 'js-task-runner:js-task-runner',
|
||||
function: 'JsTaskRunner.runForAllItems',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfdb',
|
||||
platform: 'node',
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(false);
|
||||
});
|
||||
|
||||
it('should not filter out errors without stacktrace', () => {
|
||||
const event: ErrorEvent = {
|
||||
type: undefined,
|
||||
exception: {
|
||||
values: [
|
||||
{
|
||||
type: 'Error',
|
||||
value: 'Error without stacktrace',
|
||||
},
|
||||
],
|
||||
},
|
||||
event_id: '18bb78bb3d9d44c4acf3d774c2cfbfdc',
|
||||
platform: 'node',
|
||||
};
|
||||
|
||||
expect(sentry.filterOutUserCodeErrors(event)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('initIfEnabled', () => {
|
||||
const mockErrorReporter = mock<ErrorReporter>();
|
||||
|
||||
it('should not configure sentry if dsn is not set', async () => {
|
||||
const sentry = new TaskRunnerSentry(
|
||||
{
|
||||
...commonConfig,
|
||||
dsn: '',
|
||||
},
|
||||
mockErrorReporter,
|
||||
);
|
||||
|
||||
await sentry.initIfEnabled();
|
||||
|
||||
expect(mockErrorReporter.init).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should configure sentry if dsn is set', async () => {
|
||||
const sentry = new TaskRunnerSentry(
|
||||
{
|
||||
...commonConfig,
|
||||
dsn: 'https://sentry.io/123',
|
||||
},
|
||||
mockErrorReporter,
|
||||
);
|
||||
|
||||
await sentry.initIfEnabled();
|
||||
|
||||
expect(mockErrorReporter.init).toHaveBeenCalledWith({
|
||||
dsn: 'https://sentry.io/123',
|
||||
beforeSendFilter: sentry.filterOutUserCodeErrors,
|
||||
release: 'n8n@1.0.0',
|
||||
environment: 'local',
|
||||
serverName: 'test',
|
||||
serverType: 'task_runner',
|
||||
withEventLoopBlockDetection: false,
|
||||
profilesSampleRate: 0,
|
||||
tracesSampleRate: 0,
|
||||
eligibleIntegrations: {
|
||||
Http: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('shutdown', () => {
|
||||
const mockErrorReporter = mock<ErrorReporter>();
|
||||
|
||||
it('should not shutdown sentry if dsn is not set', async () => {
|
||||
const sentry = new TaskRunnerSentry(
|
||||
{
|
||||
...commonConfig,
|
||||
dsn: '',
|
||||
},
|
||||
mockErrorReporter,
|
||||
);
|
||||
|
||||
await sentry.shutdown();
|
||||
|
||||
expect(mockErrorReporter.shutdown).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should shutdown sentry if dsn is set', async () => {
|
||||
const sentry = new TaskRunnerSentry(
|
||||
{
|
||||
...commonConfig,
|
||||
dsn: 'https://sentry.io/123',
|
||||
},
|
||||
mockErrorReporter,
|
||||
);
|
||||
|
||||
await sentry.shutdown();
|
||||
|
||||
expect(mockErrorReporter.shutdown).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Config, Env, Nested } from '@n8n/config';
|
||||
|
||||
@Config
|
||||
class HealthcheckServerConfig {
|
||||
@Env('N8N_RUNNERS_HEALTH_CHECK_SERVER_ENABLED')
|
||||
enabled: boolean = false;
|
||||
|
||||
@Env('N8N_RUNNERS_HEALTH_CHECK_SERVER_HOST')
|
||||
host: string = '127.0.0.1';
|
||||
|
||||
@Env('N8N_RUNNERS_HEALTH_CHECK_SERVER_PORT')
|
||||
port: number = 5681;
|
||||
}
|
||||
|
||||
@Config
|
||||
export class BaseRunnerConfig {
|
||||
@Env('N8N_RUNNERS_TASK_BROKER_URI')
|
||||
taskBrokerUri: string = 'http://127.0.0.1:5679';
|
||||
|
||||
@Env('N8N_RUNNERS_GRANT_TOKEN')
|
||||
grantToken: string = '';
|
||||
|
||||
@Env('N8N_RUNNERS_MAX_PAYLOAD')
|
||||
maxPayloadSize: number = 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* How many concurrent tasks can a runner execute at a time
|
||||
*
|
||||
* Kept high for backwards compatibility - n8n v2 will reduce this to `5`
|
||||
*/
|
||||
@Env('N8N_RUNNERS_MAX_CONCURRENCY')
|
||||
maxConcurrency: number = 10;
|
||||
|
||||
/**
|
||||
* How long (in seconds) a runner may be idle for before exit. Intended
|
||||
* for use in `external` mode - launcher must pass the env var when launching
|
||||
* the runner. Disabled with `0` on `internal` mode.
|
||||
*/
|
||||
@Env('N8N_RUNNERS_AUTO_SHUTDOWN_TIMEOUT')
|
||||
idleTimeout: number = 0;
|
||||
|
||||
@Env('GENERIC_TIMEZONE')
|
||||
timezone: string = 'America/New_York';
|
||||
|
||||
/**
|
||||
* How long (in seconds) a task is allowed to take for completion, else the
|
||||
* task will be aborted. (In internal mode, the runner will also be
|
||||
* restarted.) Must be greater than 0.
|
||||
*
|
||||
* Kept high for backwards compatibility - n8n v2 will reduce this to `60`
|
||||
*/
|
||||
@Env('N8N_RUNNERS_TASK_TIMEOUT')
|
||||
taskTimeout: number = 300; // 5 minutes
|
||||
|
||||
@Nested
|
||||
healthcheckServer!: HealthcheckServerConfig;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Config, Env } from '@n8n/config';
|
||||
|
||||
@Config
|
||||
export class JsRunnerConfig {
|
||||
@Env('NODE_FUNCTION_ALLOW_BUILTIN')
|
||||
allowedBuiltInModules: string = '';
|
||||
|
||||
@Env('NODE_FUNCTION_ALLOW_EXTERNAL')
|
||||
allowedExternalModules: string = '';
|
||||
|
||||
@Env('N8N_RUNNERS_INSECURE_MODE')
|
||||
insecureMode: boolean = false;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Config, Nested } from '@n8n/config';
|
||||
|
||||
import { BaseRunnerConfig } from './base-runner-config';
|
||||
import { JsRunnerConfig } from './js-runner-config';
|
||||
import { SentryConfig } from './sentry-config';
|
||||
|
||||
@Config
|
||||
export class MainConfig {
|
||||
@Nested
|
||||
baseRunnerConfig!: BaseRunnerConfig;
|
||||
|
||||
@Nested
|
||||
jsRunnerConfig!: JsRunnerConfig;
|
||||
|
||||
@Nested
|
||||
sentryConfig!: SentryConfig;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { Config, Env, sampleRateSchema } from '@n8n/config';
|
||||
|
||||
@Config
|
||||
export class SentryConfig {
|
||||
/** Sentry DSN (data source name) */
|
||||
@Env('N8N_SENTRY_DSN')
|
||||
dsn: string = '';
|
||||
|
||||
/**
|
||||
* Sample rate for Sentry profiling (0.0 to 1.0).
|
||||
* This determines what percentage of transactions are profiled.
|
||||
*
|
||||
* @default 0 (disabled)
|
||||
*/
|
||||
@Env('N8N_SENTRY_PROFILES_SAMPLE_RATE', sampleRateSchema)
|
||||
profilesSampleRate: number = 0;
|
||||
|
||||
/**
|
||||
* Sample rate for Sentry traces (0.0 to 1.0).
|
||||
* This determines what percentage of transactions are profiled.
|
||||
*
|
||||
* @default 0 (disabled)
|
||||
*/
|
||||
@Env('N8N_SENTRY_TRACES_SAMPLE_RATE', sampleRateSchema)
|
||||
tracesSampleRate: number = 0;
|
||||
|
||||
//#region Metadata about the environment
|
||||
|
||||
@Env('N8N_VERSION')
|
||||
n8nVersion: string = '';
|
||||
|
||||
@Env('ENVIRONMENT')
|
||||
environment: string = '';
|
||||
|
||||
@Env('DEPLOYMENT_NAME')
|
||||
deploymentName: string = '';
|
||||
|
||||
//#endregion
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type {
|
||||
IExecuteData,
|
||||
INode,
|
||||
INodeExecutionData,
|
||||
ITaskDataConnectionsSource,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import type { DataRequestResponse, InputDataChunkDefinition } from '@/runner-types';
|
||||
|
||||
import { DataRequestResponseReconstruct } from '../data-request-response-reconstruct';
|
||||
|
||||
describe('DataRequestResponseReconstruct', () => {
|
||||
const reconstruct = new DataRequestResponseReconstruct();
|
||||
|
||||
describe('reconstructConnectionInputItems', () => {
|
||||
it('should return all input items if no chunk is provided', () => {
|
||||
const inputData: DataRequestResponse['inputData'] = {
|
||||
main: [[{ json: { key: 'value' } }]],
|
||||
};
|
||||
|
||||
const result = reconstruct.reconstructConnectionInputItems(inputData);
|
||||
|
||||
expect(result).toEqual([{ json: { key: 'value' } }]);
|
||||
});
|
||||
|
||||
it('should reconstruct sparse array when chunk is provided', () => {
|
||||
const inputData: DataRequestResponse['inputData'] = {
|
||||
main: [[{ json: { key: 'chunked' } }]],
|
||||
};
|
||||
const chunk: InputDataChunkDefinition = { startIndex: 2, count: 1 };
|
||||
|
||||
const result = reconstruct.reconstructConnectionInputItems(inputData, chunk);
|
||||
|
||||
expect(result).toEqual([undefined, undefined, { json: { key: 'chunked' } }, undefined]);
|
||||
});
|
||||
|
||||
it('should handle empty input data gracefully', () => {
|
||||
const inputData: DataRequestResponse['inputData'] = { main: [[]] };
|
||||
const chunk: InputDataChunkDefinition = { startIndex: 1, count: 1 };
|
||||
|
||||
const result = reconstruct.reconstructConnectionInputItems(inputData, chunk);
|
||||
|
||||
expect(result).toEqual([undefined]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reconstructExecuteData', () => {
|
||||
it('should reconstruct execute data with the provided input items', () => {
|
||||
const node = mock<INode>();
|
||||
const connectionInputSource = mock<ITaskDataConnectionsSource>();
|
||||
const response = mock<DataRequestResponse>({
|
||||
inputData: { main: [[]] },
|
||||
node,
|
||||
connectionInputSource,
|
||||
});
|
||||
const inputItems: INodeExecutionData[] = [{ json: { key: 'reconstructed' } }];
|
||||
|
||||
const result = reconstruct.reconstructExecuteData(response, inputItems);
|
||||
|
||||
expect(result).toEqual<IExecuteData>({
|
||||
data: {
|
||||
main: [inputItems],
|
||||
},
|
||||
node: response.node,
|
||||
source: response.connectionInputSource,
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle empty input items gracefully', () => {
|
||||
const node = mock<INode>();
|
||||
const connectionInputSource = mock<ITaskDataConnectionsSource>();
|
||||
const inputItems: INodeExecutionData[] = [];
|
||||
const response = mock<DataRequestResponse>({
|
||||
inputData: { main: [[{ json: { key: 'value' } }]] },
|
||||
node,
|
||||
connectionInputSource,
|
||||
});
|
||||
|
||||
const result = reconstruct.reconstructExecuteData(response, inputItems);
|
||||
|
||||
expect(result).toEqual<IExecuteData>({
|
||||
data: {
|
||||
main: [inputItems],
|
||||
},
|
||||
node: response.node,
|
||||
source: response.connectionInputSource,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { IExecuteData, INodeExecutionData, ITaskDataConnections } from 'n8n-workflow';
|
||||
|
||||
import type { DataRequestResponse, InputDataChunkDefinition } from '@/runner-types';
|
||||
|
||||
/**
|
||||
* Reconstructs data from a DataRequestResponse to the initial
|
||||
* data structures.
|
||||
*/
|
||||
export class DataRequestResponseReconstruct {
|
||||
/**
|
||||
* Reconstructs `inputData` from a DataRequestResponse
|
||||
*/
|
||||
reconstructConnectionInputItems(
|
||||
inputData: DataRequestResponse['inputData'],
|
||||
chunk?: InputDataChunkDefinition,
|
||||
): Array<INodeExecutionData | undefined> {
|
||||
const inputItems = inputData?.main?.[0] ?? [];
|
||||
if (!chunk) {
|
||||
return inputItems;
|
||||
}
|
||||
|
||||
// Only a chunk of the input items was requested. We reconstruct
|
||||
// the array by filling in the missing items with `undefined`.
|
||||
let sparseInputItems: Array<INodeExecutionData | undefined> = [];
|
||||
|
||||
sparseInputItems = sparseInputItems
|
||||
.concat(Array.from({ length: chunk.startIndex }))
|
||||
.concat(inputItems)
|
||||
.concat(Array.from({ length: inputItems.length - chunk.startIndex - chunk.count }));
|
||||
|
||||
return sparseInputItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct `executeData` from a DataRequestResponse
|
||||
*/
|
||||
reconstructExecuteData(
|
||||
response: DataRequestResponse,
|
||||
inputItems: INodeExecutionData[],
|
||||
): IExecuteData {
|
||||
const inputData: ITaskDataConnections = {
|
||||
...response.inputData,
|
||||
main: [inputItems],
|
||||
};
|
||||
|
||||
return {
|
||||
data: inputData,
|
||||
node: response.node,
|
||||
source: response.connectionInputSource,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { ApplicationError } from '@n8n/errors';
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
export class HealthCheckServer {
|
||||
private server = createServer((_, res) => {
|
||||
res.writeHead(200);
|
||||
res.end('OK');
|
||||
});
|
||||
|
||||
async start(host: string, port: number) {
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
const portInUseErrorHandler = (error: NodeJS.ErrnoException) => {
|
||||
if (error.code === 'EADDRINUSE') {
|
||||
reject(new ApplicationError(`Port ${port} is already in use`));
|
||||
} else {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
|
||||
this.server.on('error', portInUseErrorHandler);
|
||||
|
||||
this.server.listen(port, host, () => {
|
||||
this.server.removeListener('error', portInUseErrorHandler);
|
||||
console.log(`Health check server listening on ${host}, port ${port}`);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async stop() {
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
this.server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from './task-runner';
|
||||
export * from './runner-types';
|
||||
export type * from './message-types';
|
||||
export * from './data-request/data-request-response-reconstruct';
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
import { ExecutionError } from '@/js-task-runner/errors/execution-error';
|
||||
|
||||
import { DisallowedModuleError } from '../errors/disallowed-module.error';
|
||||
import { createRequireResolver, type RequireResolverOpts } from '../require-resolver';
|
||||
|
||||
describe('require resolver', () => {
|
||||
let defaultOpts: RequireResolverOpts;
|
||||
|
||||
beforeEach(() => {
|
||||
defaultOpts = {
|
||||
allowedBuiltInModules: new Set(['path', 'fs']),
|
||||
allowedExternalModules: new Set(['lodash']),
|
||||
};
|
||||
});
|
||||
|
||||
describe('built-in modules', () => {
|
||||
it('should allow requiring whitelisted built-in modules', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('path')).not.toThrow();
|
||||
expect(() => resolver('fs')).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw when requiring non-whitelisted built-in modules', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('crypto')).toThrow(ExecutionError);
|
||||
});
|
||||
|
||||
it('should allow all built-in modules when allowedBuiltInModules is "*"', () => {
|
||||
const resolver = createRequireResolver({
|
||||
...defaultOpts,
|
||||
allowedBuiltInModules: '*',
|
||||
});
|
||||
|
||||
expect(() => resolver('path')).not.toThrow();
|
||||
expect(() => resolver('crypto')).not.toThrow();
|
||||
expect(() => resolver('fs')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('external modules', () => {
|
||||
it('should allow requiring whitelisted external modules', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('lodash')).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw when requiring non-allowlisted external modules', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('express')).toThrow(
|
||||
new ExecutionError(new DisallowedModuleError('express')),
|
||||
);
|
||||
});
|
||||
|
||||
it('should allow all external modules when allowedExternalModules is "*"', () => {
|
||||
const resolver = createRequireResolver({
|
||||
...defaultOpts,
|
||||
allowedExternalModules: '*',
|
||||
});
|
||||
|
||||
expect(() => resolver('lodash')).not.toThrow();
|
||||
expect(() => resolver('express')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('should wrap DisallowedModuleError in ExecutionError', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('non-existent-module')).toThrow(ExecutionError);
|
||||
});
|
||||
|
||||
it('should include the module name in the error message', () => {
|
||||
const resolver = createRequireResolver(defaultOpts);
|
||||
expect(() => resolver('non-existent-module')).toThrow(
|
||||
"Module 'non-existent-module' is disallowed",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,353 @@
|
||||
import { WebSocket } from 'ws';
|
||||
|
||||
import { newTaskState } from '@/js-task-runner/__tests__/test-data';
|
||||
import { TimeoutError } from '@/js-task-runner/errors/timeout-error';
|
||||
import { TaskRunner, type TaskRunnerOpts } from '@/task-runner';
|
||||
import type { TaskStatus } from '@/task-state';
|
||||
|
||||
class TestRunner extends TaskRunner {}
|
||||
|
||||
jest.mock('ws');
|
||||
|
||||
describe('TestRunner', () => {
|
||||
let runner: TestRunner;
|
||||
|
||||
const newTestRunner = (opts: Partial<TaskRunnerOpts> = {}) =>
|
||||
new TestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 5,
|
||||
idleTimeout: 60,
|
||||
grantToken: 'test-token',
|
||||
maxPayloadSize: 1024,
|
||||
taskBrokerUri: 'http://localhost:8080',
|
||||
timezone: 'America/New_York',
|
||||
taskTimeout: 60,
|
||||
healthcheckServer: {
|
||||
enabled: false,
|
||||
host: 'localhost',
|
||||
port: 8081,
|
||||
},
|
||||
...opts,
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
runner?.clearIdleTimer();
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should correctly construct WebSocket URI with provided taskBrokerUri', () => {
|
||||
runner = newTestRunner({
|
||||
taskBrokerUri: 'http://localhost:8080',
|
||||
});
|
||||
|
||||
expect(WebSocket).toHaveBeenCalledWith(
|
||||
`ws://localhost:8080/runners/_ws?id=${runner.id}`,
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
authorization: 'Bearer test-token',
|
||||
},
|
||||
maxPayload: 1024,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle different taskBrokerUri formats correctly', () => {
|
||||
runner = newTestRunner({
|
||||
taskBrokerUri: 'https://example.com:3000/path',
|
||||
});
|
||||
|
||||
expect(WebSocket).toHaveBeenCalledWith(
|
||||
`ws://example.com:3000/runners/_ws?id=${runner.id}`,
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
authorization: 'Bearer test-token',
|
||||
},
|
||||
maxPayload: 1024,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw an error if taskBrokerUri is invalid', () => {
|
||||
expect(() =>
|
||||
newTestRunner({
|
||||
taskBrokerUri: 'not-a-valid-uri',
|
||||
}),
|
||||
).toThrowError(/Invalid URL/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sendOffers', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
it('should not send offers if canSendOffers is false', () => {
|
||||
runner = newTestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 2,
|
||||
});
|
||||
const sendSpy = jest.spyOn(runner, 'send');
|
||||
expect(runner.canSendOffers).toBe(false);
|
||||
|
||||
runner.sendOffers();
|
||||
|
||||
expect(sendSpy).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should enable sending of offer on runnerregistered message', () => {
|
||||
runner = newTestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 2,
|
||||
});
|
||||
runner.onMessage({
|
||||
type: 'broker:runnerregistered',
|
||||
});
|
||||
|
||||
expect(runner.canSendOffers).toBe(true);
|
||||
});
|
||||
|
||||
it('should send maxConcurrency offers when there are no offers', () => {
|
||||
runner = newTestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 2,
|
||||
});
|
||||
runner.onMessage({
|
||||
type: 'broker:runnerregistered',
|
||||
});
|
||||
|
||||
const sendSpy = jest.spyOn(runner, 'send');
|
||||
|
||||
runner.sendOffers();
|
||||
runner.sendOffers();
|
||||
|
||||
expect(sendSpy).toHaveBeenCalledTimes(2);
|
||||
expect(sendSpy.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
type: 'runner:taskoffer',
|
||||
taskType: 'test-task',
|
||||
offerId: expect.any(String),
|
||||
validFor: expect.any(Number),
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'runner:taskoffer',
|
||||
taskType: 'test-task',
|
||||
offerId: expect.any(String),
|
||||
validFor: expect.any(Number),
|
||||
},
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should send up to maxConcurrency offers when there is a running task', () => {
|
||||
runner = newTestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 2,
|
||||
});
|
||||
runner.onMessage({
|
||||
type: 'broker:runnerregistered',
|
||||
});
|
||||
const taskState = newTaskState('test-task');
|
||||
runner.runningTasks.set('test-task', taskState);
|
||||
const sendSpy = jest.spyOn(runner, 'send');
|
||||
|
||||
runner.sendOffers();
|
||||
|
||||
expect(sendSpy).toHaveBeenCalledTimes(1);
|
||||
expect(sendSpy.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
type: 'runner:taskoffer',
|
||||
taskType: 'test-task',
|
||||
offerId: expect.any(String),
|
||||
validFor: expect.any(Number),
|
||||
},
|
||||
],
|
||||
]);
|
||||
taskState.cleanup();
|
||||
});
|
||||
|
||||
it('should delete stale offers and send new ones', () => {
|
||||
runner = newTestRunner({
|
||||
taskType: 'test-task',
|
||||
maxConcurrency: 2,
|
||||
});
|
||||
runner.onMessage({
|
||||
type: 'broker:runnerregistered',
|
||||
});
|
||||
|
||||
const sendSpy = jest.spyOn(runner, 'send');
|
||||
|
||||
runner.sendOffers();
|
||||
expect(sendSpy).toHaveBeenCalledTimes(2);
|
||||
sendSpy.mockClear();
|
||||
|
||||
jest.advanceTimersByTime(6000);
|
||||
runner.sendOffers();
|
||||
expect(sendSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('taskCancelled', () => {
|
||||
test.each<[TaskStatus, string]>([
|
||||
['aborting:cancelled', 'cancelled'],
|
||||
['aborting:timeout', 'timeout'],
|
||||
])('should not do anything if task status is %s', async (status, reason) => {
|
||||
runner = newTestRunner();
|
||||
|
||||
const taskId = 'test-task';
|
||||
const task = newTaskState(taskId);
|
||||
task.status = status;
|
||||
|
||||
runner.runningTasks.set(taskId, task);
|
||||
|
||||
await runner.taskCancelled(taskId, reason);
|
||||
|
||||
expect(runner.runningTasks.size).toBe(1);
|
||||
expect(task.status).toBe(status);
|
||||
});
|
||||
|
||||
it('should delete task if task is waiting for settings when task is cancelled', async () => {
|
||||
runner = newTestRunner();
|
||||
|
||||
const taskId = 'test-task';
|
||||
const task = newTaskState(taskId);
|
||||
const taskCleanupSpy = jest.spyOn(task, 'cleanup');
|
||||
runner.runningTasks.set(taskId, task);
|
||||
|
||||
await runner.taskCancelled(taskId, 'test-reason');
|
||||
|
||||
expect(runner.runningTasks.size).toBe(0);
|
||||
expect(taskCleanupSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reject pending requests when task is cancelled', async () => {
|
||||
runner = newTestRunner();
|
||||
|
||||
const taskId = 'test-task';
|
||||
const task = newTaskState(taskId);
|
||||
task.status = 'running';
|
||||
runner.runningTasks.set(taskId, task);
|
||||
|
||||
const dataRequestReject = jest.fn();
|
||||
const nodeTypesRequestReject = jest.fn();
|
||||
|
||||
runner.dataRequests.set('data-req', {
|
||||
taskId,
|
||||
requestId: 'data-req',
|
||||
resolve: jest.fn(),
|
||||
reject: dataRequestReject,
|
||||
});
|
||||
|
||||
runner.nodeTypesRequests.set('node-req', {
|
||||
taskId,
|
||||
requestId: 'node-req',
|
||||
resolve: jest.fn(),
|
||||
reject: nodeTypesRequestReject,
|
||||
});
|
||||
|
||||
await runner.taskCancelled(taskId, 'test-reason');
|
||||
|
||||
expect(dataRequestReject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Task cancelled: test-reason',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(nodeTypesRequestReject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Task cancelled: test-reason',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(runner.dataRequests.size).toBe(0);
|
||||
expect(runner.nodeTypesRequests.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('taskTimedOut', () => {
|
||||
it('should error task if task is waiting for settings', async () => {
|
||||
runner = newTestRunner();
|
||||
|
||||
const taskId = 'test-task';
|
||||
const task = newTaskState(taskId);
|
||||
task.status = 'waitingForSettings';
|
||||
runner.runningTasks.set(taskId, task);
|
||||
const sendSpy = jest.spyOn(runner, 'send');
|
||||
|
||||
await runner.taskTimedOut(taskId);
|
||||
|
||||
expect(runner.runningTasks.size).toBe(0);
|
||||
expect(sendSpy).toHaveBeenCalledWith({
|
||||
type: 'runner:taskerror',
|
||||
taskId,
|
||||
error: expect.any(TimeoutError),
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject pending requests when task is running', async () => {
|
||||
runner = newTestRunner();
|
||||
|
||||
const taskId = 'test-task';
|
||||
const task = newTaskState(taskId);
|
||||
task.status = 'running';
|
||||
runner.runningTasks.set(taskId, task);
|
||||
|
||||
const dataRequestReject = jest.fn();
|
||||
const nodeTypesRequestReject = jest.fn();
|
||||
|
||||
runner.dataRequests.set('data-req', {
|
||||
taskId,
|
||||
requestId: 'data-req',
|
||||
resolve: jest.fn(),
|
||||
reject: dataRequestReject,
|
||||
});
|
||||
|
||||
runner.nodeTypesRequests.set('node-req', {
|
||||
taskId,
|
||||
requestId: 'node-req',
|
||||
resolve: jest.fn(),
|
||||
reject: nodeTypesRequestReject,
|
||||
});
|
||||
|
||||
await runner.taskCancelled(taskId, 'test-reason');
|
||||
|
||||
expect(dataRequestReject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Task cancelled: test-reason',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(nodeTypesRequestReject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: 'Task cancelled: test-reason',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(runner.dataRequests.size).toBe(0);
|
||||
expect(runner.nodeTypesRequests.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('drain', () => {
|
||||
it('should stop sending offers on drain message', () => {
|
||||
runner = newTestRunner();
|
||||
runner.onMessage({ type: 'broker:runnerregistered' });
|
||||
expect(runner.canSendOffers).toBe(true);
|
||||
|
||||
runner.onMessage({ type: 'broker:drain' });
|
||||
|
||||
expect(runner.canSendOffers).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import type { IDataObject, INode, INodeExecutionData, ITaskData } from 'n8n-workflow';
|
||||
import { createRunExecutionData, NodeConnectionTypes } from 'n8n-workflow';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
import type { JSExecSettings } from '@/js-task-runner/js-task-runner';
|
||||
import type { DataRequestResponse } from '@/runner-types';
|
||||
import type { TaskParams } from '@/task-runner';
|
||||
import { TaskState } from '@/task-state';
|
||||
|
||||
/**
|
||||
* Creates a new task with the given settings
|
||||
*/
|
||||
export const newTaskParamsWithSettings = (
|
||||
settings: Partial<JSExecSettings> & Pick<JSExecSettings, 'code' | 'nodeMode'>,
|
||||
): TaskParams<JSExecSettings> => ({
|
||||
taskId: '1',
|
||||
settings: {
|
||||
workflowMode: 'manual',
|
||||
continueOnFail: false,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates a new node with the given options
|
||||
*/
|
||||
export const newNode = (opts: Partial<INode> = {}): INode => ({
|
||||
id: nanoid(),
|
||||
name: 'Test Node' + nanoid(),
|
||||
parameters: {},
|
||||
position: [0, 0],
|
||||
type: 'n8n-nodes-base.code',
|
||||
typeVersion: 1,
|
||||
...opts,
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates a new task data with the given options
|
||||
*/
|
||||
export const newTaskData = (opts: Partial<ITaskData> & Pick<ITaskData, 'source'>): ITaskData => ({
|
||||
startTime: Date.now(),
|
||||
executionTime: 0,
|
||||
executionIndex: 0,
|
||||
executionStatus: 'success',
|
||||
...opts,
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates a new data request response with the given options
|
||||
*/
|
||||
export const newDataRequestResponse = (
|
||||
inputData: INodeExecutionData[],
|
||||
opts: Partial<DataRequestResponse> & {
|
||||
staticData?: IDataObject;
|
||||
} = {},
|
||||
): DataRequestResponse => {
|
||||
const codeNode = newNode({
|
||||
name: 'JsCode',
|
||||
parameters: {
|
||||
mode: 'runOnceForEachItem',
|
||||
language: 'javaScript',
|
||||
jsCode: 'return item',
|
||||
},
|
||||
type: 'n8n-nodes-base.code',
|
||||
typeVersion: 2,
|
||||
});
|
||||
const manualTriggerNode = newNode({
|
||||
name: 'Trigger',
|
||||
type: 'n8n-nodes-base.manualTrigger',
|
||||
parameters: {
|
||||
manualTriggerParam: 'empty',
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
workflow: {
|
||||
id: '1',
|
||||
name: 'Test Workflow',
|
||||
active: true,
|
||||
connections: {
|
||||
[manualTriggerNode.name]: {
|
||||
main: [[{ node: codeNode.name, type: NodeConnectionTypes.Main, index: 0 }]],
|
||||
},
|
||||
},
|
||||
nodes: [manualTriggerNode, codeNode],
|
||||
staticData: opts.staticData,
|
||||
},
|
||||
inputData: {
|
||||
main: [inputData],
|
||||
},
|
||||
node: codeNode,
|
||||
runExecutionData: createRunExecutionData({
|
||||
resultData: {
|
||||
runData: {
|
||||
[manualTriggerNode.name]: [
|
||||
newTaskData({
|
||||
source: [],
|
||||
data: {
|
||||
main: [inputData],
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
pinData: {},
|
||||
lastNodeExecuted: manualTriggerNode.name,
|
||||
},
|
||||
}),
|
||||
runIndex: 0,
|
||||
itemIndex: 0,
|
||||
activeNodeName: codeNode.name,
|
||||
contextNodeName: codeNode.name,
|
||||
defaultReturnRunIndex: -1,
|
||||
siblingParameters: {},
|
||||
mode: 'manual',
|
||||
selfData: {},
|
||||
envProviderState: {
|
||||
env: {},
|
||||
isEnvAccessBlocked: true,
|
||||
isProcessAvailable: true,
|
||||
},
|
||||
additionalData: {
|
||||
executionId: 'exec-id',
|
||||
instanceBaseUrl: '',
|
||||
restartExecutionId: '',
|
||||
restApiUrl: '',
|
||||
formWaitingBaseUrl: 'http://formWaitingBaseUrl',
|
||||
webhookBaseUrl: 'http://webhookBaseUrl',
|
||||
webhookTestBaseUrl: 'http://webhookTestBaseUrl',
|
||||
webhookWaitingBaseUrl: 'http://webhookWaitingBaseUrl',
|
||||
variables: {
|
||||
var: 'value',
|
||||
},
|
||||
},
|
||||
connectionInputSource: {
|
||||
main: [
|
||||
{
|
||||
previousNode: 'Trigger',
|
||||
previousNodeOutput: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
...opts,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Wraps the given value into an INodeExecutionData object's json property
|
||||
*/
|
||||
export const wrapIntoJson = (json: IDataObject): INodeExecutionData => ({
|
||||
json,
|
||||
});
|
||||
|
||||
/**
|
||||
* Adds the given index as the pairedItem property to the given INodeExecutionData object
|
||||
*/
|
||||
export const withPairedItem = (index: number, data: INodeExecutionData): INodeExecutionData => ({
|
||||
...data,
|
||||
pairedItem: {
|
||||
item: index,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates a new task state with the given taskId
|
||||
*/
|
||||
export const newTaskState = (taskId: string) =>
|
||||
new TaskState({
|
||||
taskId,
|
||||
timeoutInS: 60,
|
||||
onTimeout: () => {},
|
||||
});
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
import { BuiltInsParserState } from '../built-ins-parser-state';
|
||||
|
||||
describe('BuiltInsParserState', () => {
|
||||
describe('toDataRequestParams', () => {
|
||||
it('should return empty array when no properties are marked as needed', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: [],
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: false,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return all nodes and input when markNeedsAllNodes is called', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markNeedsAllNodes();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: 'all',
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: true,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return specific node names when nodes are marked as needed individually', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markNodeAsNeeded('Node1');
|
||||
state.markNodeAsNeeded('Node2');
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: ['Node1', 'Node2'],
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: false,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should ignore individual nodes when needsAllNodes is marked as true', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markNodeAsNeeded('Node1');
|
||||
state.markNeedsAllNodes();
|
||||
state.markNodeAsNeeded('Node2'); // should be ignored since all nodes are needed
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: 'all',
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: true,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should mark env as needed when markEnvAsNeeded is called', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markEnvAsNeeded();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: [],
|
||||
env: true,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: false,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should mark input as needed when markInputAsNeeded is called', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markInputAsNeeded();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: [],
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: true,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should use the given chunk', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markInputAsNeeded();
|
||||
|
||||
expect(
|
||||
state.toDataRequestParams({
|
||||
count: 10,
|
||||
startIndex: 5,
|
||||
}),
|
||||
).toEqual({
|
||||
dataOfNodes: [],
|
||||
env: false,
|
||||
input: {
|
||||
chunk: {
|
||||
count: 10,
|
||||
startIndex: 5,
|
||||
},
|
||||
include: true,
|
||||
},
|
||||
prevNode: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('should mark prevNode as needed when markPrevNodeAsNeeded is called', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markPrevNodeAsNeeded();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: [],
|
||||
env: false,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: false,
|
||||
},
|
||||
prevNode: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return correct specification when multiple properties are marked as needed', () => {
|
||||
const state = new BuiltInsParserState();
|
||||
state.markNeedsAllNodes();
|
||||
state.markEnvAsNeeded();
|
||||
state.markInputAsNeeded();
|
||||
state.markPrevNodeAsNeeded();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: 'all',
|
||||
env: true,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: true,
|
||||
},
|
||||
prevNode: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return correct specification when all properties are marked as needed', () => {
|
||||
const state = BuiltInsParserState.newNeedsAllDataState();
|
||||
|
||||
expect(state.toDataRequestParams()).toEqual({
|
||||
dataOfNodes: 'all',
|
||||
env: true,
|
||||
input: {
|
||||
chunk: undefined,
|
||||
include: true,
|
||||
},
|
||||
prevNode: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
import { getAdditionalKeys } from 'n8n-core';
|
||||
import type {
|
||||
IDataObject,
|
||||
IExecuteData,
|
||||
INodeType,
|
||||
IWorkflowExecuteAdditionalData,
|
||||
} from 'n8n-workflow';
|
||||
import { Workflow, WorkflowDataProxy } from 'n8n-workflow';
|
||||
|
||||
import { newDataRequestResponse } from '../../__tests__/test-data';
|
||||
import { BuiltInsParser } from '../built-ins-parser';
|
||||
import { BuiltInsParserState } from '../built-ins-parser-state';
|
||||
|
||||
describe('BuiltInsParser', () => {
|
||||
const parser = new BuiltInsParser();
|
||||
|
||||
const parseAndExpectOk = (code: string) => {
|
||||
const result = parser.parseUsedBuiltIns(code);
|
||||
if (!result.ok) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
return result.result;
|
||||
};
|
||||
|
||||
describe('Env, input, execution and prevNode', () => {
|
||||
const cases: Array<[string, BuiltInsParserState]> = [
|
||||
['$env', new BuiltInsParserState({ needs$env: true })],
|
||||
['$execution', new BuiltInsParserState({ needs$execution: true })],
|
||||
['$prevNode', new BuiltInsParserState({ needs$prevNode: true })],
|
||||
];
|
||||
|
||||
test.each(cases)("should identify built-ins in '%s'", (code, expected) => {
|
||||
const state = parseAndExpectOk(code);
|
||||
expect(state).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Input', () => {
|
||||
it('should mark input as needed when $input is used', () => {
|
||||
const state = parseAndExpectOk(`
|
||||
$input.item.json.age = 10 + Math.floor(Math.random() * 30);
|
||||
$input.item.json.password = $input.item.json.password.split('').map(() => '*').join("")
|
||||
delete $input.item.json.lastname
|
||||
const emailParts = $input.item.json.email.split("@")
|
||||
$input.item.json.emailData = {
|
||||
user: emailParts[0],
|
||||
domain: emailParts[1]
|
||||
}
|
||||
|
||||
return $input.item;
|
||||
`);
|
||||
|
||||
expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
|
||||
});
|
||||
|
||||
it('should mark input as needed when $json is used', () => {
|
||||
const state = parseAndExpectOk(`
|
||||
$json.age = 10 + Math.floor(Math.random() * 30);
|
||||
return $json;
|
||||
`);
|
||||
|
||||
expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
|
||||
});
|
||||
|
||||
test.each([['items'], ['item']])(
|
||||
'should mark input as needed when %s is used',
|
||||
(identifier) => {
|
||||
const state = parseAndExpectOk(`return ${identifier};`);
|
||||
|
||||
expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('$(...)', () => {
|
||||
const cases: Array<[string, BuiltInsParserState]> = [
|
||||
[
|
||||
'$("nodeName").first()',
|
||||
new BuiltInsParserState({ neededNodeNames: new Set(['nodeName']) }),
|
||||
],
|
||||
[
|
||||
'$("nodeName").all(); $("secondNode").matchingItem()',
|
||||
new BuiltInsParserState({ neededNodeNames: new Set(['nodeName', 'secondNode']) }),
|
||||
],
|
||||
];
|
||||
|
||||
test.each(cases)("should identify nodes in '%s'", (code, expected) => {
|
||||
const state = parseAndExpectOk(code);
|
||||
expect(state).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should need all nodes when $() is called with a variable', () => {
|
||||
const state = parseAndExpectOk('var n = "name"; $(n)');
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
|
||||
it('should require all nodes when there are multiple usages of $() and one is with a variable', () => {
|
||||
const state = parseAndExpectOk(`
|
||||
$("nodeName");
|
||||
$("secondNode");
|
||||
var n = "name";
|
||||
$(n)
|
||||
`);
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
|
||||
test.each([
|
||||
['without parameters', '$()'],
|
||||
['number literal', '$(123)'],
|
||||
])('should ignore when $ is called %s', (_, code) => {
|
||||
const state = parseAndExpectOk(code);
|
||||
expect(state).toEqual(new BuiltInsParserState());
|
||||
});
|
||||
|
||||
test.each([
|
||||
'$("node").item',
|
||||
'$("node")["item"]',
|
||||
'$("node").pairedItem()',
|
||||
'$("node")["pairedItem"]()',
|
||||
'$("node").itemMatching(0)',
|
||||
'$("node")["itemMatching"](0)',
|
||||
'$("node")[variable]',
|
||||
'var a = $("node")',
|
||||
'let a = $("node")',
|
||||
'const a = $("node")',
|
||||
'a = $("node")',
|
||||
])('should require all nodes if %s is used', (code) => {
|
||||
const state = parseAndExpectOk(code);
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
|
||||
test.each(['$("node").first()', '$("node").last()', '$("node").all()', '$("node").params'])(
|
||||
'should require only accessed node if %s is used',
|
||||
(code) => {
|
||||
const state = parseAndExpectOk(code);
|
||||
expect(state).toEqual(
|
||||
new BuiltInsParserState({
|
||||
needsAllNodes: false,
|
||||
neededNodeNames: new Set(['node']),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('$items(...)', () => {
|
||||
it('should mark input as needed when $items() is used without arguments', () => {
|
||||
const state = parseAndExpectOk('$items()');
|
||||
expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
|
||||
});
|
||||
|
||||
it('should require the given node when $items() is used with a static value', () => {
|
||||
const state = parseAndExpectOk('$items("nodeName")');
|
||||
expect(state).toEqual(new BuiltInsParserState({ neededNodeNames: new Set(['nodeName']) }));
|
||||
});
|
||||
|
||||
it('should require all nodes when $items() is used with a variable', () => {
|
||||
const state = parseAndExpectOk('var n = "name"; $items(n)');
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('$node', () => {
|
||||
it('should require all nodes when $node is used', () => {
|
||||
const state = parseAndExpectOk('return $node["name"];');
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('$item', () => {
|
||||
it('should require all nodes and input when $item is used', () => {
|
||||
const state = parseAndExpectOk('$item("0").$node["my node"].json["title"]');
|
||||
expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('ECMAScript syntax', () => {
|
||||
describe('ES2020', () => {
|
||||
it('should parse optional chaining', () => {
|
||||
parseAndExpectOk(`
|
||||
const a = { b: { c: 1 } };
|
||||
return a.b?.c;
|
||||
`);
|
||||
});
|
||||
|
||||
it('should parse nullish coalescing', () => {
|
||||
parseAndExpectOk(`
|
||||
const a = null;
|
||||
return a ?? 1;
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ES2021', () => {
|
||||
it('should parse numeric separators', () => {
|
||||
parseAndExpectOk(`
|
||||
const a = 1_000_000;
|
||||
return a;
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('WorkflowDataProxy built-ins', () => {
|
||||
it('should have a known list of built-ins', () => {
|
||||
const data = newDataRequestResponse([]);
|
||||
const executeData: IExecuteData = {
|
||||
data: {},
|
||||
node: data.node,
|
||||
source: data.connectionInputSource,
|
||||
};
|
||||
const dataProxy = new WorkflowDataProxy(
|
||||
new Workflow({
|
||||
...data.workflow,
|
||||
nodeTypes: {
|
||||
getByName() {
|
||||
return undefined as unknown as INodeType;
|
||||
},
|
||||
getByNameAndVersion() {
|
||||
return undefined as unknown as INodeType;
|
||||
},
|
||||
getKnownTypes() {
|
||||
return undefined as unknown as IDataObject;
|
||||
},
|
||||
},
|
||||
}),
|
||||
data.runExecutionData,
|
||||
data.runIndex,
|
||||
0,
|
||||
data.activeNodeName,
|
||||
[],
|
||||
data.siblingParameters,
|
||||
data.mode,
|
||||
getAdditionalKeys(
|
||||
data.additionalData as IWorkflowExecuteAdditionalData,
|
||||
data.mode,
|
||||
data.runExecutionData,
|
||||
),
|
||||
executeData,
|
||||
data.defaultReturnRunIndex,
|
||||
data.selfData,
|
||||
data.contextNodeName,
|
||||
// Make sure that even if we don't receive the envProviderState for
|
||||
// whatever reason, we don't expose the task runner's env to the code
|
||||
data.envProviderState ?? {
|
||||
env: {},
|
||||
isEnvAccessBlocked: false,
|
||||
isProcessAvailable: true,
|
||||
},
|
||||
).getDataProxy({ throwOnMissingExecutionData: false });
|
||||
|
||||
/**
|
||||
* NOTE! If you are adding new built-ins to the WorkflowDataProxy class
|
||||
* make sure the built-ins parser and Task Runner handle them properly.
|
||||
*/
|
||||
expect(Object.keys(dataProxy)).toStrictEqual([
|
||||
'$',
|
||||
'$input',
|
||||
'$binary',
|
||||
'$data',
|
||||
'$env',
|
||||
'$evaluateExpression',
|
||||
'$item',
|
||||
'$fromAI',
|
||||
'$fromai',
|
||||
'$fromAi',
|
||||
'$items',
|
||||
'$tool',
|
||||
'$json',
|
||||
'$node',
|
||||
'$self',
|
||||
'$parameter',
|
||||
'$rawParameter',
|
||||
'$prevNode',
|
||||
'$runIndex',
|
||||
'$mode',
|
||||
'$workflow',
|
||||
'$itemIndex',
|
||||
'$now',
|
||||
'$today',
|
||||
'$jmesPath',
|
||||
'DateTime',
|
||||
'Interval',
|
||||
'Duration',
|
||||
'$execution',
|
||||
'$vars',
|
||||
'$secrets',
|
||||
'$executionId',
|
||||
'$resumeWebhookUrl',
|
||||
'$getPairedItem',
|
||||
'$jmespath',
|
||||
'$position',
|
||||
'$thisItem',
|
||||
'$thisItemIndex',
|
||||
'$thisRunIndex',
|
||||
'$nodeVersion',
|
||||
'$nodeId',
|
||||
'$agentInfo',
|
||||
'$webhookId',
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import type {
|
||||
AssignmentExpression,
|
||||
Identifier,
|
||||
Literal,
|
||||
MemberExpression,
|
||||
Node,
|
||||
VariableDeclarator,
|
||||
} from 'acorn';
|
||||
|
||||
export function isLiteral(node?: Node): node is Literal {
|
||||
return node?.type === 'Literal';
|
||||
}
|
||||
|
||||
export function isIdentifier(node?: Node): node is Identifier {
|
||||
return node?.type === 'Identifier';
|
||||
}
|
||||
|
||||
export function isMemberExpression(node?: Node): node is MemberExpression {
|
||||
return node?.type === 'MemberExpression';
|
||||
}
|
||||
|
||||
export function isVariableDeclarator(node?: Node): node is VariableDeclarator {
|
||||
return node?.type === 'VariableDeclarator';
|
||||
}
|
||||
|
||||
export function isAssignmentExpression(node?: Node): node is AssignmentExpression {
|
||||
return node?.type === 'AssignmentExpression';
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
import type { BrokerMessage } from '@/message-types';
|
||||
import type { InputDataChunkDefinition } from '@/runner-types';
|
||||
|
||||
/**
|
||||
* Class to keep track of which built-in variables are accessed in the code
|
||||
*/
|
||||
export class BuiltInsParserState {
|
||||
neededNodeNames: Set<string> = new Set();
|
||||
|
||||
needsAllNodes = false;
|
||||
|
||||
needs$env = false;
|
||||
|
||||
needs$input = false;
|
||||
|
||||
needs$execution = false;
|
||||
|
||||
needs$prevNode = false;
|
||||
|
||||
constructor(opts: Partial<BuiltInsParserState> = {}) {
|
||||
Object.assign(this, opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks that all nodes are needed, including input data
|
||||
*/
|
||||
markNeedsAllNodes() {
|
||||
this.needsAllNodes = true;
|
||||
this.needs$input = true;
|
||||
this.neededNodeNames = new Set();
|
||||
}
|
||||
|
||||
markNodeAsNeeded(nodeName: string) {
|
||||
if (this.needsAllNodes) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.neededNodeNames.add(nodeName);
|
||||
}
|
||||
|
||||
markEnvAsNeeded() {
|
||||
this.needs$env = true;
|
||||
}
|
||||
|
||||
markInputAsNeeded() {
|
||||
this.needs$input = true;
|
||||
}
|
||||
|
||||
markExecutionAsNeeded() {
|
||||
this.needs$execution = true;
|
||||
}
|
||||
|
||||
markPrevNodeAsNeeded() {
|
||||
this.needs$prevNode = true;
|
||||
}
|
||||
|
||||
toDataRequestParams(
|
||||
chunk?: InputDataChunkDefinition,
|
||||
): BrokerMessage.ToRequester.TaskDataRequest['requestParams'] {
|
||||
return {
|
||||
dataOfNodes: this.needsAllNodes ? 'all' : Array.from(this.neededNodeNames),
|
||||
env: this.needs$env,
|
||||
input: {
|
||||
include: this.needs$input,
|
||||
chunk,
|
||||
},
|
||||
prevNode: this.needs$prevNode,
|
||||
};
|
||||
}
|
||||
|
||||
static newNeedsAllDataState() {
|
||||
const obj = new BuiltInsParserState();
|
||||
obj.markNeedsAllNodes();
|
||||
obj.markEnvAsNeeded();
|
||||
obj.markInputAsNeeded();
|
||||
obj.markExecutionAsNeeded();
|
||||
obj.markPrevNodeAsNeeded();
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import type { CallExpression, Identifier, Node, Program } from 'acorn';
|
||||
import { parse } from 'acorn';
|
||||
import { ancestor } from 'acorn-walk';
|
||||
import type { Result } from 'n8n-workflow';
|
||||
import { toResult } from 'n8n-workflow';
|
||||
|
||||
import {
|
||||
isAssignmentExpression,
|
||||
isIdentifier,
|
||||
isLiteral,
|
||||
isMemberExpression,
|
||||
isVariableDeclarator,
|
||||
} from './acorn-helpers';
|
||||
import { BuiltInsParserState } from './built-ins-parser-state';
|
||||
|
||||
/**
|
||||
* Class for parsing Code Node code to identify which built-in variables
|
||||
* are accessed
|
||||
*/
|
||||
export class BuiltInsParser {
|
||||
/**
|
||||
* Parses which built-in variables are accessed in the given code
|
||||
*/
|
||||
parseUsedBuiltIns(code: string): Result<BuiltInsParserState, Error> {
|
||||
return toResult(() => {
|
||||
const wrappedCode = `async function VmCodeWrapper() { ${code} }`;
|
||||
const ast = parse(wrappedCode, { ecmaVersion: 2025, sourceType: 'module' });
|
||||
|
||||
return this.identifyBuiltInsByWalkingAst(ast);
|
||||
});
|
||||
}
|
||||
|
||||
/** Traverse the AST of the script and mark any data needed for it to run. */
|
||||
private identifyBuiltInsByWalkingAst(ast: Program) {
|
||||
const accessedBuiltIns = new BuiltInsParserState();
|
||||
|
||||
ancestor(
|
||||
ast,
|
||||
{
|
||||
CallExpression: this.visitCallExpression,
|
||||
Identifier: this.visitIdentifier,
|
||||
},
|
||||
undefined,
|
||||
accessedBuiltIns,
|
||||
);
|
||||
|
||||
return accessedBuiltIns;
|
||||
}
|
||||
|
||||
private visitCallExpression = (
|
||||
node: CallExpression,
|
||||
state: BuiltInsParserState,
|
||||
ancestors: Node[],
|
||||
) => {
|
||||
// $(...)
|
||||
const isDollar = node.callee.type === 'Identifier' && node.callee.name === '$';
|
||||
const isItems = node.callee.type === 'Identifier' && node.callee.name === '$items';
|
||||
if (isDollar) {
|
||||
this.visitDollarCallExpression(node, state, ancestors);
|
||||
} else if (isItems) {
|
||||
// $items(...) is a legacy syntax that is not documented but we still
|
||||
// need to support it for backwards compatibility
|
||||
this.visitDollarItemsCallExpression(node, state);
|
||||
}
|
||||
};
|
||||
|
||||
/** $(...) */
|
||||
private visitDollarCallExpression(
|
||||
node: CallExpression,
|
||||
state: BuiltInsParserState,
|
||||
ancestors: Node[],
|
||||
) {
|
||||
// $(): This is not valid, ignore
|
||||
if (node.arguments.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const firstArg = node.arguments[0];
|
||||
if (!isLiteral(firstArg)) {
|
||||
// $(variable): Can't easily determine statically, mark all nodes as needed
|
||||
state.markNeedsAllNodes();
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof firstArg.value !== 'string') {
|
||||
// $(123): Static value, but not a string --> invalid code --> ignore
|
||||
return;
|
||||
}
|
||||
|
||||
// $("node"): Static value, mark 'nodeName' as needed
|
||||
state.markNodeAsNeeded(firstArg.value);
|
||||
|
||||
// Determine how $("node") is used
|
||||
this.handlePrevNodeCall(node, state, ancestors);
|
||||
}
|
||||
|
||||
/** $items(...) */
|
||||
private visitDollarItemsCallExpression(node: CallExpression, state: BuiltInsParserState) {
|
||||
// $items(): This gets items from the previous node
|
||||
if (node.arguments.length === 0) {
|
||||
state.markInputAsNeeded();
|
||||
return;
|
||||
}
|
||||
|
||||
const firstArg = node.arguments[0];
|
||||
if (!isLiteral(firstArg)) {
|
||||
// $items(variable): Can't easily determine statically, mark all nodes as needed
|
||||
state.markNeedsAllNodes();
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof firstArg.value !== 'string') {
|
||||
// $items(123): Static value, but not a string --> unsupported code --> ignore
|
||||
return;
|
||||
}
|
||||
|
||||
// $items(nodeName): Static value, mark 'nodeName' as needed
|
||||
state.markNodeAsNeeded(firstArg.value);
|
||||
}
|
||||
|
||||
private handlePrevNodeCall(_node: CallExpression, state: BuiltInsParserState, ancestors: Node[]) {
|
||||
// $("node").item, .pairedItem or .itemMatching: In a case like this, the execution
|
||||
// engine will traverse back from current node (i.e. the Code Node) to
|
||||
// the "node" node and use `pairedItem`s to find which item is linked
|
||||
// to the current item. So, we need to mark all nodes as needed.
|
||||
// TODO: We could also mark all the nodes between the current node and
|
||||
// the "node" node as needed, but that would require more complex logic.
|
||||
const directParent = ancestors[ancestors.length - 2];
|
||||
if (isMemberExpression(directParent)) {
|
||||
const accessedProperty = directParent.property;
|
||||
|
||||
if (directParent.computed) {
|
||||
// $("node")["item"], ["pairedItem"] or ["itemMatching"]
|
||||
if (isLiteral(accessedProperty)) {
|
||||
if (this.isPairedItemProperty(accessedProperty.value)) {
|
||||
state.markNeedsAllNodes();
|
||||
}
|
||||
// Else: $("node")[123]: Static value, but not any of the ones above --> ignore
|
||||
}
|
||||
// $("node")[variable]
|
||||
else if (isIdentifier(accessedProperty)) {
|
||||
state.markNeedsAllNodes();
|
||||
}
|
||||
}
|
||||
// $("node").item, .pairedItem or .itemMatching
|
||||
else if (isIdentifier(accessedProperty) && this.isPairedItemProperty(accessedProperty.name)) {
|
||||
state.markNeedsAllNodes();
|
||||
}
|
||||
} else if (isVariableDeclarator(directParent) || isAssignmentExpression(directParent)) {
|
||||
// const variable = $("node") or variable = $("node"):
|
||||
// In this case we would need to track down all the possible use sites
|
||||
// of 'variable' and determine if `.item` is accessed on it. This is
|
||||
// more complex and skipped for now.
|
||||
// TODO: Optimize for this case
|
||||
state.markNeedsAllNodes();
|
||||
} else {
|
||||
// Something else than the cases above. Mark all nodes as needed as it
|
||||
// could be a dynamic access.
|
||||
state.markNeedsAllNodes();
|
||||
}
|
||||
}
|
||||
|
||||
private visitIdentifier = (node: Identifier, state: BuiltInsParserState) => {
|
||||
if (node.name === '$env') {
|
||||
state.markEnvAsNeeded();
|
||||
} else if (node.name === '$item') {
|
||||
// $item is legacy syntax that is basically an alias for WorkflowDataProxy
|
||||
// and allows accessing any data. We need to support it for backwards
|
||||
// compatibility, but we're not gonna implement any optimizations
|
||||
state.markNeedsAllNodes();
|
||||
} else if (
|
||||
node.name === '$input' ||
|
||||
node.name === '$json' ||
|
||||
node.name === 'items' ||
|
||||
// item is deprecated but we still need to support it
|
||||
node.name === 'item'
|
||||
) {
|
||||
state.markInputAsNeeded();
|
||||
} else if (node.name === '$node') {
|
||||
// $node is legacy way of accessing any node's output. We need to
|
||||
// support it for backward compatibility, but we're not gonna
|
||||
// implement any optimizations
|
||||
state.markNeedsAllNodes();
|
||||
} else if (node.name === '$execution') {
|
||||
state.markExecutionAsNeeded();
|
||||
} else if (node.name === '$prevNode') {
|
||||
state.markPrevNodeAsNeeded();
|
||||
}
|
||||
};
|
||||
|
||||
private isPairedItemProperty(
|
||||
property?: string | boolean | null | number | RegExp | bigint,
|
||||
): boolean {
|
||||
return property === 'item' || property === 'pairedItem' || property === 'itemMatching';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { ExecutionError } from '../execution-error';
|
||||
|
||||
describe('ExecutionError', () => {
|
||||
const defaultStack = `TypeError: a.unknown is not a function
|
||||
at VmCodeWrapper (evalmachine.<anonymous>:2:3)
|
||||
at evalmachine.<anonymous>:7:2
|
||||
at Script.runInContext (node:vm:148:12)
|
||||
at Script.runInNewContext (node:vm:153:17)
|
||||
at runInNewContext (node:vm:309:38)
|
||||
at JsTaskRunner.runForAllItems (/n8n/packages/@n8n/task-runner/dist/js-task-runner/js-task-runner.js:90:65)
|
||||
at JsTaskRunner.executeTask (/n8n/packages/@n8n/task-runner/dist/js-task-runner/js-task-runner.js:71:26)
|
||||
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
|
||||
at async JsTaskRunner.receivedSettings (/n8n/packages/@n8n/task-runner/dist/task-runner.js:190:26)`;
|
||||
|
||||
it('should parse error details from stack trace without itemIndex', () => {
|
||||
const error = new Error('a.unknown is not a function');
|
||||
error.stack = defaultStack;
|
||||
|
||||
const executionError = new ExecutionError(error);
|
||||
expect(executionError.message).toBe('a.unknown is not a function [line 2]');
|
||||
expect(executionError.lineNumber).toBe(2);
|
||||
expect(executionError.description).toBe('TypeError');
|
||||
expect(executionError.context).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should parse error details from stack trace with itemIndex', () => {
|
||||
const error = new Error('a.unknown is not a function');
|
||||
error.stack = defaultStack;
|
||||
|
||||
const executionError = new ExecutionError(error, 1);
|
||||
expect(executionError.message).toBe('a.unknown is not a function [line 2, for item 1]');
|
||||
expect(executionError.lineNumber).toBe(2);
|
||||
expect(executionError.description).toBe('TypeError');
|
||||
expect(executionError.context).toEqual({ itemIndex: 1 });
|
||||
});
|
||||
|
||||
it('should serialize correctly', () => {
|
||||
const error = new Error('a.unknown is not a function');
|
||||
Object.defineProperty(error, 'stack', {
|
||||
value: defaultStack,
|
||||
enumerable: true,
|
||||
});
|
||||
// error.stack = defaultStack;
|
||||
|
||||
const executionError = new ExecutionError(error, 1);
|
||||
|
||||
expect(JSON.stringify(executionError)).toBe(
|
||||
JSON.stringify({
|
||||
stack: defaultStack,
|
||||
message: 'a.unknown is not a function [line 2, for item 1]',
|
||||
description: 'TypeError',
|
||||
itemIndex: 1,
|
||||
context: { itemIndex: 1 },
|
||||
lineNumber: 2,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
import { UserError } from 'n8n-workflow';
|
||||
|
||||
export class DisallowedModuleError extends UserError {
|
||||
constructor(moduleName: string) {
|
||||
super(`Module '${moduleName}' is disallowed`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export interface ErrorLike {
|
||||
message: string;
|
||||
stack?: string;
|
||||
}
|
||||
|
||||
export function isErrorLike(value: unknown): value is ErrorLike {
|
||||
if (typeof value !== 'object' || value === null) return false;
|
||||
|
||||
const errorLike = value as ErrorLike;
|
||||
|
||||
return typeof errorLike.message === 'string';
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { ErrorLike } from './error-like';
|
||||
import { SerializableError } from './serializable-error';
|
||||
|
||||
export class ExecutionError extends SerializableError {
|
||||
description: string | null = null;
|
||||
|
||||
itemIndex: number | undefined = undefined;
|
||||
|
||||
context: { itemIndex: number } | undefined = undefined;
|
||||
|
||||
lineNumber: number | undefined = undefined;
|
||||
|
||||
constructor(error: ErrorLike, itemIndex?: number) {
|
||||
super(error.message);
|
||||
this.itemIndex = itemIndex;
|
||||
|
||||
if (this.itemIndex !== undefined) {
|
||||
this.context = { itemIndex: this.itemIndex };
|
||||
}
|
||||
|
||||
// Override the stack trace with the given error's stack trace. Since
|
||||
// node v22 it's not writable, so we can't assign it directly
|
||||
Object.defineProperty(this, 'stack', {
|
||||
value: error.stack,
|
||||
enumerable: true,
|
||||
});
|
||||
|
||||
this.populateFromStack();
|
||||
}
|
||||
|
||||
/**
|
||||
* Populate error `message` and `description` from error `stack`.
|
||||
*/
|
||||
private populateFromStack() {
|
||||
const stackRows = (this.stack ?? '').split('\n');
|
||||
|
||||
if (stackRows.length === 0) {
|
||||
this.message = 'Unknown error';
|
||||
return;
|
||||
}
|
||||
|
||||
const messageRow = stackRows.find((line) => line.includes('Error:'));
|
||||
const lineNumberDisplay = this.toLineNumberDisplay(stackRows);
|
||||
|
||||
if (!messageRow) {
|
||||
this.message = `Unknown error ${lineNumberDisplay}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const [errorDetails, errorType] = this.toErrorDetailsAndType(messageRow);
|
||||
|
||||
if (errorType) this.description = errorType;
|
||||
|
||||
if (!errorDetails) {
|
||||
this.message = `Unknown error ${lineNumberDisplay}`;
|
||||
return;
|
||||
}
|
||||
|
||||
this.message = `${errorDetails} ${lineNumberDisplay}`;
|
||||
}
|
||||
|
||||
private toLineNumberDisplay(stackRows: string[]) {
|
||||
if (!stackRows || stackRows.length === 0) return '';
|
||||
|
||||
const userFnLine = stackRows.find(
|
||||
(row) => row.match(/\(evalmachine\.<anonymous>:\d+:\d+\)/) && !row.includes('VmCodeWrapper'),
|
||||
);
|
||||
|
||||
if (userFnLine) {
|
||||
const match = userFnLine.match(/evalmachine\.<anonymous>:(\d+):/);
|
||||
if (match) this.lineNumber = Number(match[1]);
|
||||
}
|
||||
|
||||
if (this.lineNumber === undefined) {
|
||||
const topLevelLine = stackRows.find(
|
||||
(row) => row.includes('VmCodeWrapper') && row.includes('evalmachine.<anonymous>'),
|
||||
);
|
||||
|
||||
if (topLevelLine) {
|
||||
const match = topLevelLine.match(/evalmachine\.<anonymous>:(\d+):/);
|
||||
if (match) this.lineNumber = Number(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.lineNumber === undefined) return '';
|
||||
|
||||
return this.itemIndex === undefined
|
||||
? `[line ${this.lineNumber}]`
|
||||
: `[line ${this.lineNumber}, for item ${this.itemIndex}]`;
|
||||
}
|
||||
|
||||
private toErrorDetailsAndType(messageRow?: string) {
|
||||
if (!messageRow) return [null, null];
|
||||
|
||||
const segments = messageRow.split(':').map((i) => i.trim());
|
||||
if (segments[1] === "Cannot find module 'node") {
|
||||
segments[1] = `${segments[1]}:${segments[2]}`;
|
||||
segments.splice(2, 1);
|
||||
}
|
||||
|
||||
if (
|
||||
segments.length >= 3 &&
|
||||
segments[1]?.startsWith("Module 'node") &&
|
||||
segments[2]?.includes("' is disallowed")
|
||||
) {
|
||||
segments[1] = `${segments[1]}:${segments[2]}`;
|
||||
segments.splice(2, 1);
|
||||
}
|
||||
|
||||
const [errorDetails, errorType] = segments.reverse();
|
||||
return [errorDetails, errorType === 'Error' ? null : errorType];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Makes the given error's `message` and `stack` properties enumerable
|
||||
* so they can be serialized with JSON.stringify
|
||||
*/
|
||||
export function makeSerializable(error: Error) {
|
||||
Object.defineProperties(error, {
|
||||
message: {
|
||||
value: error.message,
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
},
|
||||
stack: {
|
||||
value: error.stack,
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
},
|
||||
});
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that has its message property serialized as well. Used to transport
|
||||
* errors over the wire.
|
||||
*/
|
||||
export abstract class SerializableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
|
||||
makeSerializable(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { ApplicationError } from '@n8n/errors';
|
||||
|
||||
export class TaskCancelledError extends ApplicationError {
|
||||
constructor(reason: string) {
|
||||
super(`Task cancelled: ${reason}`, { level: 'warning' });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { ApplicationError } from '@n8n/errors';
|
||||
|
||||
export class TimeoutError extends ApplicationError {
|
||||
description: string;
|
||||
|
||||
constructor(taskTimeout: number) {
|
||||
super(
|
||||
`Task execution timed out after ${taskTimeout} ${taskTimeout === 1 ? 'second' : 'seconds'}`,
|
||||
);
|
||||
|
||||
const subtitle = 'The task runner was taking too long on this task, so the task was aborted.';
|
||||
|
||||
const fixes = {
|
||||
optimizeScript:
|
||||
'Optimize your script to prevent long-running tasks, e.g. by processing data in smaller batches.',
|
||||
ensureTermination:
|
||||
'Ensure that all paths in your script are able to terminate, i.e. no infinite loops.',
|
||||
};
|
||||
|
||||
const suggestions = [fixes.optimizeScript, fixes.ensureTermination];
|
||||
|
||||
const suggestionsText = suggestions
|
||||
.map((suggestion, index) => `${index + 1}. ${suggestion}`)
|
||||
.join('<br/>');
|
||||
|
||||
const description = `${subtitle} You can try the following:<br/><br/>${suggestionsText}`;
|
||||
|
||||
this.description = description;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { ApplicationError } from '@n8n/errors';
|
||||
|
||||
/**
|
||||
* Error that indicates that a specific function is not available in the
|
||||
* Code Node.
|
||||
*/
|
||||
export class UnsupportedFunctionError extends ApplicationError {
|
||||
constructor(functionName: string) {
|
||||
super(`The function "${functionName}" is not supported in the Code Node`, {
|
||||
level: 'info',
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
import isObject from 'lodash/isObject';
|
||||
import set from 'lodash/set';
|
||||
import { DateTime, Duration, Interval } from 'luxon';
|
||||
import { getAdditionalKeys } from 'n8n-core';
|
||||
import {
|
||||
WorkflowDataProxy,
|
||||
Workflow,
|
||||
ObservableObject,
|
||||
Expression,
|
||||
jsonStringify,
|
||||
} from 'n8n-workflow';
|
||||
import type {
|
||||
CodeExecutionMode,
|
||||
IWorkflowExecuteAdditionalData,
|
||||
IDataObject,
|
||||
INodeExecutionData,
|
||||
INodeParameters,
|
||||
WorkflowExecuteMode,
|
||||
WorkflowParameters,
|
||||
ITaskDataConnections,
|
||||
INode,
|
||||
IRunExecutionData,
|
||||
EnvProviderState,
|
||||
IExecuteData,
|
||||
INodeTypeDescription,
|
||||
IWorkflowDataProxyData,
|
||||
} from 'n8n-workflow';
|
||||
import * as a from 'node:assert';
|
||||
import { type Context, createContext, runInContext } from 'node:vm';
|
||||
|
||||
import type { MainConfig } from '@/config/main-config';
|
||||
import { UnsupportedFunctionError } from '@/js-task-runner/errors/unsupported-function.error';
|
||||
import { EXPOSED_RPC_METHODS, UNSUPPORTED_HELPER_FUNCTIONS } from '@/runner-types';
|
||||
import type {
|
||||
DataRequestResponse,
|
||||
InputDataChunkDefinition,
|
||||
PartialAdditionalData,
|
||||
TaskResultData,
|
||||
} from '@/runner-types';
|
||||
import type { TaskParams } from '@/task-runner';
|
||||
import { noOp, TaskRunner } from '@/task-runner';
|
||||
|
||||
import { BuiltInsParser } from './built-ins-parser/built-ins-parser';
|
||||
import { BuiltInsParserState } from './built-ins-parser/built-ins-parser-state';
|
||||
import { isErrorLike } from './errors/error-like';
|
||||
import { ExecutionError } from './errors/execution-error';
|
||||
import { makeSerializable } from './errors/serializable-error';
|
||||
import { TimeoutError } from './errors/timeout-error';
|
||||
import type { RequireResolver } from './require-resolver';
|
||||
import { createRequireResolver } from './require-resolver';
|
||||
import { DataRequestResponseReconstruct } from '../data-request/data-request-response-reconstruct';
|
||||
|
||||
export interface RpcCallObject {
|
||||
[name: string]: ((...args: unknown[]) => Promise<unknown>) | RpcCallObject;
|
||||
}
|
||||
|
||||
/**
|
||||
* The mode in which the code is executed:
|
||||
* - 'runCode': The code is executed in a limited environment that doesn't have
|
||||
* access to builtins or RPC and doesn't fetch any input data separately.
|
||||
* - 'runOnceForAllItems': The code is executed for all items in a single run.
|
||||
* - 'runOnceForEachItem': The code is executed for each item in the input data.
|
||||
*/
|
||||
export type RunnerExecutionMode = 'runCode' | CodeExecutionMode;
|
||||
|
||||
export interface JSExecSettings {
|
||||
code: string;
|
||||
// Additional properties to add to the context
|
||||
additionalProperties?: Record<string, unknown>;
|
||||
nodeMode: RunnerExecutionMode;
|
||||
workflowMode: WorkflowExecuteMode;
|
||||
continueOnFail: boolean;
|
||||
// For executing partial input data
|
||||
chunk?: InputDataChunkDefinition;
|
||||
}
|
||||
|
||||
export interface JsTaskData {
|
||||
workflow: Omit<WorkflowParameters, 'nodeTypes'>;
|
||||
inputData: ITaskDataConnections;
|
||||
connectionInputData: INodeExecutionData[];
|
||||
node: INode;
|
||||
|
||||
runExecutionData: IRunExecutionData;
|
||||
runIndex: number;
|
||||
itemIndex: number;
|
||||
activeNodeName: string;
|
||||
siblingParameters: INodeParameters;
|
||||
mode: WorkflowExecuteMode;
|
||||
envProviderState: EnvProviderState;
|
||||
executeData?: IExecuteData;
|
||||
defaultReturnRunIndex: number;
|
||||
selfData: IDataObject;
|
||||
contextNodeName: string;
|
||||
additionalData: PartialAdditionalData;
|
||||
}
|
||||
|
||||
type GlobalFunctionWithPrototype = ((...args: unknown[]) => unknown) & {
|
||||
prototype?: object;
|
||||
};
|
||||
|
||||
type CustomConsole = {
|
||||
log: (...args: unknown[]) => void;
|
||||
};
|
||||
|
||||
export class JsTaskRunner extends TaskRunner {
|
||||
private static readonly CONSOLE_METHODS = [
|
||||
'log',
|
||||
'warn',
|
||||
'error',
|
||||
'info',
|
||||
'debug',
|
||||
'trace',
|
||||
'dir',
|
||||
'time',
|
||||
'timeEnd',
|
||||
'timeLog',
|
||||
'assert',
|
||||
'clear',
|
||||
'count',
|
||||
'countReset',
|
||||
'group',
|
||||
'groupEnd',
|
||||
'groupCollapsed',
|
||||
'table',
|
||||
'dirxml',
|
||||
'profile',
|
||||
'profileEnd',
|
||||
'timeStamp',
|
||||
] as const;
|
||||
|
||||
private readonly requireResolver: RequireResolver;
|
||||
|
||||
private readonly builtInsParser = new BuiltInsParser();
|
||||
|
||||
private readonly taskDataReconstruct = new DataRequestResponseReconstruct();
|
||||
|
||||
private readonly mode: 'secure' | 'insecure' = 'secure';
|
||||
|
||||
constructor(config: MainConfig, name = 'JS Task Runner') {
|
||||
super({
|
||||
taskType: 'javascript',
|
||||
name,
|
||||
...config.baseRunnerConfig,
|
||||
});
|
||||
const { jsRunnerConfig } = config;
|
||||
|
||||
const parseModuleAllowList = (moduleList: string) =>
|
||||
moduleList === '*'
|
||||
? '*'
|
||||
: new Set(
|
||||
moduleList
|
||||
.split(',')
|
||||
.map((x) => x.trim())
|
||||
.filter((x) => x !== ''),
|
||||
);
|
||||
|
||||
const allowedBuiltInModules = parseModuleAllowList(jsRunnerConfig.allowedBuiltInModules ?? '');
|
||||
const allowedExternalModules = parseModuleAllowList(
|
||||
jsRunnerConfig.allowedExternalModules ?? '',
|
||||
);
|
||||
this.mode = jsRunnerConfig.insecureMode ? 'insecure' : 'secure';
|
||||
|
||||
this.requireResolver = createRequireResolver({
|
||||
allowedBuiltInModules,
|
||||
allowedExternalModules,
|
||||
});
|
||||
|
||||
if (this.mode === 'secure') this.preventPrototypePollution(allowedExternalModules);
|
||||
}
|
||||
|
||||
private preventPrototypePollution(allowedExternalModules: Set<string> | '*') {
|
||||
if (allowedExternalModules instanceof Set) {
|
||||
// This is a workaround to enable the allowed external libraries to mutate
|
||||
// prototypes directly. For example momentjs overrides .toString() directly
|
||||
// on the Moment.prototype, which doesn't work if Object.prototype has been
|
||||
// frozen. This works as long as the overrides are done when the library is
|
||||
// imported.
|
||||
for (const module of allowedExternalModules) {
|
||||
try {
|
||||
require(module);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && error.code === 'MODULE_NOT_FOUND') {
|
||||
console.error(
|
||||
`Allowlisted module '${module}' is not installed. Please either install it or remove it from the allowlist in the n8n-task-runners.json config file. See: https://docs.n8n.io/hosting/configuration/task-runners/#adding-extra-dependencies`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Overwrite unsafe Buffer allocations on the real constructor
|
||||
const safeAlloc = Buffer.alloc.bind(Buffer);
|
||||
Buffer.allocUnsafe = safeAlloc as typeof Buffer.allocUnsafe;
|
||||
Buffer.allocUnsafeSlow = safeAlloc as typeof Buffer.allocUnsafeSlow;
|
||||
|
||||
// Freeze globals, except in tests because Jest needs to be able to mutate prototypes
|
||||
if (process.env.NODE_ENV !== 'test') {
|
||||
Object.getOwnPropertyNames(globalThis)
|
||||
.map((name) => Reflect.get(globalThis, name) as unknown)
|
||||
.filter((value): value is GlobalFunctionWithPrototype => typeof value === 'function')
|
||||
.forEach((fn) => {
|
||||
if (typeof fn.prototype === 'object') Object.freeze(fn.prototype);
|
||||
Object.freeze(fn);
|
||||
});
|
||||
|
||||
[Reflect, JSON, Math].forEach(Object.freeze);
|
||||
}
|
||||
|
||||
// Freeze internal classes
|
||||
[Workflow, Expression, WorkflowDataProxy, DateTime, Interval, Duration]
|
||||
.map((constructor) => constructor.prototype)
|
||||
.forEach(Object.freeze);
|
||||
}
|
||||
|
||||
async executeTask(
|
||||
taskParams: TaskParams<JSExecSettings>,
|
||||
abortSignal: AbortSignal,
|
||||
): Promise<TaskResultData> {
|
||||
const { taskId, settings } = taskParams;
|
||||
a.ok(settings, 'JS Code not sent to runner');
|
||||
|
||||
this.validateTaskSettings(settings);
|
||||
|
||||
if (settings.nodeMode === 'runCode') {
|
||||
const result = await this.runCode(settings, abortSignal);
|
||||
return {
|
||||
result,
|
||||
customData: undefined,
|
||||
staticData: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const neededBuiltInsResult = this.builtInsParser.parseUsedBuiltIns(settings.code);
|
||||
const neededBuiltIns = neededBuiltInsResult.ok
|
||||
? neededBuiltInsResult.result
|
||||
: BuiltInsParserState.newNeedsAllDataState();
|
||||
|
||||
const dataResponse = await this.requestData<DataRequestResponse>(
|
||||
taskId,
|
||||
neededBuiltIns.toDataRequestParams(settings.chunk),
|
||||
);
|
||||
|
||||
const data = this.reconstructTaskData(dataResponse, settings.chunk);
|
||||
|
||||
await this.requestNodeTypeIfNeeded(neededBuiltIns, data.workflow, taskId);
|
||||
|
||||
const workflowParams = data.workflow;
|
||||
const workflow = new Workflow({
|
||||
...workflowParams,
|
||||
nodeTypes: this.nodeTypes,
|
||||
});
|
||||
|
||||
workflow.staticData = ObservableObject.create(workflow.staticData);
|
||||
|
||||
const result =
|
||||
settings.nodeMode === 'runOnceForAllItems'
|
||||
? await this.runForAllItems(taskId, settings, data, workflow, abortSignal)
|
||||
: await this.runForEachItem(taskId, settings, data, workflow, abortSignal);
|
||||
|
||||
return {
|
||||
result,
|
||||
customData: data.runExecutionData.resultData.metadata,
|
||||
staticData: workflow.staticData.__dataChanged ? workflow.staticData : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
private validateTaskSettings(settings: JSExecSettings) {
|
||||
a.ok(settings.code, 'No code to execute');
|
||||
|
||||
if (settings.nodeMode === 'runOnceForAllItems') {
|
||||
a.ok(settings.chunk === undefined, 'Chunking is not supported for runOnceForAllItems');
|
||||
}
|
||||
}
|
||||
|
||||
private getNativeVariables() {
|
||||
return {
|
||||
// Exposed Node.js globals
|
||||
Buffer,
|
||||
setTimeout,
|
||||
setInterval,
|
||||
setImmediate,
|
||||
clearTimeout,
|
||||
clearInterval,
|
||||
clearImmediate,
|
||||
|
||||
// Missing JS natives
|
||||
btoa,
|
||||
atob,
|
||||
TextDecoder,
|
||||
TextDecoderStream,
|
||||
TextEncoder,
|
||||
TextEncoderStream,
|
||||
FormData,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the given code in an environment that doesn't have access to
|
||||
* builtins or RPC and doesn't fetch any input data separately. Any data
|
||||
* can be passed in as additional properties.
|
||||
*/
|
||||
async runCode(settings: JSExecSettings, abortSignal: AbortSignal): Promise<unknown> {
|
||||
const context = createContext({
|
||||
__isExecutionContext: true,
|
||||
...settings.additionalProperties,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new Promise<unknown>((resolve, reject) => {
|
||||
const abortHandler = () => {
|
||||
reject(new TimeoutError(this.taskTimeout));
|
||||
};
|
||||
|
||||
abortSignal.addEventListener('abort', abortHandler, { once: true });
|
||||
|
||||
// We don't need to check for the insecure mode since we are not
|
||||
// giving access to any third party libraries.
|
||||
const taskResult: Promise<unknown> = runInContext(
|
||||
this.createVmExecutableCode(settings.code),
|
||||
context,
|
||||
{ timeout: this.taskTimeout * 1000 },
|
||||
) as Promise<unknown>;
|
||||
|
||||
void taskResult
|
||||
.then(resolve)
|
||||
.catch(reject)
|
||||
.finally(() => {
|
||||
abortSignal.removeEventListener('abort', abortHandler);
|
||||
});
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (e) {
|
||||
throw this.toExecutionErrorIfNeeded(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the requested code for all items in a single run
|
||||
*/
|
||||
private async runForAllItems(
|
||||
taskId: string,
|
||||
settings: JSExecSettings,
|
||||
data: JsTaskData,
|
||||
workflow: Workflow,
|
||||
signal: AbortSignal,
|
||||
): Promise<TaskResultData['result']> {
|
||||
const dataProxy = this.createDataProxy(data, workflow, data.itemIndex);
|
||||
const inputItems = data.connectionInputData;
|
||||
|
||||
const context = this.buildContext(taskId, workflow, data.node, dataProxy, {
|
||||
items: inputItems,
|
||||
...settings.additionalProperties,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new Promise<TaskResultData['result']>((resolve, reject) => {
|
||||
const abortHandler = () => {
|
||||
reject(new TimeoutError(this.taskTimeout));
|
||||
};
|
||||
|
||||
signal.addEventListener('abort', abortHandler, { once: true });
|
||||
|
||||
let taskResult: Promise<TaskResultData['result']>;
|
||||
|
||||
if (this.mode === 'secure') {
|
||||
taskResult = runInContext(this.createVmExecutableCode(settings.code), context, {
|
||||
timeout: this.taskTimeout * 1000,
|
||||
}) as Promise<TaskResultData['result']>;
|
||||
} else {
|
||||
taskResult = this.runDirectly<TaskResultData['result']>(settings.code, context);
|
||||
}
|
||||
|
||||
void taskResult
|
||||
.then(resolve)
|
||||
.catch(reject)
|
||||
.finally(() => {
|
||||
signal.removeEventListener('abort', abortHandler);
|
||||
});
|
||||
});
|
||||
|
||||
if (result === null) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (e) {
|
||||
// Errors thrown by the VM are not instances of Error, so map them to an ExecutionError
|
||||
const error = this.toExecutionErrorIfNeeded(e);
|
||||
|
||||
if (settings.continueOnFail) {
|
||||
return [{ json: { error: error.message } }];
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the requested code for each item in the input data
|
||||
*/
|
||||
private async runForEachItem(
|
||||
taskId: string,
|
||||
settings: JSExecSettings,
|
||||
data: JsTaskData,
|
||||
workflow: Workflow,
|
||||
signal: AbortSignal,
|
||||
): Promise<INodeExecutionData[]> {
|
||||
const inputItems = data.connectionInputData;
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
// If a chunk was requested, only process the items in the chunk
|
||||
const chunkStartIdx = settings.chunk ? settings.chunk.startIndex : 0;
|
||||
const chunkEndIdx = settings.chunk
|
||||
? settings.chunk.startIndex + settings.chunk.count
|
||||
: inputItems.length;
|
||||
|
||||
const context = this.buildContext(
|
||||
taskId,
|
||||
workflow,
|
||||
data.node,
|
||||
undefined,
|
||||
settings.additionalProperties,
|
||||
);
|
||||
|
||||
for (let index = chunkStartIdx; index < chunkEndIdx; index++) {
|
||||
const dataProxy = this.createDataProxy(data, workflow, index);
|
||||
|
||||
Object.assign(context, dataProxy, { item: inputItems[index] });
|
||||
|
||||
try {
|
||||
const result = await new Promise<INodeExecutionData | undefined>((resolve, reject) => {
|
||||
const abortHandler = () => {
|
||||
reject(new TimeoutError(this.taskTimeout));
|
||||
};
|
||||
|
||||
signal.addEventListener('abort', abortHandler);
|
||||
|
||||
let taskResult: Promise<INodeExecutionData>;
|
||||
|
||||
if (this.mode === 'secure') {
|
||||
taskResult = runInContext(this.createVmExecutableCode(settings.code), context, {
|
||||
timeout: this.taskTimeout * 1000,
|
||||
}) as Promise<INodeExecutionData>;
|
||||
} else {
|
||||
taskResult = this.runDirectly<INodeExecutionData>(settings.code, context);
|
||||
}
|
||||
|
||||
void taskResult
|
||||
.then(resolve)
|
||||
.catch(reject)
|
||||
.finally(() => {
|
||||
signal.removeEventListener('abort', abortHandler);
|
||||
});
|
||||
});
|
||||
|
||||
// Filter out null values
|
||||
if (result === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result) {
|
||||
const jsonData = this.extractJsonData(result);
|
||||
|
||||
returnData.push(
|
||||
result.binary
|
||||
? {
|
||||
json: jsonData,
|
||||
pairedItem: { item: index },
|
||||
binary: result.binary,
|
||||
}
|
||||
: {
|
||||
json: jsonData,
|
||||
pairedItem: { item: index },
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
// Errors thrown by the VM are not instances of Error, so map them to an ExecutionError
|
||||
const error = this.toExecutionErrorIfNeeded(e);
|
||||
|
||||
if (!settings.continueOnFail) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
returnData.push({
|
||||
json: { error: error.message },
|
||||
pairedItem: {
|
||||
item: index,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
|
||||
private createDataProxy(data: JsTaskData, workflow: Workflow, itemIndex: number) {
|
||||
return new WorkflowDataProxy(
|
||||
workflow,
|
||||
data.runExecutionData,
|
||||
data.runIndex,
|
||||
itemIndex,
|
||||
data.activeNodeName,
|
||||
data.connectionInputData,
|
||||
data.siblingParameters,
|
||||
data.mode,
|
||||
getAdditionalKeys(
|
||||
data.additionalData as IWorkflowExecuteAdditionalData,
|
||||
data.mode,
|
||||
data.runExecutionData,
|
||||
),
|
||||
data.executeData,
|
||||
data.defaultReturnRunIndex,
|
||||
data.selfData,
|
||||
data.contextNodeName,
|
||||
// Make sure that even if we don't receive the envProviderState for
|
||||
// whatever reason, we don't expose the task runner's env to the code
|
||||
data.envProviderState ?? {
|
||||
env: {},
|
||||
isEnvAccessBlocked: false,
|
||||
isProcessAvailable: true,
|
||||
},
|
||||
// Because we optimize the needed data, it can be partially available.
|
||||
// We assign the available built-ins to the execution context, which
|
||||
// means we run the getter for '$json', and by default $json throws
|
||||
// if there is no data available.
|
||||
).getDataProxy({ throwOnMissingExecutionData: false });
|
||||
}
|
||||
|
||||
private extractJsonData(result: INodeExecutionData) {
|
||||
if (!isObject(result)) return result;
|
||||
|
||||
if ('json' in result) return result.json;
|
||||
|
||||
if ('binary' in result) {
|
||||
// Pick only json property to prevent metadata duplication
|
||||
return (result as INodeExecutionData).json ?? {};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private toExecutionErrorIfNeeded(error: unknown): Error {
|
||||
if (error instanceof Error) {
|
||||
return makeSerializable(error);
|
||||
}
|
||||
|
||||
if (isErrorLike(error)) {
|
||||
return new ExecutionError(error);
|
||||
}
|
||||
|
||||
return new ExecutionError({ message: JSON.stringify(error) });
|
||||
}
|
||||
|
||||
private reconstructTaskData(
|
||||
response: DataRequestResponse,
|
||||
chunk?: InputDataChunkDefinition,
|
||||
): JsTaskData {
|
||||
const inputData = this.taskDataReconstruct.reconstructConnectionInputItems(
|
||||
response.inputData,
|
||||
chunk,
|
||||
// This type assertion is intentional. Chunking is only supported in
|
||||
// runOnceForEachItem mode and if a chunk was requested, we intentionally
|
||||
// fill the array with undefined values for the items outside the chunk.
|
||||
// We only iterate over the chunk items but WorkflowDataProxy expects
|
||||
// the full array of items.
|
||||
) as INodeExecutionData[];
|
||||
|
||||
return {
|
||||
...response,
|
||||
connectionInputData: inputData,
|
||||
executeData: this.taskDataReconstruct.reconstructExecuteData(response, inputData),
|
||||
};
|
||||
}
|
||||
|
||||
private async requestNodeTypeIfNeeded(
|
||||
neededBuiltIns: BuiltInsParserState,
|
||||
workflow: JsTaskData['workflow'],
|
||||
taskId: string,
|
||||
) {
|
||||
/**
|
||||
* We request node types only when we know a task needs all nodes, because
|
||||
* needing all nodes means that the task relies on paired item functionality,
|
||||
* which is the same requirement for needing node types.
|
||||
*/
|
||||
if (neededBuiltIns.needsAllNodes) {
|
||||
const uniqueNodeTypes = new Map(
|
||||
workflow.nodes.map((node) => [
|
||||
`${node.type}|${node.typeVersion}`,
|
||||
{ name: node.type, version: node.typeVersion },
|
||||
]),
|
||||
);
|
||||
|
||||
const unknownNodeTypes = this.nodeTypes.onlyUnknown([...uniqueNodeTypes.values()]);
|
||||
|
||||
const nodeTypes = await this.requestNodeTypes<INodeTypeDescription[]>(
|
||||
taskId,
|
||||
unknownNodeTypes,
|
||||
);
|
||||
|
||||
this.nodeTypes.addNodeTypeDescriptions(nodeTypes);
|
||||
}
|
||||
}
|
||||
|
||||
private buildRpcCallObject(taskId: string) {
|
||||
const rpcObject: RpcCallObject = {};
|
||||
|
||||
for (const rpcMethod of EXPOSED_RPC_METHODS) {
|
||||
set(
|
||||
rpcObject,
|
||||
rpcMethod.split('.'),
|
||||
async (...args: unknown[]) => await this.makeRpcCall(taskId, rpcMethod, args),
|
||||
);
|
||||
}
|
||||
|
||||
for (const rpcMethod of UNSUPPORTED_HELPER_FUNCTIONS) {
|
||||
set(rpcObject, rpcMethod.split('.'), () => {
|
||||
throw new UnsupportedFunctionError(rpcMethod);
|
||||
});
|
||||
}
|
||||
|
||||
return rpcObject;
|
||||
}
|
||||
|
||||
private buildCustomConsole(taskId: string): CustomConsole {
|
||||
return {
|
||||
// all except `log` are dummy methods that disregard without throwing, following existing Code node behavior
|
||||
...JsTaskRunner.CONSOLE_METHODS.reduce<Record<string, () => void>>((acc, name) => {
|
||||
acc[name] = noOp;
|
||||
return acc;
|
||||
}, {}),
|
||||
|
||||
// Send log output back to the main process. It will take care of forwarding
|
||||
// it to the UI or printing to console.
|
||||
log: (...args: unknown[]) => {
|
||||
const formattedLogArgs = args.map((arg) => {
|
||||
if (isObject(arg) && '__isExecutionContext' in arg) return '[[ExecutionContext]]';
|
||||
if (typeof arg === 'string') return `'${arg}'`;
|
||||
return jsonStringify(arg, { replaceCircularRefs: true });
|
||||
});
|
||||
void this.makeRpcCall(taskId, 'logNodeOutput', formattedLogArgs);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the 'global' context object that is passed to the script
|
||||
*
|
||||
* @param taskId The ID of the task. Needed for RPC calls
|
||||
* @param workflow The workflow that is being executed. Needed for static data
|
||||
* @param node The node that is being executed. Needed for static data
|
||||
* @param dataProxy The data proxy object that provides access to built-ins
|
||||
* @param additionalProperties Additional properties to add to the context
|
||||
*/
|
||||
buildContext(
|
||||
taskId: string,
|
||||
workflow: Workflow,
|
||||
node: INode,
|
||||
dataProxy?: IWorkflowDataProxyData,
|
||||
additionalProperties: Record<string, unknown> = {},
|
||||
): Context {
|
||||
return createContext({
|
||||
__isExecutionContext: true,
|
||||
require: this.requireResolver,
|
||||
console: this.buildCustomConsole(taskId),
|
||||
$getWorkflowStaticData: (type: 'global' | 'node') => workflow.getStaticData(type, node),
|
||||
...this.getNativeVariables(),
|
||||
...dataProxy,
|
||||
...this.buildRpcCallObject(taskId),
|
||||
...additionalProperties,
|
||||
});
|
||||
}
|
||||
|
||||
private createVmExecutableCode(code: string) {
|
||||
return [
|
||||
// shim for `global` compatibility
|
||||
'globalThis.global = globalThis',
|
||||
'var module = { exports: {} }',
|
||||
|
||||
// prevent prototype manipulation
|
||||
'Object.getPrototypeOf = () => ({})',
|
||||
'Reflect.getPrototypeOf = () => ({})',
|
||||
'Object.setPrototypeOf = () => false',
|
||||
'Reflect.setPrototypeOf = () => false',
|
||||
|
||||
// prevent Error.prepareStackTrace RCE attack BEFORE disabling defineProperty
|
||||
// This V8 API allows accessing the real global object via stack frame's getThis()
|
||||
'delete Error.prepareStackTrace',
|
||||
'delete Error.captureStackTrace',
|
||||
'Object.defineProperty(Error, "prepareStackTrace", { configurable: false, writable: false, value: undefined })',
|
||||
'Object.defineProperty(Error, "captureStackTrace", { configurable: false, writable: false, value: undefined })',
|
||||
|
||||
// prevent defineProperty attacks (used to bypass sandbox via Error.prepareStackTrace)
|
||||
// Must come AFTER we've locked down Error properties above
|
||||
'Object.defineProperty = () => ({})',
|
||||
'Object.defineProperties = () => ({})',
|
||||
|
||||
// freeze constructors to prevent static method mutation
|
||||
'[Object, Function, Array, String, Number, Boolean, RegExp, Error, TypeError, RangeError, SyntaxError, ReferenceError, Promise, Symbol, Map, Set, WeakMap, WeakSet, Date, JSON, Math, Reflect, ArrayBuffer, DataView, Int8Array, Uint8Array, Float32Array, Float64Array].forEach((constructor) => { try { Object.freeze(constructor); } catch {} })',
|
||||
|
||||
// wrap user code
|
||||
`module.exports = async function VmCodeWrapper() {${code}\n}()`,
|
||||
].join('; ');
|
||||
}
|
||||
|
||||
private async runDirectly<T>(code: string, context: Context): Promise<T> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
||||
const fn = new Function(
|
||||
'context',
|
||||
`with(context) { return (async function() {${code}\n})(); }`,
|
||||
);
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return
|
||||
return await fn(context);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export function isObject(maybe: unknown): maybe is object {
|
||||
return (
|
||||
typeof maybe === 'object' && maybe !== null && !Array.isArray(maybe) && !(maybe instanceof Date)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { isBuiltin } from 'node:module';
|
||||
|
||||
import { DisallowedModuleError } from './errors/disallowed-module.error';
|
||||
import { ExecutionError } from './errors/execution-error';
|
||||
|
||||
export type RequireResolverOpts = {
|
||||
/**
|
||||
* List of built-in nodejs modules that are allowed to be required in the
|
||||
* execution sandbox. `"*"` means all are allowed.
|
||||
*/
|
||||
allowedBuiltInModules: Set<string> | '*';
|
||||
|
||||
/**
|
||||
* List of external modules that are allowed to be required in the
|
||||
* execution sandbox. `"*"` means all are allowed.
|
||||
*/
|
||||
allowedExternalModules: Set<string> | '*';
|
||||
};
|
||||
|
||||
export type RequireResolver = (request: string) => unknown;
|
||||
|
||||
export function createRequireResolver({
|
||||
allowedBuiltInModules,
|
||||
allowedExternalModules,
|
||||
}: RequireResolverOpts) {
|
||||
return (request: string) => {
|
||||
const checkIsAllowed = (allowList: Set<string> | '*', moduleName: string) => {
|
||||
return allowList === '*' || allowList.has(moduleName);
|
||||
};
|
||||
|
||||
const isAllowed = isBuiltin(request)
|
||||
? checkIsAllowed(allowedBuiltInModules, request)
|
||||
: checkIsAllowed(allowedExternalModules, request);
|
||||
|
||||
if (!isAllowed) {
|
||||
const error = new DisallowedModuleError(request);
|
||||
throw new ExecutionError(error);
|
||||
}
|
||||
|
||||
return require(request) as unknown;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
import type { INodeTypeBaseDescription } from 'n8n-workflow';
|
||||
|
||||
import type {
|
||||
NeededNodeType,
|
||||
AVAILABLE_RPC_METHODS,
|
||||
TaskDataRequestParams,
|
||||
TaskResultData,
|
||||
} from './runner-types';
|
||||
|
||||
export namespace BrokerMessage {
|
||||
export namespace ToRunner {
|
||||
export interface InfoRequest {
|
||||
type: 'broker:inforequest';
|
||||
}
|
||||
|
||||
export interface RunnerRegistered {
|
||||
type: 'broker:runnerregistered';
|
||||
}
|
||||
|
||||
export interface TaskOfferAccept {
|
||||
type: 'broker:taskofferaccept';
|
||||
taskId: string;
|
||||
offerId: string;
|
||||
}
|
||||
|
||||
export interface TaskCancel {
|
||||
type: 'broker:taskcancel';
|
||||
taskId: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface TaskSettings {
|
||||
type: 'broker:tasksettings';
|
||||
taskId: string;
|
||||
settings: unknown;
|
||||
}
|
||||
|
||||
export interface RPCResponse {
|
||||
type: 'broker:rpcresponse';
|
||||
callId: string;
|
||||
taskId: string;
|
||||
status: 'success' | 'error';
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export interface TaskDataResponse {
|
||||
type: 'broker:taskdataresponse';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export interface NodeTypes {
|
||||
type: 'broker:nodetypes';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
nodeTypes: INodeTypeBaseDescription[];
|
||||
}
|
||||
|
||||
/** Signals the runner to stop accepting tasks, complete active ones, and prepare for shutdown. */
|
||||
export interface Drain {
|
||||
type: 'broker:drain';
|
||||
}
|
||||
|
||||
export type All =
|
||||
| InfoRequest
|
||||
| TaskOfferAccept
|
||||
| TaskCancel
|
||||
| TaskSettings
|
||||
| RunnerRegistered
|
||||
| RPCResponse
|
||||
| TaskDataResponse
|
||||
| NodeTypes
|
||||
| Drain;
|
||||
}
|
||||
|
||||
export namespace ToRequester {
|
||||
export interface TaskReady {
|
||||
type: 'broker:taskready';
|
||||
requestId: string;
|
||||
taskId: string;
|
||||
}
|
||||
|
||||
export interface TaskDone {
|
||||
type: 'broker:taskdone';
|
||||
taskId: string;
|
||||
data: TaskResultData;
|
||||
}
|
||||
|
||||
export interface TaskError {
|
||||
type: 'broker:taskerror';
|
||||
taskId: string;
|
||||
error: unknown;
|
||||
}
|
||||
|
||||
export interface RequestExpired {
|
||||
type: 'broker:requestexpired';
|
||||
requestId: string;
|
||||
reason: 'timeout' | 'draining';
|
||||
}
|
||||
|
||||
export interface TaskDataRequest {
|
||||
type: 'broker:taskdatarequest';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
requestParams: TaskDataRequestParams;
|
||||
}
|
||||
|
||||
export interface NodeTypesRequest {
|
||||
type: 'broker:nodetypesrequest';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
requestParams: NeededNodeType[];
|
||||
}
|
||||
|
||||
export interface RPC {
|
||||
type: 'broker:rpc';
|
||||
callId: string;
|
||||
taskId: string;
|
||||
name: (typeof AVAILABLE_RPC_METHODS)[number];
|
||||
params: unknown[];
|
||||
}
|
||||
|
||||
export type All =
|
||||
| TaskReady
|
||||
| TaskDone
|
||||
| TaskError
|
||||
| RequestExpired
|
||||
| TaskDataRequest
|
||||
| NodeTypesRequest
|
||||
| RPC;
|
||||
}
|
||||
}
|
||||
|
||||
export namespace RequesterMessage {
|
||||
export namespace ToBroker {
|
||||
export interface TaskSettings {
|
||||
type: 'requester:tasksettings';
|
||||
taskId: string;
|
||||
settings: unknown;
|
||||
}
|
||||
|
||||
export interface TaskCancel {
|
||||
type: 'requester:taskcancel';
|
||||
taskId: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface TaskDataResponse {
|
||||
type: 'requester:taskdataresponse';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export interface NodeTypesResponse {
|
||||
type: 'requester:nodetypesresponse';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
nodeTypes: INodeTypeBaseDescription[];
|
||||
}
|
||||
|
||||
export interface RPCResponse {
|
||||
type: 'requester:rpcresponse';
|
||||
taskId: string;
|
||||
callId: string;
|
||||
status: 'success' | 'error';
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export interface TaskRequest {
|
||||
type: 'requester:taskrequest';
|
||||
requestId: string;
|
||||
taskType: string;
|
||||
}
|
||||
|
||||
export type All =
|
||||
| TaskSettings
|
||||
| TaskCancel
|
||||
| RPCResponse
|
||||
| TaskDataResponse
|
||||
| NodeTypesResponse
|
||||
| TaskRequest;
|
||||
}
|
||||
}
|
||||
|
||||
export namespace RunnerMessage {
|
||||
export namespace ToBroker {
|
||||
export interface Info {
|
||||
type: 'runner:info';
|
||||
name: string;
|
||||
types: string[];
|
||||
}
|
||||
|
||||
export interface TaskAccepted {
|
||||
type: 'runner:taskaccepted';
|
||||
taskId: string;
|
||||
}
|
||||
|
||||
export interface TaskRejected {
|
||||
type: 'runner:taskrejected';
|
||||
taskId: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** Message where launcher (impersonating runner) requests broker to hold task until runner is ready. */
|
||||
export interface TaskDeferred {
|
||||
type: 'runner:taskdeferred';
|
||||
taskId: string;
|
||||
}
|
||||
|
||||
export interface TaskDone {
|
||||
type: 'runner:taskdone';
|
||||
taskId: string;
|
||||
data: TaskResultData;
|
||||
}
|
||||
|
||||
export interface TaskError {
|
||||
type: 'runner:taskerror';
|
||||
taskId: string;
|
||||
error: unknown;
|
||||
}
|
||||
|
||||
export interface TaskOffer {
|
||||
type: 'runner:taskoffer';
|
||||
offerId: string;
|
||||
taskType: string;
|
||||
validFor: number;
|
||||
}
|
||||
|
||||
export interface TaskDataRequest {
|
||||
type: 'runner:taskdatarequest';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
requestParams: TaskDataRequestParams;
|
||||
}
|
||||
|
||||
export interface NodeTypesRequest {
|
||||
type: 'runner:nodetypesrequest';
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
|
||||
/**
|
||||
* Which node types should be included in the runner's node types request.
|
||||
*
|
||||
* Node types are needed only when the script relies on paired item functionality.
|
||||
* If so, we need only the node types not already cached in the runner.
|
||||
*
|
||||
* TODO: In future we can trim this down to only node types in the paired item chain,
|
||||
* rather than assuming we need all node types in the workflow.
|
||||
*
|
||||
* @example [{ name: 'n8n-nodes-base.httpRequest', version: 1 }]
|
||||
*/
|
||||
requestParams: NeededNodeType[];
|
||||
}
|
||||
|
||||
export interface RPC {
|
||||
type: 'runner:rpc';
|
||||
callId: string;
|
||||
taskId: string;
|
||||
name: (typeof AVAILABLE_RPC_METHODS)[number];
|
||||
params: unknown[];
|
||||
}
|
||||
|
||||
export type All =
|
||||
| Info
|
||||
| TaskDone
|
||||
| TaskError
|
||||
| TaskAccepted
|
||||
| TaskRejected
|
||||
| TaskDeferred
|
||||
| TaskOffer
|
||||
| RPC
|
||||
| TaskDataRequest
|
||||
| NodeTypesRequest;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import {
|
||||
ApplicationError,
|
||||
type IDataObject,
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
type INodeTypes,
|
||||
type IVersionedNodeType,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import type { NeededNodeType } from './runner-types';
|
||||
|
||||
type VersionedTypes = Map<number, INodeTypeDescription>;
|
||||
|
||||
export const DEFAULT_NODETYPE_VERSION = 1;
|
||||
|
||||
export class TaskRunnerNodeTypes implements INodeTypes {
|
||||
private nodeTypesByVersion: Map<string, VersionedTypes>;
|
||||
|
||||
constructor(nodeTypes: INodeTypeDescription[]) {
|
||||
this.nodeTypesByVersion = this.parseNodeTypes(nodeTypes);
|
||||
}
|
||||
|
||||
private parseNodeTypes(nodeTypes: INodeTypeDescription[]): Map<string, VersionedTypes> {
|
||||
const versionedTypes = new Map<string, VersionedTypes>();
|
||||
|
||||
for (const nt of nodeTypes) {
|
||||
const versions = Array.isArray(nt.version)
|
||||
? nt.version
|
||||
: [nt.version ?? DEFAULT_NODETYPE_VERSION];
|
||||
|
||||
const versioned: VersionedTypes =
|
||||
versionedTypes.get(nt.name) ?? new Map<number, INodeTypeDescription>();
|
||||
for (const version of versions) {
|
||||
versioned.set(version, { ...versioned.get(version), ...nt });
|
||||
}
|
||||
|
||||
versionedTypes.set(nt.name, versioned);
|
||||
}
|
||||
|
||||
return versionedTypes;
|
||||
}
|
||||
|
||||
// This isn't used in Workflow from what I can see
|
||||
getByName(_nodeType: string): INodeType | IVersionedNodeType {
|
||||
throw new ApplicationError('Unimplemented `getByName`', { level: 'error' });
|
||||
}
|
||||
|
||||
getByNameAndVersion(nodeType: string, version?: number): INodeType {
|
||||
const versions = this.nodeTypesByVersion.get(nodeType);
|
||||
if (!versions) {
|
||||
return undefined as unknown as INodeType;
|
||||
}
|
||||
const nodeVersion = versions.get(version ?? Math.max(...versions.keys()));
|
||||
if (!nodeVersion) {
|
||||
return undefined as unknown as INodeType;
|
||||
}
|
||||
return {
|
||||
description: nodeVersion,
|
||||
};
|
||||
}
|
||||
|
||||
// This isn't used in Workflow from what I can see
|
||||
getKnownTypes(): IDataObject {
|
||||
throw new ApplicationError('Unimplemented `getKnownTypes`', { level: 'error' });
|
||||
}
|
||||
|
||||
addNodeTypeDescriptions(nodeTypeDescriptions: INodeTypeDescription[]) {
|
||||
const newNodeTypes = this.parseNodeTypes(nodeTypeDescriptions);
|
||||
|
||||
for (const [name, newVersions] of newNodeTypes.entries()) {
|
||||
if (!this.nodeTypesByVersion.has(name)) {
|
||||
this.nodeTypesByVersion.set(name, newVersions);
|
||||
} else {
|
||||
const existingVersions = this.nodeTypesByVersion.get(name)!;
|
||||
for (const [version, nodeType] of newVersions.entries()) {
|
||||
existingVersions.set(version, nodeType);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Filter out node type versions that are already registered. */
|
||||
onlyUnknown(nodeTypes: NeededNodeType[]) {
|
||||
return nodeTypes.filter(({ name, version }) => {
|
||||
const existingVersions = this.nodeTypesByVersion.get(name);
|
||||
|
||||
if (!existingVersions) return true;
|
||||
|
||||
return !existingVersions.has(version);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import type {
|
||||
EnvProviderState,
|
||||
IDataObject,
|
||||
IExecuteData,
|
||||
IExecuteFunctions,
|
||||
INode,
|
||||
INodeExecutionData,
|
||||
INodeParameters,
|
||||
IRunExecutionData,
|
||||
ITaskDataConnections,
|
||||
ITaskDataConnectionsSource,
|
||||
IWorkflowExecuteAdditionalData,
|
||||
Workflow,
|
||||
WorkflowExecuteMode,
|
||||
WorkflowParameters,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export interface InputDataChunkDefinition {
|
||||
startIndex: number;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface InputDataRequestParams {
|
||||
/** Whether to include the input data in the response */
|
||||
include: boolean;
|
||||
/** Optionally request only a specific chunk of data instead of all input data */
|
||||
chunk?: InputDataChunkDefinition;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specifies what data should be included for a task data request.
|
||||
*/
|
||||
export interface TaskDataRequestParams {
|
||||
dataOfNodes: string[] | 'all';
|
||||
prevNode: boolean;
|
||||
/** Whether input data for the node should be included */
|
||||
input: InputDataRequestParams;
|
||||
/** Whether env provider's state should be included */
|
||||
env: boolean;
|
||||
}
|
||||
|
||||
export interface DataRequestResponse {
|
||||
workflow: Omit<WorkflowParameters, 'nodeTypes'>;
|
||||
inputData: ITaskDataConnections;
|
||||
connectionInputSource: ITaskDataConnectionsSource | null;
|
||||
node: INode;
|
||||
|
||||
runExecutionData: IRunExecutionData;
|
||||
runIndex: number;
|
||||
itemIndex: number;
|
||||
activeNodeName: string;
|
||||
siblingParameters: INodeParameters;
|
||||
mode: WorkflowExecuteMode;
|
||||
envProviderState: EnvProviderState;
|
||||
defaultReturnRunIndex: number;
|
||||
selfData: IDataObject;
|
||||
contextNodeName: string;
|
||||
additionalData: PartialAdditionalData;
|
||||
}
|
||||
|
||||
export interface TaskResultData {
|
||||
/** Raw user output, i.e. not yet validated or normalized. */
|
||||
result: unknown;
|
||||
customData?: Record<string, string>;
|
||||
staticData?: IDataObject;
|
||||
}
|
||||
|
||||
export interface TaskData {
|
||||
executeFunctions: IExecuteFunctions;
|
||||
inputData: ITaskDataConnections;
|
||||
node: INode;
|
||||
|
||||
workflow: Workflow;
|
||||
runExecutionData: IRunExecutionData;
|
||||
runIndex: number;
|
||||
itemIndex: number;
|
||||
activeNodeName: string;
|
||||
connectionInputData: INodeExecutionData[];
|
||||
siblingParameters: INodeParameters;
|
||||
mode: WorkflowExecuteMode;
|
||||
envProviderState: EnvProviderState;
|
||||
executeData?: IExecuteData;
|
||||
defaultReturnRunIndex: number;
|
||||
selfData: IDataObject;
|
||||
contextNodeName: string;
|
||||
additionalData: IWorkflowExecuteAdditionalData;
|
||||
}
|
||||
|
||||
export interface PartialAdditionalData {
|
||||
executionId?: string;
|
||||
restartExecutionId?: string;
|
||||
restApiUrl: string;
|
||||
instanceBaseUrl: string;
|
||||
formWaitingBaseUrl: string;
|
||||
webhookBaseUrl: string;
|
||||
webhookWaitingBaseUrl: string;
|
||||
webhookTestBaseUrl: string;
|
||||
currentNodeParameters?: INodeParameters;
|
||||
executionTimeoutTimestamp?: number;
|
||||
userId?: string;
|
||||
variables: IDataObject;
|
||||
}
|
||||
|
||||
/** RPC methods that are exposed directly to the Code Node */
|
||||
export const EXPOSED_RPC_METHODS = [
|
||||
// assertBinaryData(itemIndex: number, propertyName: string): Promise<IBinaryData>
|
||||
'helpers.assertBinaryData',
|
||||
|
||||
// getBinaryDataBuffer(itemIndex: number, propertyName: string): Promise<Buffer>
|
||||
'helpers.getBinaryDataBuffer',
|
||||
|
||||
// prepareBinaryData(binaryData: Buffer, fileName?: string, mimeType?: string): Promise<IBinaryData>
|
||||
'helpers.prepareBinaryData',
|
||||
|
||||
// setBinaryDataBuffer(metadata: IBinaryData, buffer: Buffer): Promise<IBinaryData>
|
||||
'helpers.setBinaryDataBuffer',
|
||||
|
||||
// binaryToString(body: Buffer, encoding?: string): string
|
||||
'helpers.binaryToString',
|
||||
|
||||
// httpRequest(opts: IHttpRequestOptions): Promise<IN8nHttpFullResponse | IN8nHttpResponse>
|
||||
'helpers.httpRequest',
|
||||
|
||||
// (deprecated) request(uriOrObject: string | IRequestOptions, options?: IRequestOptions): Promise<any>;
|
||||
'helpers.request',
|
||||
];
|
||||
|
||||
/** Helpers that exist but that we are not exposing to the Code Node */
|
||||
export const UNSUPPORTED_HELPER_FUNCTIONS = [
|
||||
// These rely on checking the credentials from the current node type (Code Node)
|
||||
// and hence they can't even work (Code Node doesn't have credentials)
|
||||
'helpers.httpRequestWithAuthentication',
|
||||
'helpers.requestWithAuthenticationPaginated',
|
||||
|
||||
// This has been removed
|
||||
'helpers.copyBinaryFile',
|
||||
|
||||
// We can't support streams over RPC without implementing it ourselves
|
||||
'helpers.createReadStream',
|
||||
'helpers.getBinaryStream',
|
||||
|
||||
// Makes no sense to support this, as it returns either a stream or a buffer
|
||||
// and we can't support streams over RPC
|
||||
'helpers.binaryToBuffer',
|
||||
|
||||
// These are pretty low-level, so we shouldn't expose them
|
||||
// (require binary data id, which we don't expose)
|
||||
'helpers.getBinaryMetadata',
|
||||
'helpers.getStoragePath',
|
||||
'helpers.getBinaryPath',
|
||||
|
||||
// We shouldn't allow arbitrary FS writes
|
||||
'helpers.writeContentToFile',
|
||||
|
||||
// Not something we need to expose. Can be done in the node itself
|
||||
// copyInputItems(items: INodeExecutionData[], properties: string[]): IDataObject[]
|
||||
'helpers.copyInputItems',
|
||||
|
||||
// Code Node does these automatically already
|
||||
'helpers.returnJsonArray',
|
||||
'helpers.normalizeItems',
|
||||
|
||||
// The client is instantiated and lives on the n8n instance, so we can't
|
||||
// expose it over RPC without implementing object marshalling
|
||||
'helpers.getSSHClient',
|
||||
|
||||
// Doesn't make sense to expose
|
||||
'helpers.createDeferredPromise',
|
||||
'helpers.constructExecutionMetaData',
|
||||
];
|
||||
|
||||
/** List of all RPC methods that task runner supports */
|
||||
export const AVAILABLE_RPC_METHODS = [...EXPOSED_RPC_METHODS, 'logNodeOutput'] as const;
|
||||
|
||||
/** Node types needed for the runner to execute a task. */
|
||||
export type NeededNodeType = { name: string; version: number };
|
||||
@@ -0,0 +1,87 @@
|
||||
import { Container } from '@n8n/di';
|
||||
import { ensureError, setGlobalState } from 'n8n-workflow';
|
||||
|
||||
import { MainConfig } from './config/main-config';
|
||||
import type { HealthCheckServer } from './health-check-server';
|
||||
import { JsTaskRunner } from './js-task-runner/js-task-runner';
|
||||
import { TaskRunnerSentry } from './task-runner-sentry';
|
||||
|
||||
let healthCheckServer: HealthCheckServer | undefined;
|
||||
let runner: JsTaskRunner | undefined;
|
||||
let isShuttingDown = false;
|
||||
let sentry: TaskRunnerSentry | undefined;
|
||||
|
||||
function createSignalHandler(signal: string, timeoutInS = 10) {
|
||||
return async function onSignal() {
|
||||
if (isShuttingDown) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Received ${signal} signal, shutting down...`);
|
||||
|
||||
setTimeout(() => {
|
||||
console.error('Shutdown timeout reached, forcing shutdown...');
|
||||
process.exit(1);
|
||||
}, timeoutInS * 1000).unref();
|
||||
|
||||
isShuttingDown = true;
|
||||
try {
|
||||
if (runner) {
|
||||
await runner.stop();
|
||||
runner = undefined;
|
||||
void healthCheckServer?.stop();
|
||||
}
|
||||
|
||||
if (sentry) {
|
||||
await sentry.shutdown();
|
||||
sentry = undefined;
|
||||
}
|
||||
} catch (e) {
|
||||
const error = ensureError(e);
|
||||
console.error('Error stopping task runner', { error });
|
||||
} finally {
|
||||
console.log('Task runner stopped');
|
||||
process.exit(0);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
void (async function start() {
|
||||
const config = Container.get(MainConfig);
|
||||
|
||||
setGlobalState({
|
||||
defaultTimezone: config.baseRunnerConfig.timezone,
|
||||
});
|
||||
|
||||
sentry = Container.get(TaskRunnerSentry);
|
||||
try {
|
||||
await sentry.initIfEnabled();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'FAILED TO INITIALIZE SENTRY. ERROR REPORTING WILL BE DISABLED. THIS IS LIKELY A CONFIGURATION OR ENVIRONMENT ISSUE.',
|
||||
error,
|
||||
);
|
||||
sentry = undefined;
|
||||
}
|
||||
|
||||
runner = new JsTaskRunner(config);
|
||||
runner.on('runner:reached-idle-timeout', () => {
|
||||
// Use shorter timeout since we know we don't have any tasks running
|
||||
void createSignalHandler('IDLE_TIMEOUT', 3)();
|
||||
});
|
||||
|
||||
const { enabled, host, port } = config.baseRunnerConfig.healthcheckServer;
|
||||
|
||||
if (enabled) {
|
||||
const { HealthCheckServer } = await import('./health-check-server');
|
||||
healthCheckServer = new HealthCheckServer();
|
||||
await healthCheckServer.start(host, port);
|
||||
}
|
||||
|
||||
process.on('SIGINT', createSignalHandler('SIGINT'));
|
||||
process.on('SIGTERM', createSignalHandler('SIGTERM'));
|
||||
})().catch((e) => {
|
||||
const error = ensureError(e);
|
||||
console.error('Task runner failed to start', { error });
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import { Service } from '@n8n/di';
|
||||
import type { ErrorEvent, Exception } from '@sentry/core';
|
||||
import { ErrorReporter } from 'n8n-core';
|
||||
|
||||
import { SentryConfig } from './config/sentry-config';
|
||||
|
||||
/**
|
||||
* Sentry service for the task runner.
|
||||
*/
|
||||
@Service()
|
||||
export class TaskRunnerSentry {
|
||||
constructor(
|
||||
private readonly config: SentryConfig,
|
||||
private readonly errorReporter: ErrorReporter,
|
||||
) {}
|
||||
|
||||
async initIfEnabled() {
|
||||
const { dsn, n8nVersion, environment, deploymentName, profilesSampleRate, tracesSampleRate } =
|
||||
this.config;
|
||||
|
||||
if (!dsn) return;
|
||||
|
||||
await this.errorReporter.init({
|
||||
serverType: 'task_runner',
|
||||
dsn,
|
||||
release: `n8n@${n8nVersion}`,
|
||||
environment,
|
||||
serverName: deploymentName,
|
||||
beforeSendFilter: this.filterOutUserCodeErrors,
|
||||
withEventLoopBlockDetection: false,
|
||||
tracesSampleRate,
|
||||
profilesSampleRate,
|
||||
eligibleIntegrations: {
|
||||
Http: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async shutdown() {
|
||||
if (!this.config.dsn) return;
|
||||
|
||||
await this.errorReporter.shutdown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter out errors originating from user provided code.
|
||||
* It is possible for users to create code that causes unhandledrejections
|
||||
* that end up in the sentry error reporting.
|
||||
*/
|
||||
filterOutUserCodeErrors = (event: ErrorEvent) => {
|
||||
const error = event?.exception?.values?.[0];
|
||||
|
||||
return error ? this.isUserCodeError(error) : false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if the error is originating from user provided code.
|
||||
* It is possible for users to create code that causes unhandledrejections
|
||||
* that end up in the sentry error reporting.
|
||||
*/
|
||||
private isUserCodeError(error: Exception) {
|
||||
if (
|
||||
error.type === 'EvalError' &&
|
||||
error.value === 'Code generation from strings disallowed for this context' // from --disallow-code-generation-from-strings
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const frames = error.stacktrace?.frames;
|
||||
if (!frames) return false;
|
||||
|
||||
return frames.some((frame) => {
|
||||
if (frame.filename === 'node:vm' && frame.function === 'runInContext') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (frame.filename === 'evalmachine.<anonymous>') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (frame.function === 'VmCodeWrapper') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
import { isSerializedBuffer, toBuffer } from 'n8n-core';
|
||||
import { ApplicationError, ensureError, randomInt } from 'n8n-workflow';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { type MessageEvent, WebSocket } from 'ws';
|
||||
|
||||
import type { BaseRunnerConfig } from '@/config/base-runner-config';
|
||||
import { TimeoutError } from '@/js-task-runner/errors/timeout-error';
|
||||
import type { BrokerMessage, RunnerMessage } from '@/message-types';
|
||||
import { TaskRunnerNodeTypes } from '@/node-types';
|
||||
import type { TaskResultData } from '@/runner-types';
|
||||
import { TaskState } from '@/task-state';
|
||||
|
||||
import { TaskCancelledError } from './js-task-runner/errors/task-cancelled-error';
|
||||
|
||||
export interface TaskOffer {
|
||||
offerId: string;
|
||||
validUntil: bigint;
|
||||
}
|
||||
|
||||
interface DataRequest {
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
resolve: (data: unknown) => void;
|
||||
reject: (error: unknown) => void;
|
||||
}
|
||||
|
||||
interface NodeTypesRequest {
|
||||
taskId: string;
|
||||
requestId: string;
|
||||
resolve: (data: unknown) => void;
|
||||
reject: (error: unknown) => void;
|
||||
}
|
||||
|
||||
interface RPCCall {
|
||||
callId: string;
|
||||
resolve: (data: unknown) => void;
|
||||
reject: (error: unknown) => void;
|
||||
}
|
||||
|
||||
const OFFER_VALID_TIME_MS = 5000;
|
||||
const OFFER_VALID_EXTRA_MS = 100;
|
||||
|
||||
/** Converts milliseconds to nanoseconds */
|
||||
const msToNs = (ms: number) => BigInt(ms * 1_000_000);
|
||||
|
||||
export const noOp = () => {};
|
||||
|
||||
/** Params the task receives when it is executed */
|
||||
export interface TaskParams<T = unknown> {
|
||||
taskId: string;
|
||||
settings: T;
|
||||
}
|
||||
|
||||
export interface TaskRunnerOpts extends BaseRunnerConfig {
|
||||
taskType: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
export abstract class TaskRunner extends EventEmitter {
|
||||
id: string = nanoid();
|
||||
|
||||
ws: WebSocket;
|
||||
|
||||
canSendOffers = false;
|
||||
|
||||
runningTasks: Map<TaskState['taskId'], TaskState> = new Map();
|
||||
|
||||
offerInterval: NodeJS.Timeout | undefined;
|
||||
|
||||
openOffers: Map<TaskOffer['offerId'], TaskOffer> = new Map();
|
||||
|
||||
dataRequests: Map<DataRequest['requestId'], DataRequest> = new Map();
|
||||
|
||||
nodeTypesRequests: Map<NodeTypesRequest['requestId'], NodeTypesRequest> = new Map();
|
||||
|
||||
rpcCalls: Map<RPCCall['callId'], RPCCall> = new Map();
|
||||
|
||||
nodeTypes: TaskRunnerNodeTypes = new TaskRunnerNodeTypes([]);
|
||||
|
||||
taskType: string;
|
||||
|
||||
maxConcurrency: number;
|
||||
|
||||
name: string;
|
||||
|
||||
private idleTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
/** How long (in seconds) a task is allowed to take for completion, else the task will be aborted. */
|
||||
protected readonly taskTimeout: number;
|
||||
|
||||
/** How long (in seconds) a runner may be idle for before exit. */
|
||||
private readonly idleTimeout: number;
|
||||
|
||||
constructor(opts: TaskRunnerOpts) {
|
||||
super();
|
||||
|
||||
this.taskType = opts.taskType;
|
||||
this.name = opts.name ?? 'Node.js Task Runner SDK';
|
||||
this.maxConcurrency = opts.maxConcurrency;
|
||||
this.taskTimeout = opts.taskTimeout;
|
||||
this.idleTimeout = opts.idleTimeout;
|
||||
|
||||
const { host: taskBrokerHost } = new URL(opts.taskBrokerUri);
|
||||
|
||||
const wsUrl = `ws://${taskBrokerHost}/runners/_ws?id=${this.id}`;
|
||||
this.ws = new WebSocket(wsUrl, {
|
||||
headers: {
|
||||
authorization: `Bearer ${opts.grantToken}`,
|
||||
},
|
||||
maxPayload: opts.maxPayloadSize,
|
||||
});
|
||||
|
||||
this.ws.addEventListener('error', (event) => {
|
||||
const error = ensureError(event.error);
|
||||
|
||||
if (
|
||||
'code' in error &&
|
||||
typeof error.code === 'string' &&
|
||||
['ECONNREFUSED', 'ENOTFOUND'].some((code) => code === error.code)
|
||||
) {
|
||||
console.error(
|
||||
`Error: Failed to connect to n8n task broker. Please ensure n8n task broker is reachable at: ${taskBrokerHost}`,
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.error(`Error: Failed to connect to n8n task broker at ${taskBrokerHost}`);
|
||||
console.error('Details:', event.message || 'Unknown error');
|
||||
}
|
||||
});
|
||||
this.ws.addEventListener('message', this.receiveMessage);
|
||||
this.ws.addEventListener('close', this.stopTaskOffers);
|
||||
this.resetIdleTimer();
|
||||
}
|
||||
|
||||
private resetIdleTimer() {
|
||||
if (this.idleTimeout === 0) return;
|
||||
|
||||
this.clearIdleTimer();
|
||||
|
||||
this.idleTimer = setTimeout(() => {
|
||||
if (this.runningTasks.size === 0) this.emit('runner:reached-idle-timeout');
|
||||
}, this.idleTimeout * 1000);
|
||||
}
|
||||
|
||||
private receiveMessage = (message: MessageEvent) => {
|
||||
// eslint-disable-next-line n8n-local-rules/no-uncaught-json-parse
|
||||
const data = JSON.parse(message.data as string) as BrokerMessage.ToRunner.All;
|
||||
void this.onMessage(data);
|
||||
};
|
||||
|
||||
private stopTaskOffers = () => {
|
||||
this.canSendOffers = false;
|
||||
if (this.offerInterval) {
|
||||
clearInterval(this.offerInterval);
|
||||
this.offerInterval = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
private startTaskOffers() {
|
||||
this.canSendOffers = true;
|
||||
if (this.offerInterval) {
|
||||
clearInterval(this.offerInterval);
|
||||
}
|
||||
this.offerInterval = setInterval(() => this.sendOffers(), 250);
|
||||
}
|
||||
|
||||
deleteStaleOffers() {
|
||||
this.openOffers.forEach((offer, key) => {
|
||||
if (offer.validUntil < process.hrtime.bigint()) {
|
||||
this.openOffers.delete(key);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
sendOffers() {
|
||||
this.deleteStaleOffers();
|
||||
|
||||
if (!this.canSendOffers) {
|
||||
return;
|
||||
}
|
||||
|
||||
const offersToSend = this.maxConcurrency - (this.openOffers.size + this.runningTasks.size);
|
||||
|
||||
for (let i = 0; i < offersToSend; i++) {
|
||||
// Add a bit of randomness so that not all offers expire at the same time
|
||||
const validForInMs = OFFER_VALID_TIME_MS + randomInt(500);
|
||||
// Add a little extra time to account for latency
|
||||
const validUntil = process.hrtime.bigint() + msToNs(validForInMs + OFFER_VALID_EXTRA_MS);
|
||||
const offer: TaskOffer = {
|
||||
offerId: nanoid(),
|
||||
validUntil,
|
||||
};
|
||||
this.openOffers.set(offer.offerId, offer);
|
||||
this.send({
|
||||
type: 'runner:taskoffer',
|
||||
taskType: this.taskType,
|
||||
offerId: offer.offerId,
|
||||
validFor: validForInMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
send(message: RunnerMessage.ToBroker.All) {
|
||||
this.ws.send(JSON.stringify(message));
|
||||
}
|
||||
|
||||
onMessage(message: BrokerMessage.ToRunner.All) {
|
||||
switch (message.type) {
|
||||
case 'broker:inforequest':
|
||||
this.send({
|
||||
type: 'runner:info',
|
||||
name: this.name,
|
||||
types: [this.taskType],
|
||||
});
|
||||
break;
|
||||
case 'broker:runnerregistered':
|
||||
this.startTaskOffers();
|
||||
break;
|
||||
case 'broker:taskofferaccept':
|
||||
this.offerAccepted(message.offerId, message.taskId);
|
||||
break;
|
||||
case 'broker:taskcancel':
|
||||
void this.taskCancelled(message.taskId, message.reason);
|
||||
break;
|
||||
case 'broker:tasksettings':
|
||||
void this.receivedSettings(message.taskId, message.settings);
|
||||
break;
|
||||
case 'broker:taskdataresponse':
|
||||
this.processDataResponse(message.requestId, message.data);
|
||||
break;
|
||||
case 'broker:rpcresponse':
|
||||
this.handleRpcResponse(message.callId, message.status, message.data);
|
||||
break;
|
||||
case 'broker:nodetypes':
|
||||
this.processNodeTypesResponse(message.requestId, message.nodeTypes);
|
||||
break;
|
||||
case 'broker:drain':
|
||||
this.stopTaskOffers();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
processDataResponse(requestId: string, data: unknown) {
|
||||
const request = this.dataRequests.get(requestId);
|
||||
if (!request) {
|
||||
return;
|
||||
}
|
||||
// Deleting of the request is handled in `requestData`, using a
|
||||
// `finally` wrapped around the return
|
||||
request.resolve(data);
|
||||
}
|
||||
|
||||
processNodeTypesResponse(requestId: string, nodeTypes: unknown) {
|
||||
const request = this.nodeTypesRequests.get(requestId);
|
||||
|
||||
if (!request) return;
|
||||
|
||||
// Deleting of the request is handled in `requestNodeTypes`, using a
|
||||
// `finally` wrapped around the return
|
||||
request.resolve(nodeTypes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the task runner has capacity to accept more tasks.
|
||||
*/
|
||||
hasOpenTaskSlots() {
|
||||
return this.runningTasks.size < this.maxConcurrency;
|
||||
}
|
||||
|
||||
offerAccepted(offerId: string, taskId: string) {
|
||||
if (!this.hasOpenTaskSlots()) {
|
||||
this.openOffers.delete(offerId);
|
||||
this.send({
|
||||
type: 'runner:taskrejected',
|
||||
taskId,
|
||||
reason: 'No open task slots - runner already at capacity',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const offer = this.openOffers.get(offerId);
|
||||
if (!offer) {
|
||||
this.send({
|
||||
type: 'runner:taskrejected',
|
||||
taskId,
|
||||
reason: 'Offer expired - not accepted within validity window',
|
||||
});
|
||||
return;
|
||||
} else {
|
||||
this.openOffers.delete(offerId);
|
||||
}
|
||||
|
||||
this.resetIdleTimer();
|
||||
const taskState = new TaskState({
|
||||
taskId,
|
||||
timeoutInS: this.taskTimeout,
|
||||
onTimeout: () => {
|
||||
void this.taskTimedOut(taskId);
|
||||
},
|
||||
});
|
||||
this.runningTasks.set(taskId, taskState);
|
||||
|
||||
this.send({
|
||||
type: 'runner:taskaccepted',
|
||||
taskId,
|
||||
});
|
||||
}
|
||||
|
||||
async taskCancelled(taskId: string, reason: string) {
|
||||
const taskState = this.runningTasks.get(taskId);
|
||||
if (!taskState) {
|
||||
return;
|
||||
}
|
||||
|
||||
await taskState.caseOf({
|
||||
// If the cancelled task hasn't received settings yet, we can finish it
|
||||
waitingForSettings: () => this.finishTask(taskState),
|
||||
|
||||
// If the task has already timed out or is already cancelled, we can
|
||||
// ignore the cancellation
|
||||
'aborting:timeout': noOp,
|
||||
'aborting:cancelled': noOp,
|
||||
|
||||
running: () => {
|
||||
taskState.status = 'aborting:cancelled';
|
||||
taskState.abortController.abort('cancelled');
|
||||
this.cancelTaskRequests(taskId, reason);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async taskTimedOut(taskId: string) {
|
||||
const taskState = this.runningTasks.get(taskId);
|
||||
if (!taskState) {
|
||||
return;
|
||||
}
|
||||
|
||||
await taskState.caseOf({
|
||||
// If we are still waiting for settings for the task, we can error the
|
||||
// task immediately
|
||||
waitingForSettings: () => {
|
||||
try {
|
||||
this.send({
|
||||
type: 'runner:taskerror',
|
||||
taskId,
|
||||
error: new TimeoutError(this.taskTimeout),
|
||||
});
|
||||
} finally {
|
||||
this.finishTask(taskState);
|
||||
}
|
||||
},
|
||||
|
||||
// This should never happen, the timeout timer should only fire once
|
||||
'aborting:timeout': TaskState.throwUnexpectedTaskStatus,
|
||||
|
||||
// If we are currently executing the task, abort the execution and
|
||||
// mark the task as timed out
|
||||
running: () => {
|
||||
taskState.status = 'aborting:timeout';
|
||||
taskState.abortController.abort('timeout');
|
||||
this.cancelTaskRequests(taskId, 'timeout');
|
||||
},
|
||||
|
||||
// If the task is already cancelling, we can ignore the timeout
|
||||
'aborting:cancelled': noOp,
|
||||
});
|
||||
}
|
||||
|
||||
async receivedSettings(taskId: string, settings: unknown) {
|
||||
const taskState = this.runningTasks.get(taskId);
|
||||
if (!taskState) {
|
||||
return;
|
||||
}
|
||||
|
||||
await taskState.caseOf({
|
||||
// These states should never happen, as they are handled already in
|
||||
// the other lifecycle methods and the task should be removed from the
|
||||
// running tasks
|
||||
'aborting:cancelled': TaskState.throwUnexpectedTaskStatus,
|
||||
'aborting:timeout': TaskState.throwUnexpectedTaskStatus,
|
||||
running: TaskState.throwUnexpectedTaskStatus,
|
||||
|
||||
waitingForSettings: async () => {
|
||||
taskState.status = 'running';
|
||||
|
||||
await this.executeTask(
|
||||
{
|
||||
taskId,
|
||||
settings,
|
||||
},
|
||||
taskState.abortController.signal,
|
||||
)
|
||||
.then(async (data) => await this.taskExecutionSucceeded(taskState, data))
|
||||
.catch(async (error) => await this.taskExecutionFailed(taskState, error));
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async executeTask(_taskParams: TaskParams, _signal: AbortSignal): Promise<TaskResultData> {
|
||||
throw new ApplicationError('Unimplemented');
|
||||
}
|
||||
|
||||
async requestNodeTypes<T = unknown>(
|
||||
taskId: TaskState['taskId'],
|
||||
requestParams: RunnerMessage.ToBroker.NodeTypesRequest['requestParams'],
|
||||
) {
|
||||
const requestId = nanoid();
|
||||
|
||||
const nodeTypesPromise = new Promise<T>((resolve, reject) => {
|
||||
this.nodeTypesRequests.set(requestId, {
|
||||
requestId,
|
||||
taskId,
|
||||
resolve: resolve as (data: unknown) => void,
|
||||
reject,
|
||||
});
|
||||
});
|
||||
|
||||
this.send({
|
||||
type: 'runner:nodetypesrequest',
|
||||
taskId,
|
||||
requestId,
|
||||
requestParams,
|
||||
});
|
||||
|
||||
try {
|
||||
return await nodeTypesPromise;
|
||||
} finally {
|
||||
this.nodeTypesRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
async requestData<T = unknown>(
|
||||
taskId: TaskState['taskId'],
|
||||
requestParams: RunnerMessage.ToBroker.TaskDataRequest['requestParams'],
|
||||
): Promise<T> {
|
||||
const requestId = nanoid();
|
||||
|
||||
const dataRequestPromise = new Promise<T>((resolve, reject) => {
|
||||
this.dataRequests.set(requestId, {
|
||||
requestId,
|
||||
taskId,
|
||||
resolve: resolve as (data: unknown) => void,
|
||||
reject,
|
||||
});
|
||||
});
|
||||
|
||||
this.send({
|
||||
type: 'runner:taskdatarequest',
|
||||
taskId,
|
||||
requestId,
|
||||
requestParams,
|
||||
});
|
||||
|
||||
try {
|
||||
return await dataRequestPromise;
|
||||
} finally {
|
||||
this.dataRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
async makeRpcCall(taskId: string, name: RunnerMessage.ToBroker.RPC['name'], params: unknown[]) {
|
||||
const callId = nanoid();
|
||||
|
||||
const dataPromise = new Promise((resolve, reject) => {
|
||||
this.rpcCalls.set(callId, {
|
||||
callId,
|
||||
resolve,
|
||||
reject,
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
this.send({
|
||||
type: 'runner:rpc',
|
||||
callId,
|
||||
taskId,
|
||||
name,
|
||||
params,
|
||||
});
|
||||
|
||||
const returnValue = await dataPromise;
|
||||
|
||||
return isSerializedBuffer(returnValue) ? toBuffer(returnValue) : returnValue;
|
||||
} finally {
|
||||
this.rpcCalls.delete(callId);
|
||||
}
|
||||
}
|
||||
|
||||
handleRpcResponse(
|
||||
callId: string,
|
||||
status: BrokerMessage.ToRunner.RPCResponse['status'],
|
||||
data: unknown,
|
||||
) {
|
||||
const call = this.rpcCalls.get(callId);
|
||||
if (!call) {
|
||||
return;
|
||||
}
|
||||
if (status === 'success') {
|
||||
call.resolve(data);
|
||||
} else {
|
||||
call.reject(typeof data === 'string' ? new Error(data) : data);
|
||||
}
|
||||
}
|
||||
|
||||
/** Close the connection gracefully and wait until has been closed */
|
||||
async stop() {
|
||||
this.clearIdleTimer();
|
||||
|
||||
this.stopTaskOffers();
|
||||
|
||||
await this.waitUntilAllTasksAreDone();
|
||||
|
||||
await this.closeConnection();
|
||||
}
|
||||
|
||||
clearIdleTimer() {
|
||||
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||
this.idleTimer = undefined;
|
||||
}
|
||||
|
||||
private async closeConnection() {
|
||||
// 1000 is the standard close code
|
||||
// https://www.rfc-editor.org/rfc/rfc6455.html#section-7.1.5
|
||||
this.ws.close(1000, 'Shutting down');
|
||||
|
||||
await new Promise((resolve) => {
|
||||
this.ws.once('close', resolve);
|
||||
});
|
||||
}
|
||||
|
||||
private async waitUntilAllTasksAreDone(maxWaitTimeInMs = 30_000) {
|
||||
// TODO: Make maxWaitTimeInMs configurable
|
||||
const start = Date.now();
|
||||
|
||||
while (this.runningTasks.size > 0) {
|
||||
if (Date.now() - start > maxWaitTimeInMs) {
|
||||
throw new ApplicationError('Timeout while waiting for tasks to finish');
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
|
||||
private async taskExecutionSucceeded(taskState: TaskState, data: TaskResultData) {
|
||||
try {
|
||||
const sendData = () => {
|
||||
this.send({
|
||||
type: 'runner:taskdone',
|
||||
taskId: taskState.taskId,
|
||||
data,
|
||||
});
|
||||
};
|
||||
|
||||
await taskState.caseOf({
|
||||
waitingForSettings: TaskState.throwUnexpectedTaskStatus,
|
||||
|
||||
'aborting:cancelled': noOp,
|
||||
|
||||
// If the task timed out but we ended up reaching this point, we
|
||||
// might as well send the data
|
||||
'aborting:timeout': sendData,
|
||||
running: sendData,
|
||||
});
|
||||
} finally {
|
||||
this.finishTask(taskState);
|
||||
}
|
||||
}
|
||||
|
||||
private async taskExecutionFailed(taskState: TaskState, error: unknown) {
|
||||
try {
|
||||
const sendError = () => {
|
||||
this.send({
|
||||
type: 'runner:taskerror',
|
||||
taskId: taskState.taskId,
|
||||
error,
|
||||
});
|
||||
};
|
||||
|
||||
await taskState.caseOf({
|
||||
waitingForSettings: TaskState.throwUnexpectedTaskStatus,
|
||||
|
||||
'aborting:cancelled': noOp,
|
||||
|
||||
'aborting:timeout': () => {
|
||||
console.warn(`Task ${taskState.taskId} timed out`);
|
||||
|
||||
sendError();
|
||||
},
|
||||
|
||||
running: sendError,
|
||||
});
|
||||
} finally {
|
||||
this.finishTask(taskState);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels all node type and data requests made by the given task
|
||||
*/
|
||||
private cancelTaskRequests(taskId: string, reason: string) {
|
||||
for (const [requestId, request] of this.dataRequests.entries()) {
|
||||
if (request.taskId === taskId) {
|
||||
request.reject(new TaskCancelledError(reason));
|
||||
this.dataRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [requestId, request] of this.nodeTypesRequests.entries()) {
|
||||
if (request.taskId === taskId) {
|
||||
request.reject(new TaskCancelledError(reason));
|
||||
this.nodeTypesRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finishes task by removing it from the running tasks and sending new offers
|
||||
*/
|
||||
private finishTask(taskState: TaskState) {
|
||||
taskState.cleanup();
|
||||
this.runningTasks.delete(taskState.taskId);
|
||||
this.sendOffers();
|
||||
this.resetIdleTimer();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import * as a from 'node:assert';
|
||||
|
||||
export type TaskStatus =
|
||||
| 'waitingForSettings'
|
||||
| 'running'
|
||||
| 'aborting:cancelled'
|
||||
| 'aborting:timeout';
|
||||
|
||||
export type TaskStateOpts = {
|
||||
taskId: string;
|
||||
timeoutInS: number;
|
||||
onTimeout: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The state of a task. The task can be in one of the following states:
|
||||
* - waitingForSettings: The task is waiting for settings from the broker
|
||||
* - running: The task is currently running
|
||||
* - aborting:cancelled: The task was canceled by the broker and is being aborted
|
||||
* - aborting:timeout: The task took too long to complete and is being aborted
|
||||
*
|
||||
* The task is discarded once it reaches an end state.
|
||||
*
|
||||
* The class only holds the state, and does not have any logic.
|
||||
*
|
||||
* The task has the following lifecycle:
|
||||
*
|
||||
* ┌───┐
|
||||
* └───┘
|
||||
* │
|
||||
* broker:taskofferaccept : create task state
|
||||
* │
|
||||
* ▼
|
||||
* ┌────────────────────┐ broker:taskcancel / timeout
|
||||
* │ waitingForSettings ├──────────────────────────────────┐
|
||||
* └────────┬───────────┘ │
|
||||
* │ │
|
||||
* broker:tasksettings │
|
||||
* │ │
|
||||
* ▼ │
|
||||
* ┌───────────────┐ ┌────────────────────┐ │
|
||||
* │ running │ │ aborting:timeout │ │
|
||||
* │ │ timeout │ │ │
|
||||
* ┌───────┤- execute task ├───────────►│- fire abort signal │ │
|
||||
* │ └──────┬────────┘ └──────────┬─────────┘ │
|
||||
* │ │ │ │
|
||||
* │ broker:taskcancel │ │
|
||||
* Task execution │ Task execution │
|
||||
* resolves / rejects │ resolves / rejects │
|
||||
* │ ▼ │ │
|
||||
* │ ┌─────────────────────┐ │ │
|
||||
* │ │ aborting:cancelled │ │ │
|
||||
* │ │ │ │ │
|
||||
* │ │- fire abort signal │ │ │
|
||||
* │ └──────────┬──────────┘ │ │
|
||||
* │ Task execution │ │
|
||||
* │ resolves / rejects │ │
|
||||
* │ │ │ │
|
||||
* │ ▼ │ │
|
||||
* │ ┌──┐ │ │
|
||||
* └─────────────►│ │◄────────────────────────────┴─────────────┘
|
||||
* └──┘
|
||||
*/
|
||||
export class TaskState {
|
||||
status: TaskStatus = 'waitingForSettings';
|
||||
|
||||
readonly taskId: string;
|
||||
|
||||
/** Controller for aborting the execution of the task */
|
||||
readonly abortController = new AbortController();
|
||||
|
||||
/** Timeout timer for the task */
|
||||
private timeoutTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
constructor(opts: TaskStateOpts) {
|
||||
this.taskId = opts.taskId;
|
||||
this.timeoutTimer = setTimeout(opts.onTimeout, opts.timeoutInS * 1000);
|
||||
}
|
||||
|
||||
/** Cleans up any resources before the task can be removed */
|
||||
cleanup() {
|
||||
clearTimeout(this.timeoutTimer);
|
||||
this.timeoutTimer = undefined;
|
||||
}
|
||||
|
||||
/** Custom JSON serialization for the task state for logging purposes */
|
||||
toJSON() {
|
||||
return `[Task ${this.taskId} (${this.status})]`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the function matching the current task status
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* taskState.caseOf({
|
||||
* waitingForSettings: () => {...},
|
||||
* running: () => {...},
|
||||
* aborting:cancelled: () => {...},
|
||||
* aborting:timeout: () => {...},
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
async caseOf(
|
||||
conditions: Record<TaskStatus, (taskState: TaskState) => void | Promise<void> | never>,
|
||||
) {
|
||||
if (!conditions[this.status]) {
|
||||
TaskState.throwUnexpectedTaskStatus(this);
|
||||
}
|
||||
|
||||
return await conditions[this.status](this);
|
||||
}
|
||||
|
||||
/** Throws an error that the task status is unexpected */
|
||||
static throwUnexpectedTaskStatus = (taskState: TaskState) => {
|
||||
a.fail(`Unexpected task status: ${JSON.stringify(taskState)}`);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": ["./tsconfig.json", "@n8n/typescript-config/tsconfig.build.json"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"tsBuildInfoFile": "dist/build.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/__tests__/**"]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"extends": [
|
||||
"@n8n/typescript-config/tsconfig.common.json",
|
||||
"@n8n/typescript-config/tsconfig.backend.json"
|
||||
],
|
||||
"compilerOptions": {
|
||||
"rootDir": ".",
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"baseUrl": "src",
|
||||
"paths": {
|
||||
"@/*": ["./*"]
|
||||
},
|
||||
"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user