first commit

packages/testing/performance/README.md

@@ -0,0 +1,90 @@
+# Performance Benchmarks
+
+Microbenchmarks for measuring and tracking performance of critical code paths.
+
+## When to Use Benchmarks
+
+**Good fit:**
+- Hot paths executed thousands of times (expression evaluation, data transforms)
+- Comparing implementation approaches (current vs proposed)
+- Detecting regressions in critical code
+
+**Not a good fit:**
+- API endpoint latency (use load testing - k6, artillery)
+- Database query performance (use query analysis tools)
+- Frontend rendering (use browser profiling)
+- One-off operations (startup time, migrations)
+
+**Rule of thumb:** If it runs millions of times per day across all users, benchmark it.
+
+## Commands
+
+```bash
+pnpm --filter=@n8n/performance bench          # Run benchmarks
+pnpm --filter=@n8n/performance bench:baseline  # Save baseline for local comparison
+pnpm --filter=@n8n/performance bench:compare   # Compare against baseline (>10% = fail)
+```
+
+## CI Regression Detection
+
+Benchmarks run automatically on PRs that touch `packages/testing/performance/**` or `packages/workflow/src/**`. [CodSpeed](https://codspeed.io) counts CPU instructions instead of wall-clock time, producing deterministic results regardless of runner load. It comments on PRs with results and regression warnings.
+
+You can also trigger benchmarks manually for any branch via **Actions > Test: Benchmarks > Run workflow**.
+
+### Local vs CI
+
+| | Local (`bench`) | CI |
+|---|---|---|
+| **Measurement** | Wall-clock time (Hz, ms) | CPU instruction count |
+| **Noise** | 15-30% variance | Near-zero variance |
+| **Best for** | Quick sanity checks, comparing approaches | Automated regression detection |
+
+Local benchmarks are useful for eyeballing performance during development. Use `bench:baseline` + `bench:compare` for before/after comparisons on the same machine in the same session.
+
+## Adding a Benchmark
+
+```typescript
+// benchmarks/my-feature/thing.bench.ts
+import { bench, describe } from 'vitest';
+
+// Setup runs once, not measured
+const data = createTestData();
+
+describe('My Feature', () => {
+  bench('operation name', () => {
+    doTheThing(data);
+  });
+});
+```
+
+## Reading Results
+
+```
+name                hz      min    max   mean    p99    rme   samples
+my operation    20,000   0.04   0.20   0.05   0.10  ±0.5%   10000
+```
+
+| Column | Meaning |
+|--------|---------|
+| hz | Operations per second (higher = faster) |
+| mean | Average time per operation in ms |
+| p99 | 99th percentile - worst case latency |
+| rme | Margin of error - lower = more reliable |
+| samples | Number of iterations run |
+
+## Current Benchmarks
+
+| Area | What it measures | Why it matters |
+|------|------------------|----------------|
+| Expression Engine | `={{ }}` evaluation speed | Runs for every node parameter |
+
+## Notes
+
+This package pins `vitest@^3.2.0` independently from the monorepo catalog (`^3.1.3`) because CodSpeed requires vitest 3.2+.
+
+## Tips
+
+1. **Keep benchmarks focused** - one thing per bench, not workflows
+2. **Use realistic data sizes** - 100 items is typical, 10k is stress test
+3. **Compare approaches** - benchmark both before deciding
+4. **Don't over-benchmark** - only critical hot paths need this

packages/testing/performance/benchmarks/expression-engine/evaluation.bench.ts

@@ -0,0 +1,156 @@
+/**
+ * Expression Engine Benchmarks
+ *
+ * Answers: "What's the baseline performance of expression evaluation?"
+ *
+ * These benchmarks establish the hot-path performance for comparing
+ * alternative implementations (WASM sandbox, quickjs, etc.)
+ *
+ * Run: pnpm --filter=@n8n/performance bench
+ */
+import { bench, describe } from 'vitest';
+import { Workflow } from 'n8n-workflow';
+import type { INodeTypes, INodeType, INodeTypeDescription } from 'n8n-workflow';
+
+// Minimal node types implementation for workflow instantiation
+class TestNodeTypes implements INodeTypes {
+	getByName(nodeType: string): INodeType {
+		return {
+			description: {
+				name: nodeType,
+				displayName: 'Test',
+				group: ['transform'],
+				version: 1,
+				defaults: { name: 'Test' },
+				inputs: ['main'],
+				outputs: ['main'],
+				properties: [],
+				description: '',
+			} as INodeTypeDescription,
+			execute: async () => [[{ json: {} }]],
+		};
+	}
+
+	getByNameAndVersion(): INodeType {
+		return this.getByName('test.set');
+	}
+
+	getKnownTypes(): Record<string, Record<string, unknown>> {
+		return {};
+	}
+}
+
+// Shared workflow instance (simulates production reuse)
+const nodeTypes = new TestNodeTypes();
+const workflow = new Workflow({
+	id: '1',
+	nodes: [
+		{
+			name: 'node',
+			typeVersion: 1,
+			type: 'test.set',
+			id: 'uuid-1234',
+			position: [0, 0],
+			parameters: {},
+		},
+	],
+	connections: {},
+	active: false,
+	nodeTypes,
+});
+
+// Factory for fresh workflow instances
+const createWorkflow = () =>
+	new Workflow({
+		id: '1',
+		nodes: [
+			{
+				name: 'node',
+				typeVersion: 1,
+				type: 'test.set',
+				id: 'uuid-1234',
+				position: [0, 0],
+				parameters: {},
+			},
+		],
+		connections: {},
+		active: false,
+		nodeTypes: new TestNodeTypes(),
+	});
+
+// Test data
+const smallData = [
+	{
+		json: {
+			name: 'test-user',
+			email: 'test@example.com',
+			items: Array(100)
+				.fill(null)
+				.map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })),
+		},
+	},
+];
+
+const largeData = [
+	{
+		json: {
+			name: 'test-user',
+			items: Array(10000)
+				.fill(null)
+				.map((_, i) => ({ id: i, value: i * 10, active: i % 2 === 0 })),
+		},
+	},
+];
+
+const evaluate = (expr: string, data: typeof smallData | typeof largeData) =>
+	workflow.expression.getParameterValue(expr, null, 0, 0, 'node', data, 'manual', {});
+
+describe('Expression: Hot Path', () => {
+	// Baseline: simplest possible expression
+	bench('simple property access', () => {
+		evaluate('={{ $json.name }}', smallData);
+	});
+
+	// Typical: array transform
+	bench('array map (100 items)', () => {
+		evaluate('={{ $json.items.map(i => i.value) }}', smallData);
+	});
+
+	// Complex: chained operations
+	bench('method chain', () => {
+		evaluate('={{ $json.items.filter(i => i.active).map(i => i.id) }}', smallData);
+	});
+});
+
+describe('Expression: Cold Start', () => {
+	// Answers: "What's the WASM sandbox init cost comparison?"
+	bench('first evaluation (fresh workflow)', () => {
+		const fresh = createWorkflow();
+		fresh.expression.getParameterValue(
+			'={{ $json.name }}',
+			null,
+			0,
+			0,
+			'node',
+			smallData,
+			'manual',
+			{},
+		);
+	});
+
+	// Answers: "Should we pool expression workers?"
+	bench('reused workflow', () => {
+		evaluate('={{ $json.name }}', smallData);
+	});
+});
+
+describe('Expression: Data Transfer', () => {
+	// Answers: "What's the overhead of data moving between wasm and node?"
+	bench('small context (100 items)', () => {
+		evaluate('={{ $json.items.map(i => i.id) }}', smallData);
+	});
+
+	bench('large context (10k items)', () => {
+		evaluate('={{ $json.items.map(i => i.id) }}', largeData);
+	});
+});

packages/testing/performance/package.json

@@ -0,0 +1,16 @@
+{
+	"name": "@n8n/performance",
+	"version": "1.0.0",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"bench": "vitest bench --run",
+		"bench:baseline": "node scripts/save-baseline.mjs",
+		"bench:compare": "vitest bench --run --outputJson ./profiles/benchmark-results.json && node scripts/check-regression.mjs"
+	},
+	"devDependencies": {
+		"@codspeed/vitest-plugin": "^4.0.0",
+		"vitest": "^3.2.0",
+		"n8n-workflow": "workspace:*"
+	}
+}

packages/testing/performance/profiles/.gitignore

@@ -0,0 +1,3 @@
+# Generated by benchmarks - CI manages baselines
+baseline.json
+benchmark-results.json

packages/testing/performance/scripts/check-regression.mjs

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/**
+ * Benchmark Regression Checker
+ *
+ * Compares current benchmark results against baseline and fails if any
+ * benchmark regresses beyond the threshold.
+ *
+ * Exit codes:
+ *   0 = All benchmarks within threshold
+ *   1 = Regression detected
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROFILES_DIR = resolve(__dirname, '../profiles');
+const THRESHOLD = 0.10; // 10%
+
+const BASELINE_PATH = resolve(PROFILES_DIR, 'baseline.json');
+const CURRENT_PATH = resolve(PROFILES_DIR, 'benchmark-results.json');
+
+if (!existsSync(BASELINE_PATH)) {
+	console.error('❌ No baseline found. Run: pnpm bench:baseline');
+	process.exit(1);
+}
+
+if (!existsSync(CURRENT_PATH)) {
+	console.error('❌ No current results found. Run bench:compare to generate them.');
+	process.exit(1);
+}
+
+const baseline = JSON.parse(readFileSync(BASELINE_PATH, 'utf-8'));
+const current = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8'));
+
+// Build lookup map from baseline
+const baselineMap = new Map();
+for (const file of baseline.files) {
+	for (const group of file.groups) {
+		for (const bench of group.benchmarks) {
+			baselineMap.set(`${group.fullName}::${bench.name}`, bench);
+		}
+	}
+}
+
+// Compare results
+const results = [];
+let hasRegression = false;
+
+for (const file of current.files) {
+	for (const group of file.groups) {
+		for (const bench of group.benchmarks) {
+			const key = `${group.fullName}::${bench.name}`;
+			const base = baselineMap.get(key);
+
+			if (!base) {
+				results.push({ name: bench.name, status: 'new', current: bench.hz, baseline: null, ratio: null });
+				continue;
+			}
+
+			const ratio = bench.hz / base.hz;
+			const isRegression = ratio < (1 - THRESHOLD);
+			const isImprovement = ratio > (1 + THRESHOLD);
+
+			if (isRegression) hasRegression = true;
+
+			results.push({
+				name: bench.name,
+				status: isRegression ? 'regression' : isImprovement ? 'improved' : 'ok',
+				current: bench.hz,
+				baseline: base.hz,
+				ratio,
+			});
+		}
+	}
+}
+
+// Print results
+console.log(`\nBenchmark Comparison (±${(THRESHOLD * 100).toFixed(0)}% threshold)\n`);
+console.log(''.padEnd(70, '─'));
+
+for (const r of results) {
+	const icon = r.status === 'regression' ? '❌' : r.status === 'improved' ? '✅' : r.status === 'new' ? '🆕' : '  ';
+	const changeStr = r.ratio !== null ? `${((r.ratio - 1) * 100).toFixed(1)}%` : 'new';
+	const currentStr = r.current.toFixed(0).padStart(8);
+	const baselineStr = r.baseline !== null ? r.baseline.toFixed(0).padStart(8) : '     N/A';
+
+	console.log(`${icon} ${r.name.padEnd(35)} ${currentStr} hz (was ${baselineStr}) ${changeStr.padStart(7)}`);
+}
+
+console.log(''.padEnd(70, '─'));
+
+const regressions = results.filter(r => r.status === 'regression');
+const improvements = results.filter(r => r.status === 'improved');
+
+if (hasRegression) {
+	console.log(`\n❌ FAILED: ${regressions.length} regression(s) exceeded ${(THRESHOLD * 100).toFixed(0)}% threshold\n`);
+	process.exit(1);
+} else {
+	console.log(`\n✅ PASSED: All benchmarks within threshold`);
+	if (improvements.length > 0) {
+		console.log(`   ${improvements.length} improved - consider updating baseline with: pnpm bench:baseline`);
+	}
+	console.log('');
+	process.exit(0);
+}

packages/testing/performance/scripts/save-baseline.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+/**
+ * Save Baseline
+ *
+ * Runs benchmarks and saves results as the new baseline for regression detection.
+ * Sanitizes absolute paths so baseline can be committed.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { execSync } from 'child_process';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROFILES_DIR = resolve(__dirname, '../profiles');
+const PACKAGE_DIR = resolve(__dirname, '..');
+
+console.log('Running benchmarks...\n');
+
+try {
+	execSync('pnpm vitest bench --run --outputJson ./profiles/benchmark-results.json', {
+		cwd: PACKAGE_DIR,
+		stdio: 'inherit',
+	});
+} catch {
+	console.error('\n❌ Benchmark run failed');
+	process.exit(1);
+}
+
+const resultsPath = resolve(PROFILES_DIR, 'benchmark-results.json');
+const baselinePath = resolve(PROFILES_DIR, 'baseline.json');
+
+if (!existsSync(resultsPath)) {
+	console.error('\n❌ No benchmark results found');
+	process.exit(1);
+}
+
+// Load and sanitize paths
+const results = JSON.parse(readFileSync(resultsPath, 'utf-8'));
+
+for (const file of results.files) {
+	// Convert absolute path to relative
+	if (file.filepath) {
+		file.filepath = file.filepath.replace(/^.*\/benchmarks\//, 'benchmarks/');
+	}
+}
+
+writeFileSync(baselinePath, JSON.stringify(results, null, '\t'));
+console.log('\n✅ Saved baseline.json (paths sanitized)');

packages/testing/performance/vitest.config.ts

@@ -0,0 +1,19 @@
+import codspeedPlugin from '@codspeed/vitest-plugin';
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+	plugins: [process.env.CODSPEED ? codspeedPlugin() : null].filter(Boolean),
+	test: {
+		benchmark: {
+			include: ['benchmarks/**/*.bench.ts'],
+			// Run each benchmark longer for more stable results
+			// Default is 500ms - we use 1000ms for ~2x more samples
+			time: 1000,
+			// Warmup: ensure JIT compilation is complete before measuring
+			// Default is 5 iterations - we use 100 for more thorough warmup
+			warmupIterations: 100,
+			// Default warmup time is 100ms - we use 500ms for stability
+			warmupTime: 500,
+		},
+	},
+});