first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled

This commit is contained in:
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
@@ -0,0 +1,106 @@
/**
* AST Viewer - Display the AST structure of a TypeScript file
*
* Usage:
* npx tsx scripts/ast-viewer.ts <file-path>
* npx tsx scripts/ast-viewer.ts pages/BasePage.ts
* npx tsx scripts/ast-viewer.ts --depth 3 pages/BasePage.ts
*/
import { Project, SyntaxKind, type Node } from 'ts-morph';
import * as fs from 'fs';
import * as path from 'path';
function getNodeDescription(node: Node): string {
const kind = node.getKindName();
const text = node.getText();
const preview = text.length > 50 ? text.slice(0, 50) + '...' : text;
const cleanPreview = preview.replace(/\n/g, '\\n');
switch (node.getKind()) {
case SyntaxKind.ClassDeclaration:
case SyntaxKind.FunctionDeclaration:
case SyntaxKind.MethodDeclaration:
case SyntaxKind.PropertyDeclaration:
case SyntaxKind.VariableDeclaration:
case SyntaxKind.Parameter:
case SyntaxKind.InterfaceDeclaration:
case SyntaxKind.TypeAliasDeclaration:
case SyntaxKind.EnumDeclaration: {
const name = (node as { getName?: () => string | undefined }).getName?.();
return name ? `${kind}: ${name}` : kind;
}
case SyntaxKind.Identifier:
case SyntaxKind.StringLiteral:
case SyntaxKind.NumericLiteral:
return `${kind}: "${cleanPreview}"`;
case SyntaxKind.ImportDeclaration:
case SyntaxKind.ExportDeclaration:
return `${kind}: ${cleanPreview}`;
default:
return kind;
}
}
function printAST(node: Node, indent = 0, maxDepth = 10): void {
if (indent > maxDepth) {
console.log(' '.repeat(indent) + '...');
return;
}
const description = getNodeDescription(node);
const line = node.getStartLineNumber();
console.log(' '.repeat(indent) + `${description} (line ${line})`);
for (const child of node.getChildren()) {
printAST(child, indent + 1, maxDepth);
}
}
function main() {
const args = process.argv.slice(2);
let maxDepth = 10;
let filePath: string | undefined;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--depth' && args[i + 1]) {
maxDepth = parseInt(args[i + 1], 10);
i++;
} else if (!args[i].startsWith('-')) {
filePath = args[i];
}
}
if (!filePath) {
console.error('Usage: npx tsx scripts/ast-viewer.ts [--depth N] <file-path>');
process.exit(1);
}
const resolvedPath = path.isAbsolute(filePath)
? filePath
: path.resolve(path.join(__dirname, '..'), filePath);
if (!fs.existsSync(resolvedPath)) {
console.error(`File not found: ${resolvedPath}`);
process.exit(1);
}
const project = new Project({
tsConfigFilePath: path.join(__dirname, '../tsconfig.json'),
skipAddingFilesFromTsConfig: true,
});
project.addSourceFileAtPath(resolvedPath);
const sourceFile = project.getSourceFile(resolvedPath);
if (!sourceFile) {
console.error(`Could not parse file: ${resolvedPath}`);
process.exit(1);
}
console.log(`AST for: ${path.relative(process.cwd(), resolvedPath)}\n`);
printAST(sourceFile, 0, maxDepth);
}
main();
@@ -0,0 +1,78 @@
#!/usr/bin/env node
import { execSync } from 'child_process';
import { writeFileSync, unlinkSync, existsSync, mkdirSync, copyFileSync, rmSync } from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const repoRoot = path.resolve(__dirname, '../../../..');
function execCommand(command, options = {}) {
return execSync(command, { cwd: repoRoot, stdio: 'inherit', ...options });
}
function cleanup(files = []) {
files.forEach(file => {
try {
if (file.endsWith('/')) {
rmSync(file, { recursive: true, force: true });
} else {
unlinkSync(file);
}
} catch {}
});
}
function buildTestImage(targetImage) {
const tempBuildDir = path.join(repoRoot, 'temp-build-e2e');
const dockerfilePath = path.join(repoRoot, 'Dockerfile.test');
const tempTag = `${targetImage}-temp-${Date.now()}`;
try {
console.log(`🎯 Preparing test image from ${targetImage}`);
// Build CLI with e2e controller
execCommand('pnpm turbo build --filter=n8n', {
env: { ...process.env, INCLUDE_TEST_CONTROLLER: 'true' }
});
// Pull base image
execCommand(`docker pull ${targetImage}`);
// Verify controller exists
const controllerPath = path.join(repoRoot, 'packages/cli/dist/controllers/e2e.controller.js');
if (!existsSync(controllerPath)) {
throw new Error(`E2E controller not found. Build may have failed.`);
}
// Setup temp files
cleanup([tempBuildDir, dockerfilePath]);
mkdirSync(tempBuildDir, { recursive: true });
copyFileSync(controllerPath, path.join(tempBuildDir, 'e2e.controller.js'));
// Create Dockerfile
writeFileSync(dockerfilePath, `FROM ${targetImage}
COPY temp-build-e2e/e2e.controller.js /usr/local/lib/node_modules/n8n/dist/controllers/e2e.controller.js`);
// Build and replace image
execCommand(`docker build -f Dockerfile.test -t ${tempTag} .`);
execCommand(`docker rmi ${targetImage}`);
execCommand(`docker tag ${tempTag} ${targetImage}`);
execCommand(`docker rmi ${tempTag}`);
console.log(`✅ Modified ${targetImage} with e2e controller`);
} finally {
cleanup([tempBuildDir, dockerfilePath]);
}
}
// Main execution
const targetImage = process.argv[2] || 'n8nio/n8n:nightly';
try {
buildTestImage(targetImage);
} catch (error) {
console.error('❌ Failed:', error.message);
process.exit(1);
}
@@ -0,0 +1,466 @@
#!/usr/bin/env node
/**
* Coverage gap analysis across frontend and nodes.
*
* Queries Codecov flag data to surface files with the most uncovered lines —
* the highest ROI targets for new tests.
*
* Usage:
* node scripts/coverage-analysis.mjs [options]
*
* --domain=frontend|nodes|all Which domain to analyse (default: all)
* --top=<n> Files per list (default: 20)
* --min-lines=<n> Minimum file size to include (default: 50)
* --gap-threshold=<n> Max combined coverage % to be a gap (default: 60)
* --json Structured JSON — AI-ready, includes type + recommendation fields
* --md GitHub-Flavored Markdown — for $GITHUB_STEP_SUMMARY
* --out-json=<path> Write JSON to a file alongside --md (avoids double API call in CI)
* --deep=<path> Per-line unit vs E2E breakdown for a single file
*
* Environment:
* CODECOV_API_TOKEN Required in CI. Rate limits are strict without it.
*
* JSON schema (for AI agents):
* Top-level: { generatedAt, config, results[] }
* Each result: { domain, totalGapFiles, gaps[], hotPath[] }
* Each gap entry:
* domain — "frontend" | "nodes"
* file — full repo-relative path (use with Read tool)
* type — composable | store | api | utility | component | node | other
* recommendation — "unit-test" | "e2e-test" | "nock-test"
* lines — total tracked lines
* unitPct — unit test coverage %
* e2ePct — E2E coverage % (0 for nodes domain — no E2E flag exists)
* combinedPct — optimistic union coverage % (assumes zero overlap; actual may be lower)
* uncovered — uncovered line count (primary sort key — highest = most impactful)
* codecovUrl — direct link to per-line file report on Codecov
*
* Note on combinedPct: computed as min(unit_hits + e2e_hits, total_lines) / total_lines.
* This assumes the two layers cover non-overlapping lines, so it is an upper bound.
* Real combined coverage may be lower if both layers cover the same lines.
*/
const BASE_URL = 'https://codecov.io/api/v2/github/n8n-io/repos/n8n';
const CODECOV_FILE_BASE = 'https://app.codecov.io/github/n8n-io/n8n/blob/master';
const DOMAINS = {
frontend: { label: 'Frontend (editor-ui)', unitFlag: 'frontend', e2eFlag: 'frontend-e2e' },
nodes: { label: 'Nodes (nodes-base)', unitFlag: 'nodes-unit', e2eFlag: null },
};
// ── CLI ────────────────────────────────────────────────────────────────────────
const args = Object.fromEntries(
process.argv
.slice(2)
.filter((a) => a.startsWith('--'))
.map((a) => {
const [k, v] = a.slice(2).split('=');
return [k, v ?? true];
}),
);
const DOMAIN = args.domain ?? 'all';
const TOP = parseInt(args.top ?? '20', 10);
const MIN_LINES = parseInt(args['min-lines'] ?? '50', 10);
const GAP_THRESHOLD = parseFloat(args['gap-threshold'] ?? '60');
const OUTPUT_JSON = args.json === true;
const OUTPUT_MD = args.md === true;
const OUT_JSON_FILE = args['out-json'];
const DEEP_FILE = args.deep;
const activeDomains =
DOMAIN === 'all' ? Object.entries(DOMAINS) : [[DOMAIN, DOMAINS[DOMAIN]]];
if (DOMAIN !== 'all' && !DOMAINS[DOMAIN]) {
console.error(`Unknown domain "${DOMAIN}". Use: frontend, nodes, or all.`);
process.exit(1);
}
if (DEEP_FILE && (DEEP_FILE.startsWith('http') || DEEP_FILE.includes('..'))) {
console.error('--deep must be a relative repository file path (e.g. packages/editor-ui/src/App.vue)');
process.exit(1);
}
const TOKEN = process.env.CODECOV_API_TOKEN;
const headers = TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {};
// ── File type inference ────────────────────────────────────────────────────────
/**
* Infers the file type and appropriate test recommendation from the file path.
* Used by AI agents to prioritise the easiest/most impactful gaps to address.
*
* Order matters — more specific patterns first.
*/
export function inferType(filePath) {
const name = filePath.split('/').pop() ?? '';
const lpath = filePath.toLowerCase();
// Node files (.node.ts) — test with nock HTTP mocking
if (/\.node\.(ts|js)$/.test(name)) {
return { type: 'node', recommendation: 'nock-test' };
}
// Composables — must be in /composables/ dir AND match use* prefix to avoid
// false positives like UserManager.ts, useRootStore (store), etc.
if (lpath.includes('/composables/') && /^use[A-Z]/.test(name)) {
return { type: 'composable', recommendation: 'unit-test' };
}
// Stores
if (/\.store\.(ts|js)$/.test(name) || lpath.includes('/stores/')) {
return { type: 'store', recommendation: 'unit-test' };
}
// API modules
if (/\.api\.(ts|js)$/.test(name)) {
return { type: 'api', recommendation: 'unit-test' };
}
// Utilities/helpers
if (/\.(utils|helpers|util|helper)\.(ts|js)$/.test(name)) {
return { type: 'utility', recommendation: 'unit-test' };
}
// Vue components — prefer E2E for UI behaviour, unit for pure logic
if (/\.vue$/.test(name)) {
return { type: 'component', recommendation: 'e2e-test' };
}
return { type: 'other', recommendation: 'unit-test' };
}
// ── Fetch ──────────────────────────────────────────────────────────────────────
/** Fetch with exponential backoff retry — handles 429 rate limits and transient 5xx. */
async function fetchJson(url, retries = 3) {
for (let attempt = 0; attempt <= retries; attempt++) {
const res = await fetch(url, { headers });
if (res.ok) return res.json();
// Retry on rate limit or transient server error
if ((res.status === 429 || res.status >= 500) && attempt < retries) {
const delay = Math.pow(2, attempt) * 1000 + Math.random() * 500;
process.stderr.write(` [${res.status}] retrying in ${Math.round(delay / 1000)}s...\n`);
await new Promise((r) => setTimeout(r, delay));
continue;
}
const body = await res.text();
throw new Error(`HTTP ${res.status} ${url}\n${body.slice(0, 200)}`);
}
}
/**
* Returns Map<fileName, { lines, hits, pct }> for a given Codecov flag.
* Warns if the flag returns no files — likely a propagation delay after upload.
*/
async function fetchFlagMap(flag) {
const map = new Map();
let url = `${BASE_URL}/report/?flag=${flag}&page_size=100`;
let page = 1;
process.stderr.write(` ${flag}:`);
while (url) {
process.stderr.write(` p${page}`);
const data = await fetchJson(url);
for (const f of data.files ?? []) {
const t = f.totals ?? {};
const lines = Number(t.lines) || 0;
const hits = Number(t.hits) || 0;
if (lines === 0) continue; // skip files with no tracked lines
map.set(f.name, { lines, hits, pct: (hits / lines) * 100 });
}
url = data.next ?? null;
page++;
}
process.stderr.write(`${map.size} files\n`);
if (map.size === 0) {
process.stderr.write(
` WARNING: flag "${flag}" returned 0 files. Codecov may still be indexing the upload.\n` +
` Results for this flag will be empty. Re-run in a few minutes if this looks wrong.\n`,
);
}
return map;
}
async function fetchLineData(filePath, flag) {
const data = await fetchJson(
`${BASE_URL}/file_report/${encodeURIComponent(filePath)}?flag=${flag}`,
);
return data.line_coverage ?? [];
}
// ── Analysis ───────────────────────────────────────────────────────────────────
export function findGaps(unitMap, e2eMap, domainKey) {
const all = new Set([...unitMap.keys(), ...(e2eMap?.keys() ?? [])]);
const gaps = [];
for (const name of all) {
const unit = unitMap.get(name) ?? { lines: 0, hits: 0, pct: 0 };
const e2e = e2eMap?.get(name) ?? { lines: 0, hits: 0, pct: 0 };
const lines = Math.max(unit.lines, e2e.lines);
// Both zero means no line data at all — skip entirely
if (lines === 0 || lines < MIN_LINES) continue;
const combinedHits = Math.min(unit.hits + e2e.hits, lines);
const combinedPct = (combinedHits / lines) * 100;
const uncovered = lines - combinedHits;
if (combinedPct >= GAP_THRESHOLD) continue;
const { type, recommendation } = inferType(name);
gaps.push({
domain: domainKey,
file: name,
type,
recommendation,
lines,
unitPct: unit.pct,
e2ePct: e2e.pct,
combinedPct,
uncovered,
codecovUrl: `${CODECOV_FILE_BASE}/${name}`,
});
}
const sorted = gaps.sort((a, b) => b.uncovered - a.uncovered);
return { gaps: sorted.slice(0, TOP), totalGapFiles: sorted.length };
}
export function findHotPath(unitMap, e2eMap) {
if (!e2eMap) return [];
const all = new Set([...unitMap.keys(), ...e2eMap.keys()]);
const result = [];
for (const name of all) {
const unit = unitMap.get(name) ?? { lines: 0, pct: 0 };
const e2e = e2eMap.get(name) ?? { lines: 0, pct: 0 };
const lines = Math.max(unit.lines, e2e.lines);
if (lines === 0 || lines < MIN_LINES) continue;
if (e2e.pct >= 80 && unit.pct < 15) {
result.push({ file: name, lines, unitPct: unit.pct, e2ePct: e2e.pct });
}
}
return result.sort((a, b) => b.e2ePct - a.e2ePct).slice(0, 10);
}
// ── Terminal output ────────────────────────────────────────────────────────────
const W = 112;
function bar(pct, width = 14) {
const filled = Math.round((pct / 100) * width);
return '[' + '█'.repeat(filled) + '░'.repeat(width - filled) + ']';
}
function trunc(str, max) {
return str.length > max ? '…' + str.slice(-(max - 1)) : str;
}
function printGapTable(label, gaps, hasE2E, totalGapFiles) {
console.log(`\n${'═'.repeat(W)}`);
console.log(` GAPS — ${label} (${totalGapFiles} files below ${GAP_THRESHOLD}% — top ${gaps.length} by uncovered lines)`);
console.log('═'.repeat(W));
const e2eHdr = hasE2E ? `${'E2E%'.padStart(7)} ` : '';
console.log(
` ${'#'.padEnd(4)} ${'File'.padEnd(48)} ${'Type'.padEnd(12)} ${'Lines'.padStart(6)} ${'Unit%'.padStart(7)} ${e2eHdr}${'Combined%'.padStart(11)} ${'Uncovered'.padStart(10)} Bar`,
);
console.log('─'.repeat(W));
for (const [i, f] of gaps.entries()) {
const e2eCol = hasE2E ? `${f.e2ePct.toFixed(1).padStart(6)}% ` : '';
console.log(
` ${String(i + 1).padEnd(4)} ${trunc(f.file, 48).padEnd(48)} ${f.type.padEnd(12)} ${String(f.lines).padStart(6)} ` +
`${f.unitPct.toFixed(1).padStart(6)}% ${e2eCol}${f.combinedPct.toFixed(1).padStart(10)}% ${String(f.uncovered).padStart(10)} ${bar(f.combinedPct)}`,
);
}
if (!gaps.length) console.log(' (none)');
}
function printHotPathTable(hotPath) {
if (!hotPath.length) return;
console.log(`\n${'─'.repeat(W)}`);
console.log(` E2E HOT-PATH (E2E ≥ 80%, Unit < 15%) — covered by navigation, not deliberate unit tests`);
console.log('─'.repeat(W));
for (const [i, f] of hotPath.entries()) {
console.log(
` ${String(i + 1).padEnd(3)} ${trunc(f.file, 75).padEnd(75)} unit=${f.unitPct.toFixed(1).padStart(5)}% e2e=${f.e2ePct.toFixed(1).padStart(5)}%`,
);
}
}
// ── Markdown output (GitHub Job Summary) ──────────────────────────────────────
export function mdEscape(str) {
// Escape characters that break GFM table cells or link syntax
return str.replace(/\|/g, '\\|').replace(/"/g, '&quot;');
}
function renderMarkdown(results, date) {
const lines = [];
lines.push('## Coverage Gap Report');
lines.push(
`_${date} · gap threshold <${GAP_THRESHOLD}% · min ${MIN_LINES} lines · [Full report on Codecov](https://app.codecov.io/github/n8n-io/n8n)_`,
);
lines.push('');
lines.push('> Files ranked by uncovered lines — highest ROI targets for new tests.');
lines.push('');
for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) {
lines.push(`### ${domain.label}`);
lines.push(
`_${totalGapFiles} files below ${GAP_THRESHOLD}% combined coverage — top ${gaps.length} by uncovered lines_`,
);
lines.push('');
const e2eHdr = hasE2E ? ' E2E% |' : '';
const e2eSep = hasE2E ? ' ----: |' : '';
lines.push(`| # | File | Type | Lines | Unit% |${e2eHdr} Combined% | Uncovered |`);
lines.push(`| --: | ---- | ---- | ----: | ----: |${e2eSep} --------: | --------: |`);
for (const [i, f] of gaps.entries()) {
const fileName = mdEscape(f.file.split('/').pop());
const dir = mdEscape(f.file.split('/').slice(0, -1).join('/'));
const fileCell = `[\`${fileName}\`](${f.codecovUrl} "${dir}")`;
const e2eCol = hasE2E ? ` ${f.e2ePct.toFixed(1)}% |` : '';
lines.push(
`| ${i + 1} | ${fileCell} | \`${f.type}\` | ${f.lines} | ${f.unitPct.toFixed(1)}% |${e2eCol} **${f.combinedPct.toFixed(1)}%** | **${f.uncovered}** |`,
);
}
if (hotPath.length) {
lines.push('');
lines.push('<details>');
lines.push(
`<summary>E2E hot-path — ${hotPath.length} files covered by test navigation (not deliberate unit tests)</summary>`,
);
lines.push('');
lines.push('| File | Unit% | E2E% |');
lines.push('| ---- | ----: | ---: |');
for (const f of hotPath) {
lines.push(
`| \`${mdEscape(f.file.split('/').pop())}\` | ${f.unitPct.toFixed(1)}% | ${f.e2ePct.toFixed(1)}% |`,
);
}
lines.push('</details>');
}
lines.push('');
}
return lines.join('\n');
}
// ── Deep dive ──────────────────────────────────────────────────────────────────
async function deepDive(filePath) {
const domain = DOMAINS.frontend;
process.stderr.write('Fetching line data...\n');
const [unitLines, e2eLines] = await Promise.all([
fetchLineData(filePath, domain.unitFlag),
domain.e2eFlag ? fetchLineData(filePath, domain.e2eFlag) : Promise.resolve([]),
]);
const unitMap = new Map(unitLines.map(([ln, h]) => [ln, h]));
const e2eMap = new Map(e2eLines.map(([ln, h]) => [ln, h]));
const all = new Set([...unitMap.keys(), ...e2eMap.keys()]);
let both = 0, unitOnly = 0, e2eOnly = 0, neither = 0;
const uncoveredLineNums = [];
for (const ln of all) {
const u = (unitMap.get(ln) ?? 0) > 0;
const e = (e2eMap.get(ln) ?? 0) > 0;
if (u && e) both++;
else if (u) unitOnly++;
else if (e) e2eOnly++;
else { neither++; uncoveredLineNums.push(ln); }
}
console.log(`\nPer-line breakdown: ${filePath}\n`);
console.log(`Lines tracked : ${all.size}`);
console.log(`Unit only : ${unitOnly}`);
console.log(`E2E only : ${e2eOnly}`);
console.log(`Both : ${both}`);
console.log(`Neither : ${neither} ← true gaps`);
if (uncoveredLineNums.length) {
console.log(`\nUncovered lines: ${uncoveredLineNums.sort((a, b) => a - b).join(', ')}`);
}
}
// ── Main ───────────────────────────────────────────────────────────────────────
async function main() {
if (DEEP_FILE) {
await deepDive(DEEP_FILE);
return;
}
const results = [];
for (const [domainKey, domain] of activeDomains) {
process.stderr.write(`\nFetching ${domain.label}...\n`);
const [unitMap, e2eMap] = await Promise.all([
fetchFlagMap(domain.unitFlag),
domain.e2eFlag ? fetchFlagMap(domain.e2eFlag) : Promise.resolve(null),
]);
const { gaps, totalGapFiles } = findGaps(unitMap, e2eMap, domainKey);
const hotPath = domain.e2eFlag ? findHotPath(unitMap, e2eMap) : [];
results.push({ domainKey, domain, gaps, hotPath, hasE2E: !!domain.e2eFlag, totalGapFiles });
}
const date = new Date().toISOString().split('T')[0];
const jsonPayload = {
generatedAt: date,
config: { gapThreshold: GAP_THRESHOLD, minLines: MIN_LINES, top: TOP },
results: results.map(({ domainKey, gaps, hotPath, totalGapFiles }) => ({
domain: domainKey,
totalGapFiles,
gaps,
hotPath,
})),
};
// Write JSON to file alongside any other output mode (avoids a second API call in CI)
if (OUT_JSON_FILE) {
const { writeFileSync } = await import('node:fs');
writeFileSync(OUT_JSON_FILE, JSON.stringify(jsonPayload, null, 2));
process.stderr.write(` JSON written to ${OUT_JSON_FILE}\n`);
}
if (OUTPUT_JSON) {
console.log(JSON.stringify(jsonPayload, null, 2));
return;
}
if (OUTPUT_MD) {
console.log(renderMarkdown(results, date));
return;
}
console.log(`\nCoverage Gap Analysis | ${date} | gap <${GAP_THRESHOLD}% | min ${MIN_LINES} lines`);
for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) {
printGapTable(domain.label, gaps, hasE2E, totalGapFiles);
printHotPathTable(hotPath);
}
console.log(`\n${'─'.repeat(W)}`);
console.log('Tip: --deep=<path> per-line unit vs E2E breakdown');
console.log(' --json AI-ready structured output');
console.log(' --domain=nodes nodes-only gaps\n');
}
// Allow pure-function imports in tests without triggering network calls
if (process.argv[1] === new URL(import.meta.url).pathname) {
main().catch((err) => {
console.error('Error:', err.message);
process.exit(1);
});
}
@@ -0,0 +1,194 @@
# Playwright Coverage Workflow
This document explains how to generate HTML coverage reports from your Playwright tests to see exactly which parts of your code are being hit.
## Overview
The coverage workflow consists of three main steps:
1. **Build with Coverage**: Build the editor-ui with istanbul instrumentation
2. **Run Tests with Coverage**: Execute Playwright tests that collect coverage data
3. **Generate HTML Report**: Convert NYC coverage data into readable HTML reports
## Step-by-Step Instructions
### 1. Build Editor-UI with Coverage Instrumentation
First, build the editor-ui with coverage enabled:
```bash
# From the project root
pnpm --filter n8n-editor-ui build:coverage
```
This will:
- Enable istanbul instrumentation in the Vite build
- Create instrumented JavaScript files in the `dist/` directory
- Add coverage collection hooks to your code
### 2. Run Playwright Tests with Coverage Collection
Run your Playwright tests with coverage collection enabled:
```bash
# From the playwright package directory
cd packages/testing/playwright
# Run E2E tests
pnpm test:e2e
```
This will:
- Execute your Playwright tests
- Collect coverage data as tests interact with the instrumented code
- Store coverage data in NYC format
### 3. Generate HTML Coverage Report
Generate a detailed HTML report from the collected coverage data:
```bash
# From the playwright package directory
pnpm coverage:report
```
This will:
- Find and merge all coverage files
- Generate an HTML report showing exactly which lines are covered
- Create a detailed coverage analysis
### 4. View the Coverage Report
Open the generated HTML report in your browser:
```bash
# The report will be available at:
open coverage/index.html
# Or navigate to:
file:///path/to/packages/testing/playwright/coverage/index.html
```
## Understanding the Coverage Report
The HTML report will show you:
- **Overall Coverage**: Percentage of statements, branches, functions, and lines covered
- **File-by-File Breakdown**: Coverage for each source file
- **Line-by-Line Details**: Exactly which lines are covered (green) vs uncovered (red)
- **Branch Coverage**: Which conditional branches are tested
- **Function Coverage**: Which functions are called during tests
## Troubleshooting
### No Coverage Data Found
If you see "No coverage files found":
1. Build with coverage: `BUILD_WITH_COVERAGE=true pnpm build` or `pnpm build:docker:coverage`
2. Run tests with coverage enabled: `BUILD_WITH_COVERAGE=true pnpm test:container:sqlite`
3. Check that coverage files exist in `.nyc_output/{projectName}/` directories
- For local mode: `.nyc_output/e2e/`
- For container mode: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc.
### Low Coverage Percentage
If you see low coverage (like 15%):
1. **Check the HTML report** to see which specific files/lines are uncovered
2. **Look for untested code paths** in your components
3. **Add more test scenarios** to cover missing branches
4. **Focus on critical user flows** that should have high coverage
### Coverage Data Not Merging
If multiple coverage files aren't merging:
1. Check that all coverage files are in valid NYC format
2. Ensure the files contain actual coverage data (not empty)
3. Try running the merge command manually: `npx nyc merge coverage/*.json .nyc_output/out.json`
## Advanced Usage
### Custom Coverage Configuration
You can modify `nyc.config.js` to:
- Change coverage thresholds
- Exclude specific files or patterns
- Adjust report formats
- Set custom watermarks
### CI/CD Integration
For automated coverage reporting:
```yaml
# Example GitHub Actions step
- name: Build Docker Image with Coverage
run: pnpm build:docker:coverage
- name: Run Container Coverage Tests
run: pnpm --filter n8n-playwright test:container:sqlite
env:
BUILD_WITH_COVERAGE: 'true'
- name: Generate Coverage Report
run: pnpm --filter n8n-playwright coverage:report
```
### Coverage Thresholds
Set minimum coverage requirements in `nyc.config.js`:
```javascript
checkCoverage: true,
statements: 80,
branches: 80,
functions: 80,
lines: 80
```
## File Structure
After running the coverage workflow, you'll have:
```
packages/testing/playwright/
├── coverage/ # HTML coverage reports
│ ├── index.html # Main coverage report
│ ├── base.css # Report styling
│ └── ... # Individual file reports
├── .nyc_output/ # Raw coverage data (per project)
│ ├── e2e/ # Local mode coverage
│ ├── sqlite:e2e/ # Container mode coverage
│ ├── postgres:e2e/ # Other container modes
│ └── out.json # Merged coverage data
├── nyc.config.ts # NYC configuration
└── scripts/
└── generate-coverage-report.js # Report generation script
```
## Best Practices
1. **Regular Coverage Checks**: Run coverage reports regularly to catch coverage regressions
2. **Focus on Critical Paths**: Prioritize coverage for user-facing features and business logic
3. **Review Uncovered Code**: Use the HTML report to identify and test uncovered code paths
4. **Set Realistic Thresholds**: Don't aim for 100% coverage - focus on meaningful coverage
5. **Clean Up**: Use `pnpm coverage:clean` to remove old coverage data when needed
## Common Issues
### "Cannot find module" errors
- Ensure the editor-ui is built with coverage before running tests
- Check that the build output includes instrumented files
### Empty coverage reports
- Verify that tests are actually running and interacting with the UI
- Check that the coverage instrumentation is working correctly
- Ensure tests are not running in headless mode without proper setup
### Performance impact
- Coverage instrumentation adds overhead - use only when needed
- Consider running coverage tests separately from regular test runs
- Use coverage data to guide test improvements, not as a strict requirement
@@ -0,0 +1,136 @@
#!/usr/bin/env node
// @ts-check
/**
* n8n CI Adapter for Test Distribution
*
* Thin wrapper that calls `janitor orchestrate` for generic shard distribution,
* then maps capabilities to n8n-specific Docker images for the CI matrix.
*
* Usage:
* node distribute-tests.mjs --matrix <shards> --orchestrate # GitHub Actions matrix with images
* node distribute-tests.mjs --matrix <shards> # Simple matrix (no distribution)
* node distribute-tests.mjs <shards> <index> # Specs for a single shard
*/
import { execFileSync } from 'child_process';
import path from 'path';
import { fileURLToPath } from 'url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PLAYWRIGHT_DIR = path.resolve(__dirname, '..');
const REPO_ROOT = path.resolve(__dirname, '..', '..', '..', '..');
const JANITOR_CLI = path.resolve(__dirname, '..', '..', 'janitor', 'dist', 'cli.js');
const PLAYWRIGHT_PREFIX = path.relative(REPO_ROOT, PLAYWRIGHT_DIR) + path.sep;
const CONTAINER_STARTUP_TIME = 22_500; // 22.5s average per fixture
const CAPABILITY_IMAGES = {
email: ['mailpit'],
kafka: ['kafka'],
observability: ['victoriaLogs', 'victoriaMetrics', 'vector', 'jaeger', 'n8nTracer'],
oidc: ['keycloak'],
proxy: ['mockserver'],
'source-control': ['gitea'],
};
const BASE_IMAGES = ['postgres', 'redis', 'caddy', 'n8n', 'taskRunner'];
function getRequiredImages(capabilities) {
const images = new Set(BASE_IMAGES);
for (const cap of capabilities) {
const capImages = CAPABILITY_IMAGES[cap];
if (capImages) {
for (const img of capImages) images.add(img);
}
}
return [...images].sort();
}
function getOrchestration(numShards, options = {}) {
const cliArgs = ['orchestrate', `--shards=${numShards}`];
if (options.impact) cliArgs.push('--impact');
if (options.files) {
// Normalize repo-root-relative paths to playwright-root-relative
// git diff gives 'packages/testing/playwright/foo.ts', janitor expects 'foo.ts'
const normalized = options.files
.split(',')
.map((f) => (f.startsWith(PLAYWRIGHT_PREFIX) ? f.slice(PLAYWRIGHT_PREFIX.length) : f))
.join(',');
cliArgs.push(`--files=${normalized}`);
}
const output = execFileSync('node', [JANITOR_CLI, ...cliArgs], {
cwd: PLAYWRIGHT_DIR,
encoding: 'utf-8',
stdio: ['pipe', 'pipe', 'inherit'],
});
return JSON.parse(output);
}
const args = process.argv.slice(2);
const matrixMode = args.includes('--matrix');
const orchestrateMode = args.includes('--orchestrate');
const impactMode = args.includes('--impact');
const filesArg = args.find((a) => a.startsWith('--files='))?.slice('--files='.length) || undefined;
const shards = parseInt(args.find((a) => !a.startsWith('-')) ?? '');
if (!shards || shards < 1) {
console.error('Usage: node distribute-tests.mjs --matrix <shards> [--orchestrate] [--impact]');
console.error(' node distribute-tests.mjs <shards> <index>');
process.exit(1);
}
if (matrixMode) {
if (!orchestrateMode) {
const matrix = Array.from({ length: shards }, (_, i) => ({
shard: i + 1,
specs: '',
images: '',
}));
console.log(JSON.stringify(matrix));
} else {
const result = getOrchestration(shards, { impact: impactMode, files: filesArg });
if (result.shards.length === 0) {
console.error('\n⏭️ No specs to run — all filtered out by discovery/impact. Skipping.\n');
console.log(JSON.stringify([{ shard: 1, specs: '', images: '', skip: true }]));
} else {
console.error('\n📊 Shard Distribution:');
let maxShardTime = 0;
for (const shard of result.shards) {
const overhead = shard.fixtureCount * CONTAINER_STARTUP_TIME;
const totalTime = shard.testTime + overhead;
maxShardTime = Math.max(maxShardTime, totalTime);
const testMins = (shard.testTime / 60_000).toFixed(1);
const totalMins = (totalTime / 60_000).toFixed(1);
const caps =
shard.capabilities.length > 0 ? ` [${shard.capabilities.join(', ')}]` : '';
console.error(
` Shard ${shard.shard}: ${shard.specs.length} specs, ${testMins} min test + ${(overhead / 1000).toFixed(0)}s startup = ${totalMins} min${caps}`,
);
}
const totalTestMins = (result.totalTestTime / 60_000).toFixed(1);
console.error(`\n Total test time: ${totalTestMins} min`);
console.error(
` Expected wall-clock: ~${(maxShardTime / 60_000).toFixed(1)} min (longest shard)\n`,
);
const matrix = result.shards.map((shard) => ({
shard: shard.shard,
specs: shard.specs.join(' '),
images: getRequiredImages(shard.capabilities).join(' '),
}));
console.log(JSON.stringify(matrix));
}
}
} else {
const index = parseInt(args[1]);
if (isNaN(index) || index < 0 || index >= shards) {
console.error(`Index must be between 0 and ${shards - 1}`);
process.exit(1);
}
const result = getOrchestration(shards, { impact: impactMode, files: filesArg });
const shard = result.shards[index];
if (shard) {
console.log(shard.specs.join('\n'));
}
}
@@ -0,0 +1,143 @@
#!/usr/bin/env node
/**
* Fetches test duration metrics from Currents API for test orchestration.
*
* Usage:
* CURRENTS_API_KEY=<key> node packages/testing/playwright/scripts/fetch-currents-metrics.mjs --project=<id>
*
* Output: .github/test-metrics/playwright.json
*/
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { execSync } from 'child_process';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT_DIR = path.resolve(__dirname, '../../../..');
const PLAYWRIGHT_DIR = path.join(ROOT_DIR, 'packages', 'testing', 'playwright');
const OUTPUT_PATH = path.join(ROOT_DIR, '.github', 'test-metrics', 'playwright.json');
const CURRENTS_API = 'https://api.currents.dev/v1';
const DEFAULT_DURATION = 60000; // 1 minute default for new specs (accounts for container startup)
const PROJECT_ID = process.argv.find((a) => a.startsWith('--project='))?.split('=')[1];
if (!PROJECT_ID) {
console.error('Usage: CURRENTS_API_KEY=<key> node fetch-currents-metrics.mjs --project=<id>');
process.exit(1);
}
async function fetchSpecPerformance(apiKey) {
const specs = [];
let page = 0;
let hasMore = true;
const sevenDaysAgo = new Date();
sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
while (hasMore && page < 100) {
const url = new URL(`${CURRENTS_API}/spec-files/${PROJECT_ID}`);
url.searchParams.set('limit', '50');
url.searchParams.set('page', page.toString());
url.searchParams.set('date_start', sevenDaysAgo.toISOString());
url.searchParams.set('date_end', new Date().toISOString());
const res = await fetch(url, {
headers: { Authorization: `Bearer ${apiKey}` },
});
if (!res.ok) throw new Error(`API error: ${res.status}`);
const data = await res.json();
// Filter to e2e specs only
const e2eSpecs = data.data.list.filter((s) => s.spec.startsWith('tests/e2e/'));
specs.push(...e2eSpecs);
hasMore = data.data.nextPage !== false;
page++;
process.stdout.write(`\rFetched ${specs.length} e2e specs...`);
}
console.log();
return specs;
}
function getPlaywrightSpecs() {
console.log('Getting specs from Playwright...');
try {
const output = execSync('pnpm playwright test --list --project="multi-main:e2e"', {
cwd: PLAYWRIGHT_DIR,
encoding: 'utf-8',
stdio: ['pipe', 'pipe', 'pipe'],
});
const specs = new Set([...output.matchAll(/ (tests\/e2e\/[^\s:]+\.spec\.ts)/g)].map((m) => m[1]));
console.log(`Found ${specs.size} specs in Playwright`);
return specs;
} catch (e) {
console.warn('Warning: Could not get Playwright specs, skipping validation');
return null;
}
}
async function main() {
const apiKey = process.env.CURRENTS_API_KEY;
if (!apiKey) {
console.error('CURRENTS_API_KEY required');
process.exit(1);
}
const validSpecs = getPlaywrightSpecs();
const specMetrics = await fetchSpecPerformance(apiKey);
const output = {
updatedAt: new Date().toISOString(),
source: 'currents',
projectId: PROJECT_ID,
specs: {},
};
const staleSpecs = [];
for (const item of specMetrics) {
// Skip specs not in Playwright (deleted/isolated)
if (validSpecs && !validSpecs.has(item.spec)) {
staleSpecs.push(item.spec);
continue;
}
const duration = item.metrics.avgDuration < 1000 ? DEFAULT_DURATION : Math.round(item.metrics.avgDuration);
output.specs[item.spec] = {
avgDuration: duration,
testCount: item.metrics.suiteSize,
flakyRate: Math.round(item.metrics.flakeRate * 10000) / 10000,
};
}
// Add new specs not yet in Currents
const newSpecs = [];
if (validSpecs) {
for (const spec of validSpecs) {
if (!output.specs[spec]) {
output.specs[spec] = { avgDuration: DEFAULT_DURATION, testCount: 0, flakyRate: 0 };
newSpecs.push(spec);
}
}
}
fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(output, null, 2) + '\n');
console.log(`Wrote ${Object.keys(output.specs).length} specs`);
if (staleSpecs.length) {
console.log(`\nStale specs (in Currents but not Playwright):`);
staleSpecs.forEach((s) => console.log(` - ${s}`));
}
if (newSpecs.length) {
console.log(`\nNew specs (in Playwright but not Currents, using ${DEFAULT_DURATION / 1000}s default):`);
newSpecs.forEach((s) => console.log(` - ${s}`));
}
}
main().catch((e) => {
console.error(e.message);
process.exit(1);
});
@@ -0,0 +1,137 @@
#!/usr/bin/env node
/**
* Simple script to merge coverage reports and generate HTML output
*/
const { execSync } = require('child_process');
const path = require('path');
const fs = require('fs');
const NYC_OUTPUT_DIR = path.join(__dirname, '..', '.nyc_output');
const COVERAGE_DIR = path.join(__dirname, '..', 'coverage');
const NYC_CONFIG = path.join(__dirname, '..', 'nyc.config.ts');
// Coverage directories to look for - Currents writes to .nyc_output/{projectName}/
// Project names come from playwright-projects.ts
const COVERAGE_PROJECT_PATTERNS = [
'e2e', // Local mode project
'sqlite:e2e', // Container mode projects
'postgres:e2e',
'queue:e2e',
'multi-main:e2e',
];
/**
* Find all coverage directories that exist and contain JSON files
*/
function findCoverageDirectories() {
const foundDirs = [];
for (const projectName of COVERAGE_PROJECT_PATTERNS) {
const projectDir = path.join(NYC_OUTPUT_DIR, projectName);
if (fs.existsSync(projectDir)) {
const files = fs.readdirSync(projectDir);
const jsonFiles = files.filter((f) => f.endsWith('.json'));
if (jsonFiles.length > 0) {
foundDirs.push({ dir: projectDir, projectName, fileCount: jsonFiles.length });
}
}
}
return foundDirs;
}
function main() {
console.log('🔍 Generating Coverage Report');
console.log('==============================\n');
// Find all coverage directories
const coverageDirs = findCoverageDirectories();
if (coverageDirs.length === 0) {
console.error('❌ No coverage data found in .nyc_output/');
console.log('\nSearched for coverage in these project directories:');
COVERAGE_PROJECT_PATTERNS.forEach((p) => console.log(` - .nyc_output/${p}/`));
console.log('\nTo generate coverage data:');
console.log(
'1. Build editor-ui with coverage: BUILD_WITH_COVERAGE=true pnpm --filter n8n-editor-ui build',
);
console.log(
'2. Run Playwright tests with coverage: BUILD_WITH_COVERAGE=true pnpm test:container:sqlite',
);
process.exit(1);
}
console.log('Found coverage data in:');
coverageDirs.forEach(({ projectName, fileCount }) =>
console.log(` - ${projectName}: ${fileCount} files`),
);
console.log('');
try {
// Merge coverage files from all found project directories
// nyc merge only accepts one input directory, so we need to:
// 1. Copy all JSON files to a single temp directory
// 2. Run nyc merge on that directory
console.log('Merging coverage files from all projects...');
const mergedFile = path.join(NYC_OUTPUT_DIR, 'out.json');
const tempMergeDir = path.join(NYC_OUTPUT_DIR, '_merge_temp');
// Create temp directory for merging
if (fs.existsSync(tempMergeDir)) {
fs.rmSync(tempMergeDir, { recursive: true });
}
fs.mkdirSync(tempMergeDir, { recursive: true });
// Copy all JSON files from all project directories to the temp directory
let fileIndex = 0;
for (const { dir, projectName } of coverageDirs) {
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
for (const file of files) {
const srcPath = path.join(dir, file);
// Use unique names to avoid collisions between projects
const destPath = path.join(
tempMergeDir,
`${projectName.replace(/:/g, '_')}_${fileIndex++}_${file}`,
);
fs.copyFileSync(srcPath, destPath);
}
}
console.log(`Copied ${fileIndex} coverage files to temp directory`);
// Now merge the single temp directory
execSync(`npx nyc merge "${tempMergeDir}" "${mergedFile}"`, { stdio: 'inherit' });
// Clean up temp directory
fs.rmSync(tempMergeDir, { recursive: true });
// Generate reports (HTML for viewing, LCOV for Codecov)
console.log('Generating coverage reports...');
execSync(
`npx nyc report --reporter=html --reporter=lcov --report-dir=${COVERAGE_DIR} --temp-dir=${NYC_OUTPUT_DIR} --config=${NYC_CONFIG} --exclude-after-remap=false`,
{ stdio: 'inherit' },
);
const htmlReportPath = path.join(COVERAGE_DIR, 'index.html');
const lcovPath = path.join(COVERAGE_DIR, 'lcov.info');
console.log(`\n✅ Coverage reports generated successfully!`);
if (fs.existsSync(htmlReportPath)) {
console.log(`📊 HTML Report: ${htmlReportPath}`);
}
if (fs.existsSync(lcovPath)) {
console.log(`📄 LCOV Report: ${lcovPath} (for Codecov)`);
}
} catch (error) {
console.error('❌ Failed to generate coverage report:', error.message);
process.exit(1);
}
}
if (require.main === module) {
main();
}
module.exports = { main };
@@ -0,0 +1,57 @@
#!/usr/bin/env node
/**
* Imports Victoria logs/metrics exports into local containers for analysis.
* Usage: node import-victoria-data.mjs [metrics_file] [logs_file]
* node import-victoria-data.mjs --start [metrics_file] [logs_file]
*/
import { execSync } from 'node:child_process';
import { existsSync, readFileSync } from 'node:fs';
import { setTimeout } from 'node:timers/promises';
const VM_PORT = 8428;
const VL_PORT = 9428;
const args = process.argv.slice(2);
const startContainers = args[0] === '--start';
if (startContainers) args.shift();
const metricsFile = args[0] ?? 'victoria-metrics-export.jsonl';
const logsFile = args[1] ?? 'victoria-logs-export.jsonl';
if (startContainers) {
try {
execSync(`docker run -d --name victoria-metrics-local -p ${VM_PORT}:8428 victoriametrics/victoria-metrics:v1.115.0 -storageDataPath=/victoria-metrics-data -retentionPeriod=7d`, { stdio: 'ignore' });
} catch {}
try {
execSync(`docker run -d --name victoria-logs-local -p ${VL_PORT}:9428 victoriametrics/victoria-logs:v1.21.0-victorialogs -storageDataPath=/victoria-logs-data -retentionPeriod=7d`, { stdio: 'ignore' });
} catch {}
await setTimeout(2000);
}
const hasMetrics = existsSync(metricsFile);
const hasLogs = existsSync(logsFile);
if (!hasMetrics && !hasLogs) {
console.log('No export files found');
process.exit(1);
}
if (hasMetrics) {
const res = await fetch(`http://localhost:${VM_PORT}/api/v1/import`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: readFileSync(metricsFile),
});
if (!res.ok) console.error(`Metrics import failed: ${res.status}`);
else console.log(`Metrics: http://localhost:${VM_PORT}/vmui/`);
}
if (hasLogs) {
const res = await fetch(`http://localhost:${VL_PORT}/insert/jsonline`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: readFileSync(logsFile),
});
if (!res.ok) console.error(`Logs import failed: ${res.status}`);
else console.log(`Logs: http://localhost:${VL_PORT}/select/vmui/`);
}