first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* AST Viewer - Display the AST structure of a TypeScript file
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/ast-viewer.ts <file-path>
|
||||
* npx tsx scripts/ast-viewer.ts pages/BasePage.ts
|
||||
* npx tsx scripts/ast-viewer.ts --depth 3 pages/BasePage.ts
|
||||
*/
|
||||
|
||||
import { Project, SyntaxKind, type Node } from 'ts-morph';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
function getNodeDescription(node: Node): string {
|
||||
const kind = node.getKindName();
|
||||
const text = node.getText();
|
||||
const preview = text.length > 50 ? text.slice(0, 50) + '...' : text;
|
||||
const cleanPreview = preview.replace(/\n/g, '\\n');
|
||||
|
||||
switch (node.getKind()) {
|
||||
case SyntaxKind.ClassDeclaration:
|
||||
case SyntaxKind.FunctionDeclaration:
|
||||
case SyntaxKind.MethodDeclaration:
|
||||
case SyntaxKind.PropertyDeclaration:
|
||||
case SyntaxKind.VariableDeclaration:
|
||||
case SyntaxKind.Parameter:
|
||||
case SyntaxKind.InterfaceDeclaration:
|
||||
case SyntaxKind.TypeAliasDeclaration:
|
||||
case SyntaxKind.EnumDeclaration: {
|
||||
const name = (node as { getName?: () => string | undefined }).getName?.();
|
||||
return name ? `${kind}: ${name}` : kind;
|
||||
}
|
||||
case SyntaxKind.Identifier:
|
||||
case SyntaxKind.StringLiteral:
|
||||
case SyntaxKind.NumericLiteral:
|
||||
return `${kind}: "${cleanPreview}"`;
|
||||
case SyntaxKind.ImportDeclaration:
|
||||
case SyntaxKind.ExportDeclaration:
|
||||
return `${kind}: ${cleanPreview}`;
|
||||
default:
|
||||
return kind;
|
||||
}
|
||||
}
|
||||
|
||||
function printAST(node: Node, indent = 0, maxDepth = 10): void {
|
||||
if (indent > maxDepth) {
|
||||
console.log(' '.repeat(indent) + '...');
|
||||
return;
|
||||
}
|
||||
|
||||
const description = getNodeDescription(node);
|
||||
const line = node.getStartLineNumber();
|
||||
console.log(' '.repeat(indent) + `${description} (line ${line})`);
|
||||
|
||||
for (const child of node.getChildren()) {
|
||||
printAST(child, indent + 1, maxDepth);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
let maxDepth = 10;
|
||||
let filePath: string | undefined;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--depth' && args[i + 1]) {
|
||||
maxDepth = parseInt(args[i + 1], 10);
|
||||
i++;
|
||||
} else if (!args[i].startsWith('-')) {
|
||||
filePath = args[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (!filePath) {
|
||||
console.error('Usage: npx tsx scripts/ast-viewer.ts [--depth N] <file-path>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const resolvedPath = path.isAbsolute(filePath)
|
||||
? filePath
|
||||
: path.resolve(path.join(__dirname, '..'), filePath);
|
||||
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
console.error(`File not found: ${resolvedPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const project = new Project({
|
||||
tsConfigFilePath: path.join(__dirname, '../tsconfig.json'),
|
||||
skipAddingFilesFromTsConfig: true,
|
||||
});
|
||||
|
||||
project.addSourceFileAtPath(resolvedPath);
|
||||
const sourceFile = project.getSourceFile(resolvedPath);
|
||||
|
||||
if (!sourceFile) {
|
||||
console.error(`Could not parse file: ${resolvedPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`AST for: ${path.relative(process.cwd(), resolvedPath)}\n`);
|
||||
printAST(sourceFile, 0, maxDepth);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { writeFileSync, unlinkSync, existsSync, mkdirSync, copyFileSync, rmSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const repoRoot = path.resolve(__dirname, '../../../..');
|
||||
|
||||
function execCommand(command, options = {}) {
|
||||
return execSync(command, { cwd: repoRoot, stdio: 'inherit', ...options });
|
||||
}
|
||||
|
||||
function cleanup(files = []) {
|
||||
files.forEach(file => {
|
||||
try {
|
||||
if (file.endsWith('/')) {
|
||||
rmSync(file, { recursive: true, force: true });
|
||||
} else {
|
||||
unlinkSync(file);
|
||||
}
|
||||
} catch {}
|
||||
});
|
||||
}
|
||||
|
||||
function buildTestImage(targetImage) {
|
||||
const tempBuildDir = path.join(repoRoot, 'temp-build-e2e');
|
||||
const dockerfilePath = path.join(repoRoot, 'Dockerfile.test');
|
||||
const tempTag = `${targetImage}-temp-${Date.now()}`;
|
||||
|
||||
try {
|
||||
console.log(`🎯 Preparing test image from ${targetImage}`);
|
||||
|
||||
// Build CLI with e2e controller
|
||||
execCommand('pnpm turbo build --filter=n8n', {
|
||||
env: { ...process.env, INCLUDE_TEST_CONTROLLER: 'true' }
|
||||
});
|
||||
|
||||
// Pull base image
|
||||
execCommand(`docker pull ${targetImage}`);
|
||||
|
||||
// Verify controller exists
|
||||
const controllerPath = path.join(repoRoot, 'packages/cli/dist/controllers/e2e.controller.js');
|
||||
if (!existsSync(controllerPath)) {
|
||||
throw new Error(`E2E controller not found. Build may have failed.`);
|
||||
}
|
||||
|
||||
// Setup temp files
|
||||
cleanup([tempBuildDir, dockerfilePath]);
|
||||
mkdirSync(tempBuildDir, { recursive: true });
|
||||
copyFileSync(controllerPath, path.join(tempBuildDir, 'e2e.controller.js'));
|
||||
|
||||
// Create Dockerfile
|
||||
writeFileSync(dockerfilePath, `FROM ${targetImage}
|
||||
COPY temp-build-e2e/e2e.controller.js /usr/local/lib/node_modules/n8n/dist/controllers/e2e.controller.js`);
|
||||
|
||||
// Build and replace image
|
||||
execCommand(`docker build -f Dockerfile.test -t ${tempTag} .`);
|
||||
execCommand(`docker rmi ${targetImage}`);
|
||||
execCommand(`docker tag ${tempTag} ${targetImage}`);
|
||||
execCommand(`docker rmi ${tempTag}`);
|
||||
|
||||
console.log(`✅ Modified ${targetImage} with e2e controller`);
|
||||
} finally {
|
||||
cleanup([tempBuildDir, dockerfilePath]);
|
||||
}
|
||||
}
|
||||
|
||||
// Main execution
|
||||
const targetImage = process.argv[2] || 'n8nio/n8n:nightly';
|
||||
try {
|
||||
buildTestImage(targetImage);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Coverage gap analysis across frontend and nodes.
|
||||
*
|
||||
* Queries Codecov flag data to surface files with the most uncovered lines —
|
||||
* the highest ROI targets for new tests.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/coverage-analysis.mjs [options]
|
||||
*
|
||||
* --domain=frontend|nodes|all Which domain to analyse (default: all)
|
||||
* --top=<n> Files per list (default: 20)
|
||||
* --min-lines=<n> Minimum file size to include (default: 50)
|
||||
* --gap-threshold=<n> Max combined coverage % to be a gap (default: 60)
|
||||
* --json Structured JSON — AI-ready, includes type + recommendation fields
|
||||
* --md GitHub-Flavored Markdown — for $GITHUB_STEP_SUMMARY
|
||||
* --out-json=<path> Write JSON to a file alongside --md (avoids double API call in CI)
|
||||
* --deep=<path> Per-line unit vs E2E breakdown for a single file
|
||||
*
|
||||
* Environment:
|
||||
* CODECOV_API_TOKEN Required in CI. Rate limits are strict without it.
|
||||
*
|
||||
* JSON schema (for AI agents):
|
||||
* Top-level: { generatedAt, config, results[] }
|
||||
* Each result: { domain, totalGapFiles, gaps[], hotPath[] }
|
||||
* Each gap entry:
|
||||
* domain — "frontend" | "nodes"
|
||||
* file — full repo-relative path (use with Read tool)
|
||||
* type — composable | store | api | utility | component | node | other
|
||||
* recommendation — "unit-test" | "e2e-test" | "nock-test"
|
||||
* lines — total tracked lines
|
||||
* unitPct — unit test coverage %
|
||||
* e2ePct — E2E coverage % (0 for nodes domain — no E2E flag exists)
|
||||
* combinedPct — optimistic union coverage % (assumes zero overlap; actual may be lower)
|
||||
* uncovered — uncovered line count (primary sort key — highest = most impactful)
|
||||
* codecovUrl — direct link to per-line file report on Codecov
|
||||
*
|
||||
* Note on combinedPct: computed as min(unit_hits + e2e_hits, total_lines) / total_lines.
|
||||
* This assumes the two layers cover non-overlapping lines, so it is an upper bound.
|
||||
* Real combined coverage may be lower if both layers cover the same lines.
|
||||
*/
|
||||
|
||||
const BASE_URL = 'https://codecov.io/api/v2/github/n8n-io/repos/n8n';
|
||||
const CODECOV_FILE_BASE = 'https://app.codecov.io/github/n8n-io/n8n/blob/master';
|
||||
|
||||
const DOMAINS = {
|
||||
frontend: { label: 'Frontend (editor-ui)', unitFlag: 'frontend', e2eFlag: 'frontend-e2e' },
|
||||
nodes: { label: 'Nodes (nodes-base)', unitFlag: 'nodes-unit', e2eFlag: null },
|
||||
};
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const args = Object.fromEntries(
|
||||
process.argv
|
||||
.slice(2)
|
||||
.filter((a) => a.startsWith('--'))
|
||||
.map((a) => {
|
||||
const [k, v] = a.slice(2).split('=');
|
||||
return [k, v ?? true];
|
||||
}),
|
||||
);
|
||||
|
||||
const DOMAIN = args.domain ?? 'all';
|
||||
const TOP = parseInt(args.top ?? '20', 10);
|
||||
const MIN_LINES = parseInt(args['min-lines'] ?? '50', 10);
|
||||
const GAP_THRESHOLD = parseFloat(args['gap-threshold'] ?? '60');
|
||||
const OUTPUT_JSON = args.json === true;
|
||||
const OUTPUT_MD = args.md === true;
|
||||
const OUT_JSON_FILE = args['out-json'];
|
||||
const DEEP_FILE = args.deep;
|
||||
|
||||
const activeDomains =
|
||||
DOMAIN === 'all' ? Object.entries(DOMAINS) : [[DOMAIN, DOMAINS[DOMAIN]]];
|
||||
|
||||
if (DOMAIN !== 'all' && !DOMAINS[DOMAIN]) {
|
||||
console.error(`Unknown domain "${DOMAIN}". Use: frontend, nodes, or all.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (DEEP_FILE && (DEEP_FILE.startsWith('http') || DEEP_FILE.includes('..'))) {
|
||||
console.error('--deep must be a relative repository file path (e.g. packages/editor-ui/src/App.vue)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const TOKEN = process.env.CODECOV_API_TOKEN;
|
||||
const headers = TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {};
|
||||
|
||||
// ── File type inference ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Infers the file type and appropriate test recommendation from the file path.
|
||||
* Used by AI agents to prioritise the easiest/most impactful gaps to address.
|
||||
*
|
||||
* Order matters — more specific patterns first.
|
||||
*/
|
||||
export function inferType(filePath) {
|
||||
const name = filePath.split('/').pop() ?? '';
|
||||
const lpath = filePath.toLowerCase();
|
||||
|
||||
// Node files (.node.ts) — test with nock HTTP mocking
|
||||
if (/\.node\.(ts|js)$/.test(name)) {
|
||||
return { type: 'node', recommendation: 'nock-test' };
|
||||
}
|
||||
// Composables — must be in /composables/ dir AND match use* prefix to avoid
|
||||
// false positives like UserManager.ts, useRootStore (store), etc.
|
||||
if (lpath.includes('/composables/') && /^use[A-Z]/.test(name)) {
|
||||
return { type: 'composable', recommendation: 'unit-test' };
|
||||
}
|
||||
// Stores
|
||||
if (/\.store\.(ts|js)$/.test(name) || lpath.includes('/stores/')) {
|
||||
return { type: 'store', recommendation: 'unit-test' };
|
||||
}
|
||||
// API modules
|
||||
if (/\.api\.(ts|js)$/.test(name)) {
|
||||
return { type: 'api', recommendation: 'unit-test' };
|
||||
}
|
||||
// Utilities/helpers
|
||||
if (/\.(utils|helpers|util|helper)\.(ts|js)$/.test(name)) {
|
||||
return { type: 'utility', recommendation: 'unit-test' };
|
||||
}
|
||||
// Vue components — prefer E2E for UI behaviour, unit for pure logic
|
||||
if (/\.vue$/.test(name)) {
|
||||
return { type: 'component', recommendation: 'e2e-test' };
|
||||
}
|
||||
return { type: 'other', recommendation: 'unit-test' };
|
||||
}
|
||||
|
||||
// ── Fetch ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Fetch with exponential backoff retry — handles 429 rate limits and transient 5xx. */
|
||||
async function fetchJson(url, retries = 3) {
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
const res = await fetch(url, { headers });
|
||||
|
||||
if (res.ok) return res.json();
|
||||
|
||||
// Retry on rate limit or transient server error
|
||||
if ((res.status === 429 || res.status >= 500) && attempt < retries) {
|
||||
const delay = Math.pow(2, attempt) * 1000 + Math.random() * 500;
|
||||
process.stderr.write(` [${res.status}] retrying in ${Math.round(delay / 1000)}s...\n`);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
continue;
|
||||
}
|
||||
|
||||
const body = await res.text();
|
||||
throw new Error(`HTTP ${res.status} ${url}\n${body.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns Map<fileName, { lines, hits, pct }> for a given Codecov flag.
|
||||
* Warns if the flag returns no files — likely a propagation delay after upload.
|
||||
*/
|
||||
async function fetchFlagMap(flag) {
|
||||
const map = new Map();
|
||||
let url = `${BASE_URL}/report/?flag=${flag}&page_size=100`;
|
||||
let page = 1;
|
||||
|
||||
process.stderr.write(` ${flag}:`);
|
||||
while (url) {
|
||||
process.stderr.write(` p${page}`);
|
||||
const data = await fetchJson(url);
|
||||
for (const f of data.files ?? []) {
|
||||
const t = f.totals ?? {};
|
||||
const lines = Number(t.lines) || 0;
|
||||
const hits = Number(t.hits) || 0;
|
||||
if (lines === 0) continue; // skip files with no tracked lines
|
||||
map.set(f.name, { lines, hits, pct: (hits / lines) * 100 });
|
||||
}
|
||||
url = data.next ?? null;
|
||||
page++;
|
||||
}
|
||||
process.stderr.write(` → ${map.size} files\n`);
|
||||
|
||||
if (map.size === 0) {
|
||||
process.stderr.write(
|
||||
` WARNING: flag "${flag}" returned 0 files. Codecov may still be indexing the upload.\n` +
|
||||
` Results for this flag will be empty. Re-run in a few minutes if this looks wrong.\n`,
|
||||
);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
async function fetchLineData(filePath, flag) {
|
||||
const data = await fetchJson(
|
||||
`${BASE_URL}/file_report/${encodeURIComponent(filePath)}?flag=${flag}`,
|
||||
);
|
||||
return data.line_coverage ?? [];
|
||||
}
|
||||
|
||||
// ── Analysis ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function findGaps(unitMap, e2eMap, domainKey) {
|
||||
const all = new Set([...unitMap.keys(), ...(e2eMap?.keys() ?? [])]);
|
||||
const gaps = [];
|
||||
|
||||
for (const name of all) {
|
||||
const unit = unitMap.get(name) ?? { lines: 0, hits: 0, pct: 0 };
|
||||
const e2e = e2eMap?.get(name) ?? { lines: 0, hits: 0, pct: 0 };
|
||||
const lines = Math.max(unit.lines, e2e.lines);
|
||||
|
||||
// Both zero means no line data at all — skip entirely
|
||||
if (lines === 0 || lines < MIN_LINES) continue;
|
||||
|
||||
const combinedHits = Math.min(unit.hits + e2e.hits, lines);
|
||||
const combinedPct = (combinedHits / lines) * 100;
|
||||
const uncovered = lines - combinedHits;
|
||||
|
||||
if (combinedPct >= GAP_THRESHOLD) continue;
|
||||
|
||||
const { type, recommendation } = inferType(name);
|
||||
gaps.push({
|
||||
domain: domainKey,
|
||||
file: name,
|
||||
type,
|
||||
recommendation,
|
||||
lines,
|
||||
unitPct: unit.pct,
|
||||
e2ePct: e2e.pct,
|
||||
combinedPct,
|
||||
uncovered,
|
||||
codecovUrl: `${CODECOV_FILE_BASE}/${name}`,
|
||||
});
|
||||
}
|
||||
|
||||
const sorted = gaps.sort((a, b) => b.uncovered - a.uncovered);
|
||||
return { gaps: sorted.slice(0, TOP), totalGapFiles: sorted.length };
|
||||
}
|
||||
|
||||
export function findHotPath(unitMap, e2eMap) {
|
||||
if (!e2eMap) return [];
|
||||
const all = new Set([...unitMap.keys(), ...e2eMap.keys()]);
|
||||
const result = [];
|
||||
|
||||
for (const name of all) {
|
||||
const unit = unitMap.get(name) ?? { lines: 0, pct: 0 };
|
||||
const e2e = e2eMap.get(name) ?? { lines: 0, pct: 0 };
|
||||
const lines = Math.max(unit.lines, e2e.lines);
|
||||
if (lines === 0 || lines < MIN_LINES) continue;
|
||||
if (e2e.pct >= 80 && unit.pct < 15) {
|
||||
result.push({ file: name, lines, unitPct: unit.pct, e2ePct: e2e.pct });
|
||||
}
|
||||
}
|
||||
|
||||
return result.sort((a, b) => b.e2ePct - a.e2ePct).slice(0, 10);
|
||||
}
|
||||
|
||||
// ── Terminal output ────────────────────────────────────────────────────────────
|
||||
|
||||
const W = 112;
|
||||
|
||||
function bar(pct, width = 14) {
|
||||
const filled = Math.round((pct / 100) * width);
|
||||
return '[' + '█'.repeat(filled) + '░'.repeat(width - filled) + ']';
|
||||
}
|
||||
|
||||
function trunc(str, max) {
|
||||
return str.length > max ? '…' + str.slice(-(max - 1)) : str;
|
||||
}
|
||||
|
||||
function printGapTable(label, gaps, hasE2E, totalGapFiles) {
|
||||
console.log(`\n${'═'.repeat(W)}`);
|
||||
console.log(` GAPS — ${label} (${totalGapFiles} files below ${GAP_THRESHOLD}% — top ${gaps.length} by uncovered lines)`);
|
||||
console.log('═'.repeat(W));
|
||||
|
||||
const e2eHdr = hasE2E ? `${'E2E%'.padStart(7)} ` : '';
|
||||
console.log(
|
||||
` ${'#'.padEnd(4)} ${'File'.padEnd(48)} ${'Type'.padEnd(12)} ${'Lines'.padStart(6)} ${'Unit%'.padStart(7)} ${e2eHdr}${'Combined%'.padStart(11)} ${'Uncovered'.padStart(10)} Bar`,
|
||||
);
|
||||
console.log('─'.repeat(W));
|
||||
|
||||
for (const [i, f] of gaps.entries()) {
|
||||
const e2eCol = hasE2E ? `${f.e2ePct.toFixed(1).padStart(6)}% ` : '';
|
||||
console.log(
|
||||
` ${String(i + 1).padEnd(4)} ${trunc(f.file, 48).padEnd(48)} ${f.type.padEnd(12)} ${String(f.lines).padStart(6)} ` +
|
||||
`${f.unitPct.toFixed(1).padStart(6)}% ${e2eCol}${f.combinedPct.toFixed(1).padStart(10)}% ${String(f.uncovered).padStart(10)} ${bar(f.combinedPct)}`,
|
||||
);
|
||||
}
|
||||
if (!gaps.length) console.log(' (none)');
|
||||
}
|
||||
|
||||
function printHotPathTable(hotPath) {
|
||||
if (!hotPath.length) return;
|
||||
console.log(`\n${'─'.repeat(W)}`);
|
||||
console.log(` E2E HOT-PATH (E2E ≥ 80%, Unit < 15%) — covered by navigation, not deliberate unit tests`);
|
||||
console.log('─'.repeat(W));
|
||||
for (const [i, f] of hotPath.entries()) {
|
||||
console.log(
|
||||
` ${String(i + 1).padEnd(3)} ${trunc(f.file, 75).padEnd(75)} unit=${f.unitPct.toFixed(1).padStart(5)}% e2e=${f.e2ePct.toFixed(1).padStart(5)}%`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Markdown output (GitHub Job Summary) ──────────────────────────────────────
|
||||
|
||||
export function mdEscape(str) {
|
||||
// Escape characters that break GFM table cells or link syntax
|
||||
return str.replace(/\|/g, '\\|').replace(/"/g, '"');
|
||||
}
|
||||
|
||||
function renderMarkdown(results, date) {
|
||||
const lines = [];
|
||||
lines.push('## Coverage Gap Report');
|
||||
lines.push(
|
||||
`_${date} · gap threshold <${GAP_THRESHOLD}% · min ${MIN_LINES} lines · [Full report on Codecov](https://app.codecov.io/github/n8n-io/n8n)_`,
|
||||
);
|
||||
lines.push('');
|
||||
lines.push('> Files ranked by uncovered lines — highest ROI targets for new tests.');
|
||||
lines.push('');
|
||||
|
||||
for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) {
|
||||
lines.push(`### ${domain.label}`);
|
||||
lines.push(
|
||||
`_${totalGapFiles} files below ${GAP_THRESHOLD}% combined coverage — top ${gaps.length} by uncovered lines_`,
|
||||
);
|
||||
lines.push('');
|
||||
|
||||
const e2eHdr = hasE2E ? ' E2E% |' : '';
|
||||
const e2eSep = hasE2E ? ' ----: |' : '';
|
||||
lines.push(`| # | File | Type | Lines | Unit% |${e2eHdr} Combined% | Uncovered |`);
|
||||
lines.push(`| --: | ---- | ---- | ----: | ----: |${e2eSep} --------: | --------: |`);
|
||||
|
||||
for (const [i, f] of gaps.entries()) {
|
||||
const fileName = mdEscape(f.file.split('/').pop());
|
||||
const dir = mdEscape(f.file.split('/').slice(0, -1).join('/'));
|
||||
const fileCell = `[\`${fileName}\`](${f.codecovUrl} "${dir}")`;
|
||||
const e2eCol = hasE2E ? ` ${f.e2ePct.toFixed(1)}% |` : '';
|
||||
lines.push(
|
||||
`| ${i + 1} | ${fileCell} | \`${f.type}\` | ${f.lines} | ${f.unitPct.toFixed(1)}% |${e2eCol} **${f.combinedPct.toFixed(1)}%** | **${f.uncovered}** |`,
|
||||
);
|
||||
}
|
||||
|
||||
if (hotPath.length) {
|
||||
lines.push('');
|
||||
lines.push('<details>');
|
||||
lines.push(
|
||||
`<summary>E2E hot-path — ${hotPath.length} files covered by test navigation (not deliberate unit tests)</summary>`,
|
||||
);
|
||||
lines.push('');
|
||||
lines.push('| File | Unit% | E2E% |');
|
||||
lines.push('| ---- | ----: | ---: |');
|
||||
for (const f of hotPath) {
|
||||
lines.push(
|
||||
`| \`${mdEscape(f.file.split('/').pop())}\` | ${f.unitPct.toFixed(1)}% | ${f.e2ePct.toFixed(1)}% |`,
|
||||
);
|
||||
}
|
||||
lines.push('</details>');
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ── Deep dive ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function deepDive(filePath) {
|
||||
const domain = DOMAINS.frontend;
|
||||
process.stderr.write('Fetching line data...\n');
|
||||
|
||||
const [unitLines, e2eLines] = await Promise.all([
|
||||
fetchLineData(filePath, domain.unitFlag),
|
||||
domain.e2eFlag ? fetchLineData(filePath, domain.e2eFlag) : Promise.resolve([]),
|
||||
]);
|
||||
|
||||
const unitMap = new Map(unitLines.map(([ln, h]) => [ln, h]));
|
||||
const e2eMap = new Map(e2eLines.map(([ln, h]) => [ln, h]));
|
||||
const all = new Set([...unitMap.keys(), ...e2eMap.keys()]);
|
||||
|
||||
let both = 0, unitOnly = 0, e2eOnly = 0, neither = 0;
|
||||
const uncoveredLineNums = [];
|
||||
|
||||
for (const ln of all) {
|
||||
const u = (unitMap.get(ln) ?? 0) > 0;
|
||||
const e = (e2eMap.get(ln) ?? 0) > 0;
|
||||
if (u && e) both++;
|
||||
else if (u) unitOnly++;
|
||||
else if (e) e2eOnly++;
|
||||
else { neither++; uncoveredLineNums.push(ln); }
|
||||
}
|
||||
|
||||
console.log(`\nPer-line breakdown: ${filePath}\n`);
|
||||
console.log(`Lines tracked : ${all.size}`);
|
||||
console.log(`Unit only : ${unitOnly}`);
|
||||
console.log(`E2E only : ${e2eOnly}`);
|
||||
console.log(`Both : ${both}`);
|
||||
console.log(`Neither : ${neither} ← true gaps`);
|
||||
|
||||
if (uncoveredLineNums.length) {
|
||||
console.log(`\nUncovered lines: ${uncoveredLineNums.sort((a, b) => a - b).join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
if (DEEP_FILE) {
|
||||
await deepDive(DEEP_FILE);
|
||||
return;
|
||||
}
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const [domainKey, domain] of activeDomains) {
|
||||
process.stderr.write(`\nFetching ${domain.label}...\n`);
|
||||
const [unitMap, e2eMap] = await Promise.all([
|
||||
fetchFlagMap(domain.unitFlag),
|
||||
domain.e2eFlag ? fetchFlagMap(domain.e2eFlag) : Promise.resolve(null),
|
||||
]);
|
||||
|
||||
const { gaps, totalGapFiles } = findGaps(unitMap, e2eMap, domainKey);
|
||||
const hotPath = domain.e2eFlag ? findHotPath(unitMap, e2eMap) : [];
|
||||
results.push({ domainKey, domain, gaps, hotPath, hasE2E: !!domain.e2eFlag, totalGapFiles });
|
||||
}
|
||||
|
||||
const date = new Date().toISOString().split('T')[0];
|
||||
|
||||
const jsonPayload = {
|
||||
generatedAt: date,
|
||||
config: { gapThreshold: GAP_THRESHOLD, minLines: MIN_LINES, top: TOP },
|
||||
results: results.map(({ domainKey, gaps, hotPath, totalGapFiles }) => ({
|
||||
domain: domainKey,
|
||||
totalGapFiles,
|
||||
gaps,
|
||||
hotPath,
|
||||
})),
|
||||
};
|
||||
|
||||
// Write JSON to file alongside any other output mode (avoids a second API call in CI)
|
||||
if (OUT_JSON_FILE) {
|
||||
const { writeFileSync } = await import('node:fs');
|
||||
writeFileSync(OUT_JSON_FILE, JSON.stringify(jsonPayload, null, 2));
|
||||
process.stderr.write(` JSON written to ${OUT_JSON_FILE}\n`);
|
||||
}
|
||||
|
||||
if (OUTPUT_JSON) {
|
||||
console.log(JSON.stringify(jsonPayload, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (OUTPUT_MD) {
|
||||
console.log(renderMarkdown(results, date));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\nCoverage Gap Analysis | ${date} | gap <${GAP_THRESHOLD}% | min ${MIN_LINES} lines`);
|
||||
for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) {
|
||||
printGapTable(domain.label, gaps, hasE2E, totalGapFiles);
|
||||
printHotPathTable(hotPath);
|
||||
}
|
||||
console.log(`\n${'─'.repeat(W)}`);
|
||||
console.log('Tip: --deep=<path> per-line unit vs E2E breakdown');
|
||||
console.log(' --json AI-ready structured output');
|
||||
console.log(' --domain=nodes nodes-only gaps\n');
|
||||
}
|
||||
|
||||
// Allow pure-function imports in tests without triggering network calls
|
||||
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
||||
main().catch((err) => {
|
||||
console.error('Error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
# Playwright Coverage Workflow
|
||||
|
||||
This document explains how to generate HTML coverage reports from your Playwright tests to see exactly which parts of your code are being hit.
|
||||
|
||||
## Overview
|
||||
|
||||
The coverage workflow consists of three main steps:
|
||||
|
||||
1. **Build with Coverage**: Build the editor-ui with istanbul instrumentation
|
||||
2. **Run Tests with Coverage**: Execute Playwright tests that collect coverage data
|
||||
3. **Generate HTML Report**: Convert NYC coverage data into readable HTML reports
|
||||
|
||||
## Step-by-Step Instructions
|
||||
|
||||
### 1. Build Editor-UI with Coverage Instrumentation
|
||||
|
||||
First, build the editor-ui with coverage enabled:
|
||||
|
||||
```bash
|
||||
# From the project root
|
||||
pnpm --filter n8n-editor-ui build:coverage
|
||||
```
|
||||
|
||||
This will:
|
||||
- Enable istanbul instrumentation in the Vite build
|
||||
- Create instrumented JavaScript files in the `dist/` directory
|
||||
- Add coverage collection hooks to your code
|
||||
|
||||
### 2. Run Playwright Tests with Coverage Collection
|
||||
|
||||
Run your Playwright tests with coverage collection enabled:
|
||||
|
||||
```bash
|
||||
# From the playwright package directory
|
||||
cd packages/testing/playwright
|
||||
|
||||
# Run E2E tests
|
||||
pnpm test:e2e
|
||||
|
||||
```
|
||||
|
||||
This will:
|
||||
- Execute your Playwright tests
|
||||
- Collect coverage data as tests interact with the instrumented code
|
||||
- Store coverage data in NYC format
|
||||
|
||||
### 3. Generate HTML Coverage Report
|
||||
|
||||
Generate a detailed HTML report from the collected coverage data:
|
||||
|
||||
```bash
|
||||
# From the playwright package directory
|
||||
pnpm coverage:report
|
||||
```
|
||||
|
||||
This will:
|
||||
- Find and merge all coverage files
|
||||
- Generate an HTML report showing exactly which lines are covered
|
||||
- Create a detailed coverage analysis
|
||||
|
||||
### 4. View the Coverage Report
|
||||
|
||||
Open the generated HTML report in your browser:
|
||||
|
||||
```bash
|
||||
# The report will be available at:
|
||||
open coverage/index.html
|
||||
|
||||
# Or navigate to:
|
||||
file:///path/to/packages/testing/playwright/coverage/index.html
|
||||
```
|
||||
|
||||
## Understanding the Coverage Report
|
||||
|
||||
The HTML report will show you:
|
||||
|
||||
- **Overall Coverage**: Percentage of statements, branches, functions, and lines covered
|
||||
- **File-by-File Breakdown**: Coverage for each source file
|
||||
- **Line-by-Line Details**: Exactly which lines are covered (green) vs uncovered (red)
|
||||
- **Branch Coverage**: Which conditional branches are tested
|
||||
- **Function Coverage**: Which functions are called during tests
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### No Coverage Data Found
|
||||
|
||||
If you see "No coverage files found":
|
||||
|
||||
1. Build with coverage: `BUILD_WITH_COVERAGE=true pnpm build` or `pnpm build:docker:coverage`
|
||||
2. Run tests with coverage enabled: `BUILD_WITH_COVERAGE=true pnpm test:container:sqlite`
|
||||
3. Check that coverage files exist in `.nyc_output/{projectName}/` directories
|
||||
- For local mode: `.nyc_output/e2e/`
|
||||
- For container mode: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc.
|
||||
|
||||
### Low Coverage Percentage
|
||||
|
||||
If you see low coverage (like 15%):
|
||||
|
||||
1. **Check the HTML report** to see which specific files/lines are uncovered
|
||||
2. **Look for untested code paths** in your components
|
||||
3. **Add more test scenarios** to cover missing branches
|
||||
4. **Focus on critical user flows** that should have high coverage
|
||||
|
||||
### Coverage Data Not Merging
|
||||
|
||||
If multiple coverage files aren't merging:
|
||||
|
||||
1. Check that all coverage files are in valid NYC format
|
||||
2. Ensure the files contain actual coverage data (not empty)
|
||||
3. Try running the merge command manually: `npx nyc merge coverage/*.json .nyc_output/out.json`
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Coverage Configuration
|
||||
|
||||
You can modify `nyc.config.js` to:
|
||||
- Change coverage thresholds
|
||||
- Exclude specific files or patterns
|
||||
- Adjust report formats
|
||||
- Set custom watermarks
|
||||
|
||||
### CI/CD Integration
|
||||
|
||||
For automated coverage reporting:
|
||||
|
||||
```yaml
|
||||
# Example GitHub Actions step
|
||||
- name: Build Docker Image with Coverage
|
||||
run: pnpm build:docker:coverage
|
||||
|
||||
- name: Run Container Coverage Tests
|
||||
run: pnpm --filter n8n-playwright test:container:sqlite
|
||||
env:
|
||||
BUILD_WITH_COVERAGE: 'true'
|
||||
|
||||
- name: Generate Coverage Report
|
||||
run: pnpm --filter n8n-playwright coverage:report
|
||||
```
|
||||
|
||||
### Coverage Thresholds
|
||||
|
||||
Set minimum coverage requirements in `nyc.config.js`:
|
||||
|
||||
```javascript
|
||||
checkCoverage: true,
|
||||
statements: 80,
|
||||
branches: 80,
|
||||
functions: 80,
|
||||
lines: 80
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
After running the coverage workflow, you'll have:
|
||||
|
||||
```
|
||||
packages/testing/playwright/
|
||||
├── coverage/ # HTML coverage reports
|
||||
│ ├── index.html # Main coverage report
|
||||
│ ├── base.css # Report styling
|
||||
│ └── ... # Individual file reports
|
||||
├── .nyc_output/ # Raw coverage data (per project)
|
||||
│ ├── e2e/ # Local mode coverage
|
||||
│ ├── sqlite:e2e/ # Container mode coverage
|
||||
│ ├── postgres:e2e/ # Other container modes
|
||||
│ └── out.json # Merged coverage data
|
||||
├── nyc.config.ts # NYC configuration
|
||||
└── scripts/
|
||||
└── generate-coverage-report.js # Report generation script
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Regular Coverage Checks**: Run coverage reports regularly to catch coverage regressions
|
||||
2. **Focus on Critical Paths**: Prioritize coverage for user-facing features and business logic
|
||||
3. **Review Uncovered Code**: Use the HTML report to identify and test uncovered code paths
|
||||
4. **Set Realistic Thresholds**: Don't aim for 100% coverage - focus on meaningful coverage
|
||||
5. **Clean Up**: Use `pnpm coverage:clean` to remove old coverage data when needed
|
||||
|
||||
## Common Issues
|
||||
|
||||
### "Cannot find module" errors
|
||||
- Ensure the editor-ui is built with coverage before running tests
|
||||
- Check that the build output includes instrumented files
|
||||
|
||||
### Empty coverage reports
|
||||
- Verify that tests are actually running and interacting with the UI
|
||||
- Check that the coverage instrumentation is working correctly
|
||||
- Ensure tests are not running in headless mode without proper setup
|
||||
|
||||
### Performance impact
|
||||
- Coverage instrumentation adds overhead - use only when needed
|
||||
- Consider running coverage tests separately from regular test runs
|
||||
- Use coverage data to guide test improvements, not as a strict requirement
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env node
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* n8n CI Adapter for Test Distribution
|
||||
*
|
||||
* Thin wrapper that calls `janitor orchestrate` for generic shard distribution,
|
||||
* then maps capabilities to n8n-specific Docker images for the CI matrix.
|
||||
*
|
||||
* Usage:
|
||||
* node distribute-tests.mjs --matrix <shards> --orchestrate # GitHub Actions matrix with images
|
||||
* node distribute-tests.mjs --matrix <shards> # Simple matrix (no distribution)
|
||||
* node distribute-tests.mjs <shards> <index> # Specs for a single shard
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const PLAYWRIGHT_DIR = path.resolve(__dirname, '..');
|
||||
const REPO_ROOT = path.resolve(__dirname, '..', '..', '..', '..');
|
||||
const JANITOR_CLI = path.resolve(__dirname, '..', '..', 'janitor', 'dist', 'cli.js');
|
||||
const PLAYWRIGHT_PREFIX = path.relative(REPO_ROOT, PLAYWRIGHT_DIR) + path.sep;
|
||||
const CONTAINER_STARTUP_TIME = 22_500; // 22.5s average per fixture
|
||||
|
||||
const CAPABILITY_IMAGES = {
|
||||
email: ['mailpit'],
|
||||
kafka: ['kafka'],
|
||||
observability: ['victoriaLogs', 'victoriaMetrics', 'vector', 'jaeger', 'n8nTracer'],
|
||||
oidc: ['keycloak'],
|
||||
proxy: ['mockserver'],
|
||||
'source-control': ['gitea'],
|
||||
};
|
||||
|
||||
const BASE_IMAGES = ['postgres', 'redis', 'caddy', 'n8n', 'taskRunner'];
|
||||
|
||||
function getRequiredImages(capabilities) {
|
||||
const images = new Set(BASE_IMAGES);
|
||||
for (const cap of capabilities) {
|
||||
const capImages = CAPABILITY_IMAGES[cap];
|
||||
if (capImages) {
|
||||
for (const img of capImages) images.add(img);
|
||||
}
|
||||
}
|
||||
return [...images].sort();
|
||||
}
|
||||
|
||||
function getOrchestration(numShards, options = {}) {
|
||||
const cliArgs = ['orchestrate', `--shards=${numShards}`];
|
||||
if (options.impact) cliArgs.push('--impact');
|
||||
if (options.files) {
|
||||
// Normalize repo-root-relative paths to playwright-root-relative
|
||||
// git diff gives 'packages/testing/playwright/foo.ts', janitor expects 'foo.ts'
|
||||
const normalized = options.files
|
||||
.split(',')
|
||||
.map((f) => (f.startsWith(PLAYWRIGHT_PREFIX) ? f.slice(PLAYWRIGHT_PREFIX.length) : f))
|
||||
.join(',');
|
||||
cliArgs.push(`--files=${normalized}`);
|
||||
}
|
||||
const output = execFileSync('node', [JANITOR_CLI, ...cliArgs], {
|
||||
cwd: PLAYWRIGHT_DIR,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'inherit'],
|
||||
});
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const matrixMode = args.includes('--matrix');
|
||||
const orchestrateMode = args.includes('--orchestrate');
|
||||
const impactMode = args.includes('--impact');
|
||||
const filesArg = args.find((a) => a.startsWith('--files='))?.slice('--files='.length) || undefined;
|
||||
const shards = parseInt(args.find((a) => !a.startsWith('-')) ?? '');
|
||||
|
||||
if (!shards || shards < 1) {
|
||||
console.error('Usage: node distribute-tests.mjs --matrix <shards> [--orchestrate] [--impact]');
|
||||
console.error(' node distribute-tests.mjs <shards> <index>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (matrixMode) {
|
||||
if (!orchestrateMode) {
|
||||
const matrix = Array.from({ length: shards }, (_, i) => ({
|
||||
shard: i + 1,
|
||||
specs: '',
|
||||
images: '',
|
||||
}));
|
||||
console.log(JSON.stringify(matrix));
|
||||
} else {
|
||||
const result = getOrchestration(shards, { impact: impactMode, files: filesArg });
|
||||
|
||||
if (result.shards.length === 0) {
|
||||
console.error('\n⏭️ No specs to run — all filtered out by discovery/impact. Skipping.\n');
|
||||
console.log(JSON.stringify([{ shard: 1, specs: '', images: '', skip: true }]));
|
||||
} else {
|
||||
console.error('\n📊 Shard Distribution:');
|
||||
let maxShardTime = 0;
|
||||
for (const shard of result.shards) {
|
||||
const overhead = shard.fixtureCount * CONTAINER_STARTUP_TIME;
|
||||
const totalTime = shard.testTime + overhead;
|
||||
maxShardTime = Math.max(maxShardTime, totalTime);
|
||||
const testMins = (shard.testTime / 60_000).toFixed(1);
|
||||
const totalMins = (totalTime / 60_000).toFixed(1);
|
||||
const caps =
|
||||
shard.capabilities.length > 0 ? ` [${shard.capabilities.join(', ')}]` : '';
|
||||
console.error(
|
||||
` Shard ${shard.shard}: ${shard.specs.length} specs, ${testMins} min test + ${(overhead / 1000).toFixed(0)}s startup = ${totalMins} min${caps}`,
|
||||
);
|
||||
}
|
||||
const totalTestMins = (result.totalTestTime / 60_000).toFixed(1);
|
||||
console.error(`\n Total test time: ${totalTestMins} min`);
|
||||
console.error(
|
||||
` Expected wall-clock: ~${(maxShardTime / 60_000).toFixed(1)} min (longest shard)\n`,
|
||||
);
|
||||
|
||||
const matrix = result.shards.map((shard) => ({
|
||||
shard: shard.shard,
|
||||
specs: shard.specs.join(' '),
|
||||
images: getRequiredImages(shard.capabilities).join(' '),
|
||||
}));
|
||||
console.log(JSON.stringify(matrix));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const index = parseInt(args[1]);
|
||||
if (isNaN(index) || index < 0 || index >= shards) {
|
||||
console.error(`Index must be between 0 and ${shards - 1}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const result = getOrchestration(shards, { impact: impactMode, files: filesArg });
|
||||
const shard = result.shards[index];
|
||||
if (shard) {
|
||||
console.log(shard.specs.join('\n'));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Fetches test duration metrics from Currents API for test orchestration.
|
||||
*
|
||||
* Usage:
|
||||
* CURRENTS_API_KEY=<key> node packages/testing/playwright/scripts/fetch-currents-metrics.mjs --project=<id>
|
||||
*
|
||||
* Output: .github/test-metrics/playwright.json
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT_DIR = path.resolve(__dirname, '../../../..');
|
||||
const PLAYWRIGHT_DIR = path.join(ROOT_DIR, 'packages', 'testing', 'playwright');
|
||||
const OUTPUT_PATH = path.join(ROOT_DIR, '.github', 'test-metrics', 'playwright.json');
|
||||
|
||||
const CURRENTS_API = 'https://api.currents.dev/v1';
|
||||
const DEFAULT_DURATION = 60000; // 1 minute default for new specs (accounts for container startup)
|
||||
|
||||
const PROJECT_ID = process.argv.find((a) => a.startsWith('--project='))?.split('=')[1];
|
||||
if (!PROJECT_ID) {
|
||||
console.error('Usage: CURRENTS_API_KEY=<key> node fetch-currents-metrics.mjs --project=<id>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function fetchSpecPerformance(apiKey) {
|
||||
const specs = [];
|
||||
let page = 0;
|
||||
let hasMore = true;
|
||||
|
||||
const sevenDaysAgo = new Date();
|
||||
sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
|
||||
|
||||
while (hasMore && page < 100) {
|
||||
const url = new URL(`${CURRENTS_API}/spec-files/${PROJECT_ID}`);
|
||||
url.searchParams.set('limit', '50');
|
||||
url.searchParams.set('page', page.toString());
|
||||
url.searchParams.set('date_start', sevenDaysAgo.toISOString());
|
||||
url.searchParams.set('date_end', new Date().toISOString());
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error(`API error: ${res.status}`);
|
||||
const data = await res.json();
|
||||
|
||||
// Filter to e2e specs only
|
||||
const e2eSpecs = data.data.list.filter((s) => s.spec.startsWith('tests/e2e/'));
|
||||
specs.push(...e2eSpecs);
|
||||
hasMore = data.data.nextPage !== false;
|
||||
page++;
|
||||
|
||||
process.stdout.write(`\rFetched ${specs.length} e2e specs...`);
|
||||
}
|
||||
console.log();
|
||||
|
||||
return specs;
|
||||
}
|
||||
|
||||
function getPlaywrightSpecs() {
|
||||
console.log('Getting specs from Playwright...');
|
||||
try {
|
||||
const output = execSync('pnpm playwright test --list --project="multi-main:e2e"', {
|
||||
cwd: PLAYWRIGHT_DIR,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
const specs = new Set([...output.matchAll(/› (tests\/e2e\/[^\s:]+\.spec\.ts)/g)].map((m) => m[1]));
|
||||
console.log(`Found ${specs.size} specs in Playwright`);
|
||||
return specs;
|
||||
} catch (e) {
|
||||
console.warn('Warning: Could not get Playwright specs, skipping validation');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const apiKey = process.env.CURRENTS_API_KEY;
|
||||
if (!apiKey) {
|
||||
console.error('CURRENTS_API_KEY required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const validSpecs = getPlaywrightSpecs();
|
||||
const specMetrics = await fetchSpecPerformance(apiKey);
|
||||
|
||||
const output = {
|
||||
updatedAt: new Date().toISOString(),
|
||||
source: 'currents',
|
||||
projectId: PROJECT_ID,
|
||||
specs: {},
|
||||
};
|
||||
|
||||
const staleSpecs = [];
|
||||
for (const item of specMetrics) {
|
||||
// Skip specs not in Playwright (deleted/isolated)
|
||||
if (validSpecs && !validSpecs.has(item.spec)) {
|
||||
staleSpecs.push(item.spec);
|
||||
continue;
|
||||
}
|
||||
const duration = item.metrics.avgDuration < 1000 ? DEFAULT_DURATION : Math.round(item.metrics.avgDuration);
|
||||
output.specs[item.spec] = {
|
||||
avgDuration: duration,
|
||||
testCount: item.metrics.suiteSize,
|
||||
flakyRate: Math.round(item.metrics.flakeRate * 10000) / 10000,
|
||||
};
|
||||
}
|
||||
|
||||
// Add new specs not yet in Currents
|
||||
const newSpecs = [];
|
||||
if (validSpecs) {
|
||||
for (const spec of validSpecs) {
|
||||
if (!output.specs[spec]) {
|
||||
output.specs[spec] = { avgDuration: DEFAULT_DURATION, testCount: 0, flakyRate: 0 };
|
||||
newSpecs.push(spec);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
|
||||
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(output, null, 2) + '\n');
|
||||
|
||||
console.log(`Wrote ${Object.keys(output.specs).length} specs`);
|
||||
if (staleSpecs.length) {
|
||||
console.log(`\nStale specs (in Currents but not Playwright):`);
|
||||
staleSpecs.forEach((s) => console.log(` - ${s}`));
|
||||
}
|
||||
if (newSpecs.length) {
|
||||
console.log(`\nNew specs (in Playwright but not Currents, using ${DEFAULT_DURATION / 1000}s default):`);
|
||||
newSpecs.forEach((s) => console.log(` - ${s}`));
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Simple script to merge coverage reports and generate HTML output
|
||||
*/
|
||||
|
||||
const { execSync } = require('child_process');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const NYC_OUTPUT_DIR = path.join(__dirname, '..', '.nyc_output');
|
||||
const COVERAGE_DIR = path.join(__dirname, '..', 'coverage');
|
||||
const NYC_CONFIG = path.join(__dirname, '..', 'nyc.config.ts');
|
||||
|
||||
// Coverage directories to look for - Currents writes to .nyc_output/{projectName}/
|
||||
// Project names come from playwright-projects.ts
|
||||
const COVERAGE_PROJECT_PATTERNS = [
|
||||
'e2e', // Local mode project
|
||||
'sqlite:e2e', // Container mode projects
|
||||
'postgres:e2e',
|
||||
'queue:e2e',
|
||||
'multi-main:e2e',
|
||||
];
|
||||
|
||||
/**
|
||||
* Find all coverage directories that exist and contain JSON files
|
||||
*/
|
||||
function findCoverageDirectories() {
|
||||
const foundDirs = [];
|
||||
|
||||
for (const projectName of COVERAGE_PROJECT_PATTERNS) {
|
||||
const projectDir = path.join(NYC_OUTPUT_DIR, projectName);
|
||||
if (fs.existsSync(projectDir)) {
|
||||
const files = fs.readdirSync(projectDir);
|
||||
const jsonFiles = files.filter((f) => f.endsWith('.json'));
|
||||
if (jsonFiles.length > 0) {
|
||||
foundDirs.push({ dir: projectDir, projectName, fileCount: jsonFiles.length });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return foundDirs;
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('🔍 Generating Coverage Report');
|
||||
console.log('==============================\n');
|
||||
|
||||
// Find all coverage directories
|
||||
const coverageDirs = findCoverageDirectories();
|
||||
|
||||
if (coverageDirs.length === 0) {
|
||||
console.error('❌ No coverage data found in .nyc_output/');
|
||||
console.log('\nSearched for coverage in these project directories:');
|
||||
COVERAGE_PROJECT_PATTERNS.forEach((p) => console.log(` - .nyc_output/${p}/`));
|
||||
console.log('\nTo generate coverage data:');
|
||||
console.log(
|
||||
'1. Build editor-ui with coverage: BUILD_WITH_COVERAGE=true pnpm --filter n8n-editor-ui build',
|
||||
);
|
||||
console.log(
|
||||
'2. Run Playwright tests with coverage: BUILD_WITH_COVERAGE=true pnpm test:container:sqlite',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('Found coverage data in:');
|
||||
coverageDirs.forEach(({ projectName, fileCount }) =>
|
||||
console.log(` - ${projectName}: ${fileCount} files`),
|
||||
);
|
||||
console.log('');
|
||||
|
||||
try {
|
||||
// Merge coverage files from all found project directories
|
||||
// nyc merge only accepts one input directory, so we need to:
|
||||
// 1. Copy all JSON files to a single temp directory
|
||||
// 2. Run nyc merge on that directory
|
||||
console.log('Merging coverage files from all projects...');
|
||||
const mergedFile = path.join(NYC_OUTPUT_DIR, 'out.json');
|
||||
const tempMergeDir = path.join(NYC_OUTPUT_DIR, '_merge_temp');
|
||||
|
||||
// Create temp directory for merging
|
||||
if (fs.existsSync(tempMergeDir)) {
|
||||
fs.rmSync(tempMergeDir, { recursive: true });
|
||||
}
|
||||
fs.mkdirSync(tempMergeDir, { recursive: true });
|
||||
|
||||
// Copy all JSON files from all project directories to the temp directory
|
||||
let fileIndex = 0;
|
||||
for (const { dir, projectName } of coverageDirs) {
|
||||
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
|
||||
for (const file of files) {
|
||||
const srcPath = path.join(dir, file);
|
||||
// Use unique names to avoid collisions between projects
|
||||
const destPath = path.join(
|
||||
tempMergeDir,
|
||||
`${projectName.replace(/:/g, '_')}_${fileIndex++}_${file}`,
|
||||
);
|
||||
fs.copyFileSync(srcPath, destPath);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Copied ${fileIndex} coverage files to temp directory`);
|
||||
|
||||
// Now merge the single temp directory
|
||||
execSync(`npx nyc merge "${tempMergeDir}" "${mergedFile}"`, { stdio: 'inherit' });
|
||||
|
||||
// Clean up temp directory
|
||||
fs.rmSync(tempMergeDir, { recursive: true });
|
||||
|
||||
// Generate reports (HTML for viewing, LCOV for Codecov)
|
||||
console.log('Generating coverage reports...');
|
||||
execSync(
|
||||
`npx nyc report --reporter=html --reporter=lcov --report-dir=${COVERAGE_DIR} --temp-dir=${NYC_OUTPUT_DIR} --config=${NYC_CONFIG} --exclude-after-remap=false`,
|
||||
{ stdio: 'inherit' },
|
||||
);
|
||||
|
||||
const htmlReportPath = path.join(COVERAGE_DIR, 'index.html');
|
||||
const lcovPath = path.join(COVERAGE_DIR, 'lcov.info');
|
||||
|
||||
console.log(`\n✅ Coverage reports generated successfully!`);
|
||||
if (fs.existsSync(htmlReportPath)) {
|
||||
console.log(`📊 HTML Report: ${htmlReportPath}`);
|
||||
}
|
||||
if (fs.existsSync(lcovPath)) {
|
||||
console.log(`📄 LCOV Report: ${lcovPath} (for Codecov)`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to generate coverage report:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = { main };
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Imports Victoria logs/metrics exports into local containers for analysis.
|
||||
* Usage: node import-victoria-data.mjs [metrics_file] [logs_file]
|
||||
* node import-victoria-data.mjs --start [metrics_file] [logs_file]
|
||||
*/
|
||||
import { execSync } from 'node:child_process';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { setTimeout } from 'node:timers/promises';
|
||||
|
||||
const VM_PORT = 8428;
|
||||
const VL_PORT = 9428;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const startContainers = args[0] === '--start';
|
||||
if (startContainers) args.shift();
|
||||
|
||||
const metricsFile = args[0] ?? 'victoria-metrics-export.jsonl';
|
||||
const logsFile = args[1] ?? 'victoria-logs-export.jsonl';
|
||||
|
||||
if (startContainers) {
|
||||
try {
|
||||
execSync(`docker run -d --name victoria-metrics-local -p ${VM_PORT}:8428 victoriametrics/victoria-metrics:v1.115.0 -storageDataPath=/victoria-metrics-data -retentionPeriod=7d`, { stdio: 'ignore' });
|
||||
} catch {}
|
||||
try {
|
||||
execSync(`docker run -d --name victoria-logs-local -p ${VL_PORT}:9428 victoriametrics/victoria-logs:v1.21.0-victorialogs -storageDataPath=/victoria-logs-data -retentionPeriod=7d`, { stdio: 'ignore' });
|
||||
} catch {}
|
||||
await setTimeout(2000);
|
||||
}
|
||||
|
||||
const hasMetrics = existsSync(metricsFile);
|
||||
const hasLogs = existsSync(logsFile);
|
||||
|
||||
if (!hasMetrics && !hasLogs) {
|
||||
console.log('No export files found');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (hasMetrics) {
|
||||
const res = await fetch(`http://localhost:${VM_PORT}/api/v1/import`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: readFileSync(metricsFile),
|
||||
});
|
||||
if (!res.ok) console.error(`Metrics import failed: ${res.status}`);
|
||||
else console.log(`Metrics: http://localhost:${VM_PORT}/vmui/`);
|
||||
}
|
||||
|
||||
if (hasLogs) {
|
||||
const res = await fetch(`http://localhost:${VL_PORT}/insert/jsonline`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: readFileSync(logsFile),
|
||||
});
|
||||
if (!res.ok) console.error(`Logs import failed: ${res.status}`);
|
||||
else console.log(`Logs: http://localhost:${VL_PORT}/select/vmui/`);
|
||||
}
|
||||
Reference in New Issue
Block a user