first commit

packages/testing/containers/helm-stack.ts

@@ -0,0 +1,473 @@
+import { K3sContainer, type StartedK3sContainer } from '@testcontainers/k3s';
+import { execSync } from 'node:child_process';
+import {
+	copyFileSync,
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	unlinkSync,
+	writeFileSync,
+} from 'node:fs';
+import { homedir, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+
+import { TEST_CONTAINER_IMAGES } from './test-containers';
+
+const DEFAULT_K3S_IMAGE = 'rancher/k3s:v1.32.2-k3s1';
+const DEFAULT_CHART_REPO = 'https://github.com/n8n-io/n8n-hosting.git';
+const DEFAULT_CHART_REF = 'main';
+const N8N_NODE_PORT = 30080;
+const HEALTH_POLL_INTERVAL_MS = 2_000;
+const CONTAINERD_READY_TIMEOUT_MS = 30_000;
+const K3S_STARTUP_TIMEOUT_MS = 120_000;
+const COMMAND_TIMEOUT_MS = 600_000;
+
+export type HelmStackMode = 'standalone' | 'queue';
+
+export interface HelmStackConfig {
+	/** n8n Docker image to deploy (default: TEST_CONTAINER_IMAGES.n8n) */
+	n8nImage?: string;
+	/** K3s image (default: rancher/k3s:v1.32.2-k3s1) */
+	k3sImage?: string;
+	/** Git ref for the n8n-hosting repo (default: main) */
+	helmChartRef?: string;
+	/** Git repo URL for the Helm chart (default: n8n-io/n8n-hosting) */
+	helmChartRepo?: string;
+	/** Total startup timeout in ms (default: 300_000) */
+	startupTimeoutMs?: number;
+	/** Deployment mode: standalone (SQLite) or queue (PostgreSQL + Redis + workers) */
+	mode?: HelmStackMode;
+	/** Additional environment variables to inject into n8n pods via Helm set-flags (merge-last-wins over defaults) */
+	env?: Record<string, string>;
+}
+
+export interface HelmStack {
+	/** Base URL to access n8n running inside K3s */
+	baseUrl: string;
+	/** Stop the K3s container and clean up */
+	stop: () => Promise<void>;
+	/** Path to kubeconfig file (use with kubectl/helm from your terminal) */
+	kubeConfigPath: string;
+}
+
+// -- Logging ------------------------------------------------------------------
+
+function log(message: string) {
+	const timestamp = new Date().toISOString().slice(11, 19);
+	console.log(`[helm-stack ${timestamp}] ${message}`);
+}
+
+// -- Host command execution ---------------------------------------------------
+
+/** Execute a shell command on the host with the given environment. Returns stdout or throws with stderr. */
+function execOnHost(cmd: string, env: NodeJS.ProcessEnv, description: string): string {
+	try {
+		return execSync(cmd, { env, stdio: 'pipe', encoding: 'utf-8', timeout: COMMAND_TIMEOUT_MS });
+	} catch (error: unknown) {
+		const stderr = (error as { stderr?: string }).stderr ?? '';
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`${description} failed:\n${stderr || message}`);
+	}
+}
+
+// -- Image preloading (must run inside K3s containerd) ------------------------
+
+async function preloadImage(container: StartedK3sContainer, imageName: string): Promise<void> {
+	// Try crictl pull first (fast for public registry images like GHCR).
+	// Falls back to docker save + ctr import for local-only images (e.g. n8nio/n8n:local).
+	log(`Pulling ${imageName} inside K3s...`);
+	const pullResult = await container.exec(['crictl', 'pull', imageName]);
+	if (pullResult.exitCode !== 0) {
+		log('Registry pull failed, importing from local Docker...');
+		const tarPath = `/tmp/n8n-helm-${Date.now()}.tar`;
+		try {
+			execSync(`docker save ${imageName} -o ${tarPath}`, { stdio: 'pipe' });
+			execSync(`docker cp ${tarPath} ${container.getId()}:/tmp/n8n-image.tar`, {
+				stdio: 'pipe',
+			});
+
+			const importResult = await container.exec([
+				'ctr',
+				'--namespace',
+				'k8s.io',
+				'images',
+				'import',
+				'/tmp/n8n-image.tar',
+			]);
+			if (importResult.exitCode !== 0) {
+				throw new Error(`ctr import failed: ${importResult.output}`);
+			}
+			await container.exec(['rm', '-f', '/tmp/n8n-image.tar']);
+		} finally {
+			try {
+				unlinkSync(tarPath);
+			} catch (cleanupError: unknown) {
+				log(
+					`Warning: failed to clean up temp file ${tarPath}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`,
+				);
+			}
+		}
+	}
+
+	const { output } = await container.exec(['crictl', 'images']);
+	log(`Available images after preload:\n${output}`);
+}
+
+// -- Chart download -----------------------------------------------------------
+
+function cloneChartToHost(repo: string, ref: string): string {
+	log(`Downloading chart from ${repo} @ ${ref}...`);
+	const dir = mkdtempSync(join(tmpdir(), 'n8n-chart-'));
+	const repoPath = repo.replace('https://github.com/', '').replace('.git', '');
+	const tarUrl = `https://github.com/${repoPath}/archive/${ref}.tar.gz`;
+
+	execSync(`curl -fsSL "${tarUrl}" | tar xz -C "${dir}" --strip-components=1`, { stdio: 'pipe' });
+	log('Chart downloaded');
+	return dir;
+}
+
+// -- Example values file selection --------------------------------------------
+
+const EXAMPLE_VALUES_FILES: Record<HelmStackMode, string> = {
+	standalone: 'standalone.yaml',
+	queue: 'minimal.yaml',
+};
+
+function getExampleValuesFile(chartDir: string, mode: HelmStackMode): string {
+	return join(chartDir, 'charts', 'n8n', 'examples', EXAMPLE_VALUES_FILES[mode]);
+}
+
+// -- Helm install flags -------------------------------------------------------
+
+function parseImageName(imageName: string): { repository: string; tag: string } {
+	// Split on last colon to handle registry ports (e.g. localhost:5000/repo:tag)
+	const lastColon = imageName.lastIndexOf(':');
+	const hasTag = lastColon > 0 && !imageName.substring(lastColon).includes('/');
+	return hasTag
+		? { repository: imageName.substring(0, lastColon), tag: imageName.substring(lastColon + 1) }
+		: { repository: imageName, tag: 'latest' };
+}
+
+function buildHelmSetFlags(
+	imageName: string,
+	mode: HelmStackMode,
+	baseUrl: string,
+	envOverrides?: Record<string, string>,
+): string[] {
+	const { repository, tag } = parseImageName(imageName);
+
+	// Collect env vars as an array, then convert to indexed --set flags.
+	// This avoids fragile manual index tracking and makes it easy to add conditional entries.
+	const extraEnvs: Array<{ name: string; value: string }> = [
+		{ name: 'N8N_DIAGNOSTICS_ENABLED', value: 'false' },
+		{ name: 'N8N_DYNAMIC_BANNERS_ENABLED', value: 'false' },
+		// WEBHOOK_URL tells n8n its externally-accessible address (for invitation links, webhooks, etc.)
+		{ name: 'WEBHOOK_URL', value: baseUrl },
+	];
+
+	// License env vars from host (same pattern as testcontainers stack)
+	if (process.env.N8N_LICENSE_TENANT_ID) {
+		extraEnvs.push({ name: 'N8N_LICENSE_TENANT_ID', value: process.env.N8N_LICENSE_TENANT_ID });
+	}
+	if (process.env.N8N_LICENSE_ACTIVATION_KEY) {
+		extraEnvs.push({
+			name: 'N8N_LICENSE_ACTIVATION_KEY',
+			value: process.env.N8N_LICENSE_ACTIVATION_KEY,
+		});
+	}
+	if (process.env.N8N_LICENSE_CERT) {
+		extraEnvs.push({ name: 'N8N_LICENSE_CERT', value: process.env.N8N_LICENSE_CERT });
+	}
+
+	// Caller-provided env overrides (merge-last-wins)
+	if (envOverrides) {
+		for (const [name, value] of Object.entries(envOverrides)) {
+			extraEnvs.push({ name, value });
+		}
+	}
+
+	// Dynamic overrides on top of the example values file (-f).
+	// Mode-specific config (database type, queue settings, etc.) comes from the example file.
+	const flags = [
+		`--set image.repository=${repository}`,
+		`--set image.tag=${tag}`,
+		// --set-string because K8s env values must be strings
+		...extraEnvs.flatMap((env, i) => [
+			`--set config.extraEnv[${i}].name=${env.name}`,
+			`--set-string config.extraEnv[${i}].value=${env.value}`,
+		]),
+	];
+
+	if (mode === 'standalone') {
+		// Override persistence size (example uses 5Gi, we need less for tests)
+		flags.push('--set persistence.size=1Gi');
+	} else {
+		// Override placeholder hosts to point at our Bitnami services
+		flags.push(
+			'--set database.host=postgresql',
+			'--set redis.host=redis-master',
+			// Example file references n8n-core-secrets, we use n8n-secrets
+			'--set secretRefs.existingSecret=n8n-secrets',
+		);
+	}
+
+	return flags;
+}
+
+// -- K8s secrets --------------------------------------------------------------
+
+function createN8nSecret(env: NodeJS.ProcessEnv): void {
+	log('Creating n8n core secrets...');
+	execOnHost(
+		'kubectl create secret generic n8n-secrets --from-literal=N8N_ENCRYPTION_KEY=test-encryption-key-for-e2e-testing --from-literal=N8N_HOST=localhost --from-literal=N8N_PORT=5678 --from-literal=N8N_PROTOCOL=http',
+		env,
+		'Create n8n core secrets',
+	);
+}
+
+// -- Queue mode infrastructure ------------------------------------------------
+
+function deployQueueInfrastructure(env: NodeJS.ProcessEnv): void {
+	log('Adding Bitnami Helm repo...');
+	execOnHost(
+		'helm repo add bitnami https://charts.bitnami.com/bitnami && helm repo update',
+		env,
+		'Add Bitnami repo',
+	);
+
+	log('Deploying PostgreSQL...');
+	execOnHost(
+		'helm install postgresql bitnami/postgresql --set auth.username=n8n --set auth.password=n8n-test-password --set auth.database=n8n --set primary.resources.requests.cpu=100m --set primary.resources.requests.memory=256Mi --set primary.resources.limits.cpu=500m --set primary.resources.limits.memory=512Mi --wait --timeout 3m',
+		env,
+		'Deploy PostgreSQL',
+	);
+	log('PostgreSQL deployed');
+
+	log('Deploying Redis...');
+	execOnHost(
+		"helm install redis bitnami/redis --set architecture=standalone --set auth.enabled=false --set-json 'master.disableCommands=[]' --set master.resources.requests.cpu=100m --set master.resources.requests.memory=128Mi --set master.resources.limits.cpu=250m --set master.resources.limits.memory=256Mi --wait --timeout 3m",
+		env,
+		'Deploy Redis',
+	);
+	log('Redis deployed');
+
+	execOnHost(
+		'kubectl create secret generic n8n-db-secret --from-literal=password=n8n-test-password',
+		env,
+		'Create DB password secret',
+	);
+}
+
+// -- Health check -------------------------------------------------------------
+
+async function pollHealthEndpoint(baseUrl: string, timeoutMs: number): Promise<void> {
+	const url = `${baseUrl}/healthz/readiness`;
+	const startTime = Date.now();
+
+	while (Date.now() - startTime < timeoutMs) {
+		try {
+			const response = await fetch(url);
+			if (response.status === 200) {
+				return;
+			}
+		} catch {
+			// Retry
+		}
+		await wait(HEALTH_POLL_INTERVAL_MS);
+	}
+
+	throw new Error(`n8n health check at ${url} did not return 200 within ${timeoutMs / 1000}s`);
+}
+
+// -- Main entry point ---------------------------------------------------------
+
+export async function createHelmStack(config: HelmStackConfig = {}): Promise<HelmStack> {
+	const {
+		n8nImage = TEST_CONTAINER_IMAGES.n8n,
+		k3sImage = DEFAULT_K3S_IMAGE,
+		helmChartRef = DEFAULT_CHART_REF,
+		helmChartRepo = DEFAULT_CHART_REPO,
+		startupTimeoutMs = 300_000,
+		mode = 'standalone',
+		env: envOverrides,
+	} = config;
+
+	const containerName = `n8n-helm-${mode}-${Date.now().toString(36)}`;
+
+	log('Starting K3s + Helm stack');
+	log(`  Mode: ${mode}`);
+	log(`  Container: ${containerName}`);
+	log(`  n8n image: ${n8nImage}`);
+	log(`  K3s image: ${k3sImage}`);
+	log(`  Chart: ${helmChartRepo} @ ${helmChartRef}`);
+
+	// Step 1: Start K3s with NodePort exposed (bypasses flaky kubectl port-forward)
+	// Ryuk is disabled so the container survives process exit — clean up via stack:helm:clean.
+	log('Starting K3s container (privileged)...');
+	const k3s = await new K3sContainer(k3sImage)
+		.withName(containerName)
+		.withLabels({ 'n8n.helm': 'true', 'n8n.helm.mode': mode })
+		.withExposedPorts(N8N_NODE_PORT)
+		.withStartupTimeout(K3S_STARTUP_TIMEOUT_MS)
+		.start();
+	const hostPort = k3s.getMappedPort(N8N_NODE_PORT);
+	const baseUrl = `http://localhost:${hostPort}`;
+	log(`K3s started (NodePort ${N8N_NODE_PORT} -> host ${hostPort})`);
+
+	// Step 2: Write kubeconfig to a stable path and merge into ~/.kube/config
+	const kubeDir = join(homedir(), '.kube');
+	mkdirSync(kubeDir, { recursive: true });
+	const contextName = `n8n-helm-${mode}`;
+	const kubeConfigPath = join(kubeDir, `${contextName}.yaml`);
+
+	// K3s names everything 'default' — rename to avoid clashing with existing contexts
+	const rawKubeconfig = k3s.getKubeConfig();
+	const namedKubeconfig = rawKubeconfig
+		.replace(/\bname: default\b/g, `name: ${contextName}`)
+		.replace(/\bcurrent-context: default\b/, `current-context: ${contextName}`)
+		.replace(/\bcluster: default\b/g, `cluster: ${contextName}`)
+		.replace(/\buser: default\b/g, `user: ${contextName}`);
+	writeFileSync(kubeConfigPath, namedKubeconfig);
+
+	// Merge into ~/.kube/config so kubectl works without KUBECONFIG env var
+	const defaultKubeconfig = join(kubeDir, 'config');
+	if (existsSync(defaultKubeconfig)) {
+		copyFileSync(defaultKubeconfig, `${defaultKubeconfig}.bak`);
+	}
+	try {
+		const merged = execSync('kubectl config view --flatten', {
+			env: {
+				...process.env,
+				KUBECONFIG: existsSync(defaultKubeconfig)
+					? `${kubeConfigPath}:${defaultKubeconfig}`
+					: kubeConfigPath,
+			},
+			stdio: 'pipe',
+			encoding: 'utf-8',
+		});
+		writeFileSync(defaultKubeconfig, merged);
+		execSync(`kubectl config use-context ${contextName}`, { stdio: 'pipe' });
+		log(`Kubeconfig merged into ~/.kube/config (context: ${contextName})`);
+	} catch {
+		log(`Warning: could not merge kubeconfig — use: export KUBECONFIG=${kubeConfigPath}`);
+	}
+
+	const env: NodeJS.ProcessEnv = { ...process.env, KUBECONFIG: kubeConfigPath };
+
+	let chartDir = '';
+
+	try {
+		// Step 3: Wait for containerd readiness
+		log('Waiting for containerd...');
+		const deadline = Date.now() + CONTAINERD_READY_TIMEOUT_MS;
+		while (Date.now() < deadline) {
+			const result = await k3s.exec(['crictl', 'images']);
+			if (result.exitCode === 0) break;
+			await wait(HEALTH_POLL_INTERVAL_MS);
+		}
+
+		// Step 4: Preload n8n image into K3s containerd
+		await preloadImage(k3s, n8nImage);
+		log('Image preloaded');
+
+		// Step 5: Download chart to host
+		chartDir = cloneChartToHost(helmChartRepo, helmChartRef);
+
+		// Step 6: Create n8n core secrets (encryption key, host config)
+		createN8nSecret(env);
+
+		// Step 7: Deploy queue infrastructure if needed
+		if (mode === 'queue') {
+			deployQueueInfrastructure(env);
+		}
+
+		// Step 8: Install n8n chart using published example values file + dynamic overrides
+		log('Installing Helm chart (this may take a few minutes)...');
+		const valuesFile = getExampleValuesFile(chartDir, mode);
+		const setFlags = buildHelmSetFlags(n8nImage, mode, baseUrl, envOverrides).join(' ');
+		log(`Using values file: ${valuesFile}`);
+		const helmOutput = execOnHost(
+			`helm install n8n "${chartDir}/charts/n8n" -f "${valuesFile}" ${setFlags} --wait --timeout 5m`,
+			env,
+			'Helm install',
+		);
+		log(`Helm install complete:\n${helmOutput.trim()}`);
+
+		// Step 9: Patch service to NodePort so traffic goes through K3s's exposed port
+		// (bypasses kubectl port-forward which silently breaks after many connections)
+		log(`Patching n8n service to NodePort ${N8N_NODE_PORT}...`);
+		execOnHost(
+			`kubectl patch svc n8n-main --type merge -p '{"spec":{"type":"NodePort","ports":[{"port":5678,"targetPort":5678,"nodePort":${N8N_NODE_PORT}}]}}'`,
+			env,
+			'Patch service to NodePort',
+		);
+
+		// Step 10: Poll health endpoint
+		log(`Polling ${baseUrl}/healthz/readiness...`);
+		await pollHealthEndpoint(baseUrl, Math.min(startupTimeoutMs, 120_000));
+		log(`n8n is ready at ${baseUrl}`);
+
+		return {
+			baseUrl,
+			kubeConfigPath,
+			stop: async () => {
+				log('Shutting down...');
+				try {
+					unlinkSync(kubeConfigPath);
+				} catch {
+					/* ignore */
+				}
+				try {
+					execSync(`rm -rf "${chartDir}"`, { stdio: 'pipe' });
+				} catch {
+					/* ignore */
+				}
+				await k3s.stop();
+				log('K3s stopped');
+			},
+		};
+	} catch (error) {
+		// Dump debug info from host kubectl
+		try {
+			const podStatus = execOnHost('kubectl get pods -o wide 2>/dev/null || true', env, 'debug');
+			console.error('\n--- Pod Status ---');
+			console.error(podStatus);
+
+			const podLogs = execOnHost(
+				'kubectl logs -l app.kubernetes.io/name=n8n --tail=50 2>/dev/null || true',
+				env,
+				'debug',
+			);
+			if (podLogs.trim()) {
+				console.error('\n--- Pod Logs ---');
+				console.error(podLogs);
+			}
+
+			const events = execOnHost(
+				'kubectl get events --sort-by=.lastTimestamp 2>/dev/null || true',
+				env,
+				'debug',
+			);
+			console.error('\n--- Events ---');
+			console.error(events);
+			console.error('----------------\n');
+		} catch {
+			// Best-effort debugging output
+		}
+
+		try {
+			unlinkSync(kubeConfigPath);
+		} catch {
+			/* ignore */
+		}
+		if (chartDir)
+			try {
+				execSync(`rm -rf "${chartDir}"`, { stdio: 'pipe' });
+			} catch {
+				/* ignore */
+			}
+		await k3s.stop();
+		throw error;
+	}
+}