first commit

packages/testing/playwright/tests/infrastructure/chaos/multimain.spec.ts

@@ -0,0 +1,50 @@
+import type { LogEntry } from 'n8n-containers';
+
+import { test, expect } from '../../../fixtures/base';
+
+// Enable observability to use VictoriaLogs for log queries
+test.use({ capability: 'observability' });
+
+// Helper to extract container name from log entry
+// Vector enriches logs with Docker metadata including container_name
+const getContainerName = (log: LogEntry): string | undefined => log.container_name ?? log.container;
+
+test('Leader election @mode:multi-main @chaostest @capability:observability', {
+	annotation: [
+		{ type: 'owner', description: 'Catalysts' },
+	],
+}, async ({
+	n8nContainer,
+	services,
+}) => {
+	// Find the current leader by querying VictoriaLogs
+	// Vector enriches logs with container_name from Docker metadata
+	const leaderLog = await services.observability.logs.waitForLog('Leader is now this', {
+		timeoutMs: 30000,
+		start: '-5m',
+	});
+
+	expect(leaderLog, 'Leader should be found').toBeDefined();
+
+	// Extract the leader container name from the log entry
+	const currentLeader = getContainerName(leaderLog!);
+	expect(currentLeader, 'Leader container_name should be in log entry').toBeDefined();
+
+	// Stop the leader container
+	await n8nContainer.stopContainer(currentLeader!);
+
+	// Wait for new leader election (another instance should take over)
+	// Use LogsQL to exclude logs from the stopped container
+	const newLeaderLog = await services.observability.logs.waitForLog(
+		`Leader is now this AND NOT container_name:${currentLeader}`,
+		{
+			timeoutMs: 30000,
+			start: '-5m', // Look at all recent logs, filtering by container excludes the old leader
+		},
+	);
+
+	expect(newLeaderLog).toBeDefined();
+	const newLeader = getContainerName(newLeaderLog!);
+	expect(newLeader).toBeDefined();
+	expect(newLeader).not.toBe(currentLeader);
+});

packages/testing/playwright/tests/infrastructure/chaos/postgres.spec.ts

@@ -0,0 +1,72 @@
+import { Time } from '@n8n/constants';
+
+import { test, expect } from '../../../fixtures/base';
+
+// Enable observability to use VictoriaLogs for log queries
+test.use({ capability: 'observability' });
+
+// @CATS team to look at this.  This works locally, but not in CI. Maybe IP table rules are not working in CI?
+// eslint-disable-next-line playwright/no-skipped-test
+test.skip(
+	'Database connection timeout health check bug @mode:postgres @chaostest @capability:observability',
+	{
+		annotation: [
+			{ type: 'owner', description: 'Catalysts' },
+			{ type: 'issue', description: 'CAT-1018' },
+		],
+	},
+	async ({ api, n8nContainer, services }) => {
+		test.setTimeout(300000);
+
+		// ========== SETUP: Verify Initial Health ==========
+		// Ensure n8n starts in a healthy state before we begin chaos testing
+		{
+			const isLive = await api.isHealthy('liveness');
+			const isReady = await api.isHealthy('readiness');
+			expect(isLive).toBe(true);
+			expect(isReady).toBe(true);
+		}
+
+		// ========== CHAOS INJECTION: Block Database Traffic ==========
+		// Find postgres container and install iptables to simulate network issues
+		const postgres = n8nContainer.findContainers('postgres*')[0];
+
+		// Install iptables in the postgres container (Alpine Linux)
+		const apkUpdate = await postgres.exec(['apk', 'update']);
+		const apkInstall = await postgres.exec(['apk', 'add', 'iptables']);
+		// Block all incoming TCP traffic to PostgreSQL port 5432
+		// This simulates a network partition between n8n and the database
+		const rule = ['INPUT', '-p', 'tcp', '--dport', '5432', '-j', 'DROP'];
+		const blockPostgresTraffic = await postgres.exec(['iptables', '-A', ...rule]);
+		expect(apkUpdate.exitCode).toBe(0);
+		expect(apkInstall.exitCode).toBe(0);
+		expect(blockPostgresTraffic.exitCode).toBe(0);
+
+		// ========== WAIT FOR CONNECTION ISSUES ==========
+		// Query VictoriaLogs for database timeout messages
+		await services.observability.logs.waitForLog('Database connection timed out', {
+			timeoutMs: 20 * Time.seconds.toMilliseconds,
+			start: '-1m',
+		});
+
+		// ========== VERIFY: Health Checks ==========
+		{
+			const isLive = await api.isHealthy('liveness');
+			const isReady = await api.isHealthy('readiness');
+			expect(isLive).toBe(true);
+			expect(isReady).toBe(false);
+		}
+
+		// ========== RESTORE DATABASE CONNECTION ==========
+		// Remove the iptables rule to allow traffic again
+		const allowPostgresTraffic = await postgres.exec(['iptables', '-D', ...rule]);
+		expect(allowPostgresTraffic.exitCode).toBe(0);
+
+		// ========== VERIFY: Automatic Recovery ==========
+		// Query VictoriaLogs for database recovery messages
+		await services.observability.logs.waitForLog('Database connection recovered', {
+			timeoutMs: 20 * Time.seconds.toMilliseconds,
+			start: '-1m',
+		});
+	},
+);

packages/testing/playwright/tests/infrastructure/observability/multi-main-observability.spec.ts

@@ -0,0 +1,152 @@
+/**
+ * Multi-Main Observability E2E Tests
+ *
+ * These tests verify that the observability stack (VictoriaMetrics + VictoriaLogs)
+ * works correctly with n8n's multi-main architecture in queue mode.
+ *
+ * Architecture under test:
+ * ┌─────────────────────────────────────────────────────────────────────────┐
+ * │  Queue Mode Cluster (2 mains + 1 worker)                               │
+ * │                                                                         │
+ * │  ┌──────────────┐    ┌──────────────┐    ┌──────────────┐              │
+ * │  │   Main 1     │    │   Main 2     │    │   Worker 1   │              │
+ * │  │ (leader)     │    │ (follower)   │    │              │              │
+ * │  │ :5678/metrics│    │ :5679/metrics│    │ :5680/metrics│              │
+ * │  └──────┬───────┘    └──────┬───────┘    └──────┬───────┘              │
+ * │         │                   │                   │                       │
+ * │         │    Syslog (TCP)   │                   │                       │
+ * │         └───────────────────┼───────────────────┘                       │
+ * │                             │                                           │
+ * │                             ▼                                           │
+ * │  ┌──────────────────────────────────────────────────────────────────┐  │
+ * │  │              VictoriaLogs (:9428)                                 │  │
+ * │  │  - Receives syslog on port 514                                   │  │
+ * │  │  - LogsQL queries via HTTP API                                   │  │
+ * │  └──────────────────────────────────────────────────────────────────┘  │
+ * │                                                                         │
+ * │  ┌──────────────────────────────────────────────────────────────────┐  │
+ * │  │              VictoriaMetrics (:8428)                              │  │
+ * │  │  - Scrapes /metrics from all n8n instances every 5s              │  │
+ * │  │  - PromQL queries via HTTP API                                   │  │
+ * │  └──────────────────────────────────────────────────────────────────┘  │
+ * └─────────────────────────────────────────────────────────────────────────┘
+ *
+ * Test scenarios:
+ * 1. Metrics scraping - Verify VictoriaMetrics discovers and scrapes all instances
+ * 2. Log streaming - Verify logs from cluster reach VictoriaLogs via syslog
+ *
+ * Run with:
+ *   pnpm playwright test -- --grep "Multi-main Observability"
+ */
+
+import { test, expect } from '../../../fixtures/base';
+
+// Configure test to run with multi-main queue mode and observability stack
+test.use({
+	capability: {
+		services: ['victoriaLogs', 'victoriaMetrics', 'vector'],
+		mains: 2,
+		workers: 1,
+	},
+});
+
+test.describe('Multi-main Observability @capability:observability @mode:multi-main', {
+	annotation: [
+		{ type: 'owner', description: 'Catalysts' },
+	],
+}, () => {
+	/**
+	 * Test: Metrics scraping from multi-main cluster
+	 *
+	 * Verifies that VictoriaMetrics can discover and scrape metrics from all
+	 * n8n instances in the queue mode cluster (2 mains + 1 worker).
+	 *
+	 * This tests the Prometheus-compatible /metrics endpoint exposure and
+	 * service discovery configuration in VictoriaMetrics.
+	 */
+	test('should scrape metrics from all n8n instances', async ({ services }) => {
+		const obs = services.observability;
+
+		// Expected targets: 2 mains + 1 worker = 3 instances
+		const expectedTargets = 3;
+
+		// ========== STEP 1: Wait for all targets to be healthy ==========
+		// The 'up' metric indicates which targets are being scraped (1=up, 0=down)
+		// Poll until all expected targets are healthy (containers may still be starting)
+		const healthyTarget = await obs.metrics.waitForMetric('up', {
+			timeoutMs: 90000, // Allow time for all containers to start and be scraped
+			intervalMs: 2000,
+			predicate: (results) => {
+				const healthy = results.filter((r) => r.value === 1);
+				console.log(
+					`Waiting for healthy targets: ${healthy.length}/${expectedTargets}`,
+					healthy.map((r) => r.labels.instance),
+				);
+				return healthy.length >= expectedTargets;
+			},
+		});
+
+		expect(healthyTarget, 'Expected all scrape targets to become healthy').toBeTruthy();
+
+		// ========== STEP 2: Verify final state ==========
+		const allInstances = await obs.metrics.query('up');
+		const healthyTargets = allInstances.filter((m) => m.value === 1);
+
+		console.log(
+			`Final state: ${healthyTargets.length}/${allInstances.length} healthy targets:`,
+			healthyTargets.map((m) => m.labels.instance),
+		);
+
+		expect(healthyTargets.length).toBe(expectedTargets);
+	});
+
+	/**
+	 * Test: Log streaming to VictoriaLogs in multi-main setup
+	 *
+	 * Verifies that log streaming can be configured via API and that events
+	 * are correctly delivered to VictoriaLogs via syslog protocol.
+	 *
+	 * This tests:
+	 * - Log streaming feature flag enablement
+	 * - Syslog destination configuration via REST API
+	 * - TCP syslog delivery from n8n to VictoriaLogs
+	 * - LogsQL query capability in VictoriaLogs
+	 */
+	test('should configure log streaming and receive events', async ({ api, services }) => {
+		// ========== STEP 1: Enable log streaming feature ==========
+		await api.enableFeature('logStreaming');
+
+		const obs = services.observability;
+
+		// ========== STEP 2: Configure syslog destination ==========
+		// Create a syslog destination pointing to VictoriaLogs
+		const destination = await api.createSyslogDestination({
+			host: obs.syslog.host,
+			port: obs.syslog.port,
+			protocol: obs.syslog.protocol,
+			label: 'Multi-main VictoriaLogs',
+		});
+
+		console.log('Created syslog destination:', destination.id);
+		console.log(`  Target: ${obs.syslog.host}:${obs.syslog.port} (${obs.syslog.protocol})`);
+
+		// ========== STEP 3: Send test message ==========
+		// The test message triggers n8n to send a "n8n.destination.test" event
+		const testResult = await api.testLogStreamingDestination(destination.id);
+		expect(testResult, 'Test message should be sent successfully').toBe(true);
+		console.log('Test message sent to log streaming destination');
+
+		// ========== STEP 4: Verify message arrives in VictoriaLogs ==========
+		// Query VictoriaLogs for the test message using LogsQL
+		const logEntry = await obs.logs.waitForLog('n8n.destination.test', {
+			timeoutMs: 30000,
+		});
+
+		expect(logEntry, 'Test message should appear in VictoriaLogs').toBeTruthy();
+		console.log('Test message received in VictoriaLogs:', logEntry?.message);
+
+		// ========== CLEANUP ==========
+		await api.deleteLogStreamingDestination(destination.id);
+		console.log('Cleaned up syslog destination');
+	});
+});