first commit

packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/python/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for n8n workflow comparison"""

packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/python/tests/test_graph_builder.py

@@ -0,0 +1,161 @@
+"""
+Tests for graph_builder module.
+"""
+
+from src.graph_builder import build_workflow_graph, graph_stats, _is_trigger_node
+from src.config_loader import WorkflowComparisonConfig
+
+
+def test_build_simple_workflow_graph():
+    """Test building graph from simple workflow"""
+    workflow = {
+        "name": "Test Workflow",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {"path": "/test"},
+            },
+            {
+                "id": "2",
+                "name": "Process",
+                "type": "n8n-nodes-base.code",
+                "parameters": {},
+            },
+        ],
+        "connections": {
+            "Trigger": {"main": [[{"node": "Process", "type": "main", "index": 0}]]}
+        },
+    }
+
+    graph = build_workflow_graph(workflow)
+
+    assert graph.number_of_nodes() == 2
+    assert graph.number_of_edges() == 1
+    assert "Trigger" in graph.nodes
+    assert "Process" in graph.nodes
+    assert graph.has_edge("Trigger", "Process")
+
+
+def test_build_graph_with_filtering():
+    """Test that ignored nodes are filtered out"""
+    workflow = {
+        "name": "Test Workflow",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Note",
+                "type": "n8n-nodes-base.stickyNote",
+                "parameters": {"content": "This is a note"},
+            },
+            {
+                "id": "2",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {},
+            },
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    config.ignored_node_types.add("n8n-nodes-base.stickyNote")
+
+    graph = build_workflow_graph(workflow, config)
+
+    # Sticky note should be filtered out
+    assert graph.number_of_nodes() == 1
+    assert "Trigger" in graph.nodes
+    assert "Note" not in graph.nodes
+
+
+def test_parameter_filtering():
+    """Test that ignored parameters are filtered"""
+    workflow = {
+        "name": "Test",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Node1",
+                "type": "test.node",
+                "position": [100, 200],
+                "parameters": {
+                    "important": "value",
+                    "position": [100, 200],
+                    "id": "123",
+                },
+            }
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    config.ignored_global_parameters = {"position", "id"}
+
+    graph = build_workflow_graph(workflow, config)
+
+    node_data = graph.nodes["Node1"]
+    params = node_data["parameters"]
+
+    assert "important" in params
+    assert "position" not in params
+    assert "id" not in params
+
+
+def test_is_trigger_node():
+    """Test trigger node detection"""
+    trigger_node = {"type": "n8n-nodes-base.webhook", "name": "Webhook Trigger"}
+    assert _is_trigger_node(trigger_node) is True
+
+    regular_node = {"type": "n8n-nodes-base.httpRequest", "name": "HTTP Request"}
+    assert _is_trigger_node(regular_node) is False
+
+
+def test_graph_stats():
+    """Test graph statistics calculation"""
+    workflow = {
+        "name": "Test",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.manualTrigger",
+                "parameters": {},
+            },
+            {
+                "id": "2",
+                "name": "Node1",
+                "type": "n8n-nodes-base.code",
+                "parameters": {},
+            },
+            {
+                "id": "3",
+                "name": "Node2",
+                "type": "n8n-nodes-base.code",
+                "parameters": {},
+            },
+        ],
+        "connections": {
+            "Trigger": {"main": [[{"node": "Node1", "type": "main", "index": 0}]]},
+            "Node1": {"main": [[{"node": "Node2", "type": "main", "index": 0}]]},
+        },
+    }
+
+    graph = build_workflow_graph(workflow)
+    stats = graph_stats(graph)
+
+    assert stats["node_count"] == 3
+    assert stats["edge_count"] == 2
+    assert stats["trigger_count"] == 1
+    assert stats["is_connected"] is True
+
+
+def test_empty_workflow():
+    """Test building graph from empty workflow"""
+    workflow = {"name": "Empty", "nodes": [], "connections": {}}
+
+    graph = build_workflow_graph(workflow)
+
+    assert graph.number_of_nodes() == 0
+    assert graph.number_of_edges() == 0

packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/python/tests/test_similarity.py

@@ -0,0 +1,328 @@
+"""
+Tests for similarity module.
+"""
+
+from src.graph_builder import build_workflow_graph
+from src.similarity import calculate_graph_edit_distance
+from src.config_loader import WorkflowComparisonConfig
+
+
+def test_identical_workflows():
+    """Test that identical workflows have 100% similarity"""
+    workflow = {
+        "name": "Test",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {"path": "/test"},
+            }
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow, config)
+    g2 = build_workflow_graph(workflow, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    assert result["similarity_score"] == 1.0
+    assert result["edit_cost"] == 0.0
+    assert len(result["top_edits"]) == 0
+
+
+def test_empty_workflows():
+    """Test that empty workflows are identical"""
+    workflow = {"name": "Empty", "nodes": [], "connections": {}}
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow, config)
+    g2 = build_workflow_graph(workflow, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    assert result["similarity_score"] == 1.0
+    assert result["edit_cost"] == 0.0
+
+
+def test_missing_node():
+    """Test similarity when one workflow is missing a node"""
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [{"id": "1", "name": "Node1", "type": "test.node", "parameters": {}}],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {"id": "1", "name": "Node1", "type": "test.node", "parameters": {}},
+            {"id": "2", "name": "Node2", "type": "test.node", "parameters": {}},
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should have less than 100% similarity
+    assert result["similarity_score"] < 1.0
+    # Should have edit operations
+    assert len(result["top_edits"]) > 0
+    # Should have a node insertion edit
+    assert any(edit["type"] == "node_insert" for edit in result["top_edits"])
+
+
+def test_trigger_mismatch():
+    """Test that trigger mismatches have high cost"""
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {},
+            }
+        ],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Trigger",
+                "type": "n8n-nodes-base.manualTrigger",
+                "parameters": {},
+            }
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Trigger mismatch should result in low similarity
+    assert result["similarity_score"] < 0.8
+
+    # NetworkX may choose delete+insert (cost 20) over substitution (cost 50)
+    # but the important thing is that we detect it as a critical issue
+    assert result["edit_cost"] >= (
+        config.node_deletion_cost + config.node_insertion_cost
+    )
+
+    # Should have critical priority edit in the top edits
+    assert len(result["top_edits"]) > 0
+    assert any(edit["priority"] == "critical" for edit in result["top_edits"])
+
+
+def test_parameter_differences():
+    """Test that parameter differences affect similarity"""
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Node",
+                "type": "test.node",
+                "parameters": {"value": "a"},
+            }
+        ],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Node",
+                "type": "test.node",
+                "parameters": {"value": "b"},
+            }
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should not be identical due to parameter difference
+    assert result["similarity_score"] < 1.0
+    # But should still be fairly similar (same structure)
+    assert result["similarity_score"] > 0.8
+
+
+def test_connection_difference():
+    """Test that connection differences are detected"""
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [
+            {"id": "1", "name": "Node1", "type": "test.node", "parameters": {}},
+            {"id": "2", "name": "Node2", "type": "test.node", "parameters": {}},
+        ],
+        "connections": {},  # No connections
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {"id": "1", "name": "Node1", "type": "test.node", "parameters": {}},
+            {"id": "2", "name": "Node2", "type": "test.node", "parameters": {}},
+        ],
+        "connections": {
+            "Node1": {"main": [[{"node": "Node2", "type": "main", "index": 0}]]}
+        },
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should have lower similarity due to missing connection
+    assert result["similarity_score"] < 1.0
+    # Should have edge insertion in edits
+    assert any(edit["type"] == "edge_insert" for edit in result["top_edits"])
+
+
+def test_trigger_parameter_update_priority():
+    """Test that minor trigger parameter updates are not marked as critical."""
+    from src.config_loader import load_config
+
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Webhook",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {"path": "test1"},
+            }
+        ],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Webhook",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {"path": "test2"},
+            }
+        ],
+        "connections": {},
+    }
+
+    config = load_config("preset:lenient")
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should have high similarity (just parameter change)
+    assert result["similarity_score"] > 0.9
+
+    # Should have one edit (parameter update)
+    assert len(result["top_edits"]) == 1
+    edit = result["top_edits"][0]
+
+    # Should NOT be critical priority (just a minor parameter change)
+    assert edit["priority"] != "critical"
+    assert edit["type"] == "node_substitute"
+
+
+def test_trigger_deletion_is_critical():
+    """Test that trigger deletions are marked as critical."""
+    from src.config_loader import WorkflowComparisonConfig
+
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Webhook",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {},
+            },
+            {"id": "2", "name": "Node", "type": "test.node", "parameters": {}},
+        ],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [{"id": "2", "name": "Node", "type": "test.node", "parameters": {}}],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should have lower similarity
+    assert result["similarity_score"] < 0.8
+
+    # Should have a critical priority edit (trigger deletion)
+    assert any(
+        edit["priority"] == "critical" and edit["type"] == "node_delete"
+        for edit in result["top_edits"]
+    )
+
+
+def test_trigger_insertion_is_critical():
+    """Test that trigger insertions are marked as critical."""
+    from src.config_loader import WorkflowComparisonConfig
+
+    workflow1 = {
+        "name": "Test1",
+        "nodes": [{"id": "1", "name": "Node", "type": "test.node", "parameters": {}}],
+        "connections": {},
+    }
+
+    workflow2 = {
+        "name": "Test2",
+        "nodes": [
+            {
+                "id": "1",
+                "name": "Webhook",
+                "type": "n8n-nodes-base.webhook",
+                "parameters": {},
+            },
+            {"id": "2", "name": "Node", "type": "test.node", "parameters": {}},
+        ],
+        "connections": {},
+    }
+
+    config = WorkflowComparisonConfig()
+    g1 = build_workflow_graph(workflow1, config)
+    g2 = build_workflow_graph(workflow2, config)
+
+    result = calculate_graph_edit_distance(g1, g2, config)
+
+    # Should have lower similarity
+    assert result["similarity_score"] < 0.8
+
+    # Should have a critical priority edit (trigger insertion)
+    assert any(
+        edit["priority"] == "critical" and edit["type"] == "node_insert"
+        for edit in result["top_edits"]
+    )