alighasami/n8n
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Details
Test: Benchmark Nightly / build (push) Has been cancelled
Details
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
Details
CI: Python / Checks (push) Has been cancelled
Details
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Details
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Details
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Details
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Details
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Details
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Details
Test: Benchmark Destroy Env / build (push) Has been cancelled
Details
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Details
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
alighasami
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json
+29
View File
@@ -0,0 +1,29 @@
{
"node": "n8n-nodes-base.removeDuplicates",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/"
}
],
"generic": []
},
"alias": [
"Dedupe",
"Deduplicate",
"Duplicates",
"Remove",
"Unique",
"Transform",
"Array",
"List",
"Item"
],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts
+25
View File
@@ -0,0 +1,25 @@
import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow';
import { VersionedNodeType } from 'n8n-workflow';
import { RemoveDuplicatesV1 } from './v1/RemoveDuplicatesV1.node';
import { RemoveDuplicatesV2 } from './v2/RemoveDuplicatesV2.node';
export class RemoveDuplicates extends VersionedNodeType {
constructor() {
const baseDescription: INodeTypeBaseDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
defaultVersion: 2,
description: 'Delete items with matching field values',
};
const nodeVersions: IVersionedNodeType['nodeVersions'] = {
1: new RemoveDuplicatesV1(baseDescription),
1.1: new RemoveDuplicatesV1(baseDescription),
2: new RemoveDuplicatesV2(baseDescription),
};
super(nodeVersions, baseDescription);
}
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#54B8C9" clip-path="url(#a)"><path d="M134.097 111h38.829v32.508H138.16v34.635h-32.508v-38.699c0-15.709 12.735-28.444 28.445-28.444m77.658 32.508V111h77.657v32.508zm116.486 0V111h77.658v32.508zm116.487 0V111h38.829c15.71 0 28.445 12.735 28.445 28.444v38.699h-32.508v-34.635zm34.766 73.238h32.508v38.698c0 15.71-12.735 28.445-28.445 28.445h-38.829v-32.508h34.766zM0 244.537C0 229.329 12.735 217 28.444 217h349.461c15.709 0 28.444 12.329 28.444 27.537v129.815c0 15.208-12.735 27.537-28.444 27.537H28.445C12.734 401.889 0 389.56 0 374.352z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>
Side by Side

After

Width:  |  Height:  |  Size: 715 B

packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts
+65
View File
@@ -0,0 +1,65 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import type { INode } from 'n8n-workflow';
import { validateInputData } from '../utils';
describe('Test Remove Duplicates Node', () => {
new NodeTestHarness().setupTests();
});
describe('Test Remove Duplicates Node, validateInputData util', () => {
test('Should throw error for version 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).toThrow("'country' isn't always the same type");
});
test('Should ignore null values and not throw error for version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).not.toThrow();
});
test('Should throw error for different types, version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[{ json: { id: 1 } }, { json: { id: '1' } }, { json: { id: 2 } }, { json: { id: null } }],
['id'],
false,
),
).toThrow("'id' isn't always the same type");
});
});
packages/nodes-base/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json
+316
View File
@@ -0,0 +1,316 @@
{
"name": "Remove Duplicates",
"nodes": [
{
"parameters": {},
"id": "a4da10da-991f-48ab-b873-9d633a11311f",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [760, 420]
},
{
"parameters": {
"data": [
{
"id": 1,
"name": "John Doe",
"age": 18
},
{
"id": 1,
"name": "John Doe",
"age": 18
},
{
"id": 1,
"name": "John Doe",
"age": 98
},
{
"id": 3,
"name": "Bob Johnson",
"age": 34
}
]
},
"id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [980, 420]
},
{
"parameters": {},
"id": "c336939c-062e-475e-ba7c-8601e3662e8c",
"name": "Remove Duplicates (All Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 260]
},
{
"parameters": {
"compare": "selectedFields",
"fieldsToCompare": "name",
"options": {}
},
"id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80",
"name": "Remove Duplicates (Selected Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 420]
},
{
"parameters": {
"compare": "allFieldsExcept",
"fieldsToExclude": "age",
"options": {}
},
"id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b",
"name": "Remove Duplicates (Except Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 580]
},
{
"parameters": {},
"id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba",
"name": "All Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 260]
},
{
"parameters": {},
"id": "b5c5c946-2e96-451b-b9a6-78e478504d6c",
"name": "Selected Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 420]
},
{
"parameters": {},
"id": "afb92bc5-beba-4b0a-aefb-b47cc708a125",
"name": "Except Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 580]
},
{
"parameters": {
"compare": "allFieldsExcept",
"fieldsToExclude": "age",
"options": {
"removeOtherFields": true
}
},
"id": "f92c5533-ac29-476c-aebb-4849ddd22110",
"name": "Remove Duplicates (Remove)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 760]
},
{
"parameters": {},
"id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89",
"name": "Remove",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 760]
}
],
"pinData": {
"Code": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 98
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"All Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 98
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Selected Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Except Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Remove": [
{
"json": {
"id": 1,
"name": "John Doe"
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Remove Duplicates (All Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Selected Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Except Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Remove)",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (All Fields)": {
"main": [
[
{
"node": "All Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Selected Fields)": {
"main": [
[
{
"node": "Selected Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Except Fields)": {
"main": [
[
{
"node": "Except Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Remove)": {
"main": [
[
{
"node": "Remove",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {
"executionOrder": "v1"
},
"versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3",
"id": "74gMYOHjjPArZg4q",
"meta": {
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
},
"tags": []
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts
+170
View File
@@ -0,0 +1,170 @@
import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import lt from 'lodash/lt';
import pick from 'lodash/pick';
import { NodeOperationError } from 'n8n-workflow';
import type { IExecuteFunctions, INode, INodeExecutionData } from 'n8n-workflow';
import { compareItems, flattenKeys } from '@utils/utilities';
import { prepareFieldsArray } from '../utils/utils';
export const validateInputData = (
node: INode,
items: INodeExecutionData[],
keysToCompare: string[],
disableDotNotation: boolean,
) => {
for (const key of keysToCompare) {
let type: any = undefined;
for (const [i, item] of items.entries()) {
if (key === '') {
throw new NodeOperationError(node, 'Name of field to compare is blank');
}
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
if (value === null && node.typeVersion > 1) continue;
if (value === undefined && disableDotNotation && key.includes('.')) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
description:
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
});
} else if (value === undefined) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
}
if (type !== undefined && value !== undefined && type !== typeof value) {
const description =
'The type of this field varies between items' +
(node.typeVersion > 1
? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
: '');
throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
description,
});
} else {
type = typeof value;
}
}
}
};
export function removeDuplicateInputItems(context: IExecuteFunctions, items: INodeExecutionData[]) {
const compare = context.getNodeParameter('compare', 0) as string;
const disableDotNotation = context.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
const removeOtherFields = context.getNodeParameter(
'options.removeOtherFields',
0,
false,
) as boolean;
let keys = disableDotNotation
? Object.keys(items[0].json)
: Object.keys(flattenKeys(items[0].json));
for (const item of items) {
const itemKeys = disableDotNotation
? Object.keys(item.json)
: Object.keys(flattenKeys(item.json));
for (const key of itemKeys) {
if (!keys.includes(key)) {
keys.push(key);
}
}
}
if (compare === 'allFieldsExcept') {
const fieldsToExclude = prepareFieldsArray(
context.getNodeParameter('fieldsToExclude', 0, '') as string,
'Fields To Exclude',
);
if (!fieldsToExclude.length) {
throw new NodeOperationError(
context.getNode(),
'No fields specified. Please add a field to exclude from comparison',
);
}
if (!disableDotNotation) {
keys = Object.keys(flattenKeys(items[0].json));
}
keys = keys.filter((key) => !fieldsToExclude.includes(key));
}
if (compare === 'selectedFields') {
const fieldsToCompare = prepareFieldsArray(
context.getNodeParameter('fieldsToCompare', 0, '') as string,
'Fields To Compare',
);
if (!fieldsToCompare.length) {
throw new NodeOperationError(
context.getNode(),
'No fields specified. Please add a field to compare on',
);
}
if (!disableDotNotation) {
keys = Object.keys(flattenKeys(items[0].json));
}
keys = fieldsToCompare.map((key) => key.trim());
}
// This solution is O(nlogn)
// add original index to the items
const newItems = items.map(
(item, index) =>
({
json: { ...item.json, __INDEX: index },
pairedItem: { item: index },
}) as INodeExecutionData,
);
//sort items using the compare keys
newItems.sort((a, b) => {
let result = 0;
for (const key of keys) {
let equal;
if (!disableDotNotation) {
equal = isEqual(get(a.json, key), get(b.json, key));
} else {
equal = isEqual(a.json[key], b.json[key]);
}
if (!equal) {
let lessThan;
if (!disableDotNotation) {
lessThan = lt(get(a.json, key), get(b.json, key));
} else {
lessThan = lt(a.json[key], b.json[key]);
}
result = lessThan ? -1 : 1;
break;
}
}
return result;
});
validateInputData(context.getNode(), newItems, keys, disableDotNotation);
// collect the original indexes of items to be removed
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];
}
}
let updatedItems: INodeExecutionData[] = items.filter(
(_, index) => !removedIndexes.includes(index),
);
if (removeOtherFields) {
updatedItems = updatedItems.map((item, index) => ({
json: pick(item.json, ...keys),
pairedItem: { item: index },
}));
}
return [updatedItems];
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/v1/RemoveDuplicatesV1.node.ts
+121
View File
@@ -0,0 +1,121 @@
import { NodeConnectionTypes } from 'n8n-workflow';
import type {
INodeTypeBaseDescription,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import { removeDuplicateInputItems } from '../utils';
const versionDescription: INodeTypeDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
subtitle: '',
version: [1, 1.1],
description: 'Delete items with matching field values',
defaults: {
name: 'Remove Duplicates',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Compare',
name: 'compare',
type: 'options',
options: [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
{
name: 'Selected Fields',
value: 'selectedFields',
},
],
default: 'allFields',
description: 'The fields of the input items to compare to see if they are the same',
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to exclude from the comparison',
default: '',
displayOptions: {
show: {
compare: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Compare',
name: 'fieldsToCompare',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to add to the comparison',
default: '',
displayOptions: {
show: {
compare: ['selectedFields'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
show: {
compare: ['allFieldsExcept', 'selectedFields'],
},
},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Remove Other Fields',
name: 'removeOtherFields',
type: 'boolean',
default: false,
description:
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
},
],
},
],
};
export class RemoveDuplicatesV1 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
return removeDuplicateInputItems(this, items);
}
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/RemoveDuplicatesV2.description.ts
+278
View File
@@ -0,0 +1,278 @@
import type { INodeProperties } from 'n8n-workflow';
const operationOptions = [
{
name: 'Remove Items Repeated Within Current Input',
value: 'removeDuplicateInputItems',
description: 'Remove duplicates from incoming items',
action: 'Remove items repeated within current input',
},
{
name: 'Remove Items Processed in Previous Executions',
value: 'removeItemsSeenInPreviousExecutions',
description: 'Deduplicate items already seen in previous executions',
action: 'Remove items processed in previous executions',
},
{
name: 'Clear Deduplication History',
value: 'clearDeduplicationHistory',
description: 'Wipe the store of previous items',
action: 'Clear deduplication history',
},
];
const compareOptions = [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
{
name: 'Selected Fields',
value: 'selectedFields',
},
];
const logicOptions = [
{
name: 'Value Is New',
value: 'removeItemsWithAlreadySeenKeyValues',
description: 'Remove all input items with values matching those already processed',
},
{
name: 'Value Is Higher than Any Previous Value',
value: 'removeItemsUpToStoredIncrementalKey',
description:
'Works with incremental values, removes all input items with values up to the stored value',
},
{
name: 'Value Is a Date Later than Any Previous Date',
value: 'removeItemsUpToStoredDate',
description:
'Works with date values, removes all input items with values up to the stored date',
},
];
const manageDatabaseModeOptions = [
{
name: 'Clean Database',
value: 'cleanDatabase',
description: 'Clear all values stored for a key in the database',
},
];
export const removeDuplicatesNodeFields: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: operationOptions,
default: 'removeDuplicateInputItems',
},
{
displayName: 'Compare',
name: 'compare',
type: 'options',
options: compareOptions,
default: 'allFields',
description: 'The fields of the input items to compare to see if they are the same',
displayOptions: {
show: {
operation: ['removeDuplicateInputItems'],
},
},
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to exclude from the comparison',
default: '',
displayOptions: {
show: {
compare: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Compare',
name: 'fieldsToCompare',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to add to the comparison',
default: '',
displayOptions: {
show: {
compare: ['selectedFields'],
},
},
},
// ----------------------------------
{
displayName: 'Keep Items Where',
name: 'logic',
type: 'options',
noDataExpression: true,
options: logicOptions,
default: 'removeItemsWithAlreadySeenKeyValues',
description:
'How to select input items to remove by comparing them with key values previously processed',
displayOptions: {
show: {
operation: ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'dedupeValue',
type: 'string',
default: '',
description: 'Use an input field (or a combination of fields) that has a unique ID value',
hint: 'The input field value to compare between items',
placeholder: 'e.g. ID',
required: true,
displayOptions: {
show: {
logic: ['removeItemsWithAlreadySeenKeyValues'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'incrementalDedupeValue',
type: 'number',
default: '',
description: 'Use an input field (or a combination of fields) that has an incremental value',
hint: 'The input field value to compare between items, an incremental value is expected',
placeholder: 'e.g. ID',
displayOptions: {
show: {
logic: ['removeItemsUpToStoredIncrementalKey'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'dateDedupeValue',
type: 'dateTime',
default: '',
description: 'Use an input field that has a date value in ISO format',
hint: 'The input field value to compare between items, a date is expected',
placeholder: ' e.g. 2024-08-09T13:44:16Z',
displayOptions: {
show: {
logic: ['removeItemsUpToStoredDate'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Mode',
name: 'mode',
type: 'options',
default: 'cleanDatabase',
description:
'How you want to modify the key values stored on the database. None of these modes removes input items.',
displayOptions: {
show: {
operation: ['clearDeduplicationHistory'],
},
},
options: manageDatabaseModeOptions,
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
show: {
operation: [
'removeDuplicateInputItems',
'removeItemsSeenInPreviousExecutions',
'clearDeduplicationHistory',
],
},
},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
displayOptions: {
show: {
'/operation': ['removeDuplicateInputItems'],
},
hide: {
'/compare': ['allFields'],
},
},
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Remove Other Fields',
name: 'removeOtherFields',
type: 'boolean',
default: false,
displayOptions: {
show: {
'/operation': ['removeDuplicateInputItems'],
},
hide: {
'/compare': ['allFields'],
},
},
description:
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
},
{
displayName: 'Scope',
name: 'scope',
type: 'options',
default: 'node',
displayOptions: {
show: {
'/operation': ['clearDeduplicationHistory', 'removeItemsSeenInPreviousExecutions'],
},
},
description:
'If set to ‘workflow,’ key values will be shared across all nodes in the workflow. If set to ‘node,’ key values will be specific to this node.',
options: [
{
name: 'Workflow',
value: 'workflow',
description: 'Deduplication info will be shared by all the nodes in the workflow',
},
{
name: 'Node',
value: 'node',
description: 'Deduplication info will be stored only for this node',
},
],
},
{
displayName: 'History Size',
name: 'historySize',
type: 'number',
default: 10000,
hint: 'The max number of past items to store for deduplication',
displayOptions: {
show: {
'/logic': ['removeItemsWithAlreadySeenKeyValues'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
],
},
];
packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/RemoveDuplicatesV2.node.ts
+271
View File
@@ -0,0 +1,271 @@
import { NodeConnectionTypes, NodeOperationError, tryToParseDateTime } from 'n8n-workflow';
import type {
INodeTypeBaseDescription,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
DeduplicationScope,
} from 'n8n-workflow';
import { removeDuplicatesNodeFields } from './RemoveDuplicatesV2.description';
import { removeDuplicateInputItems } from '../utils';
const versionDescription: INodeTypeDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
subtitle: '',
version: [2],
description: 'Delete items with matching field values',
defaults: {
name: 'Remove Duplicates',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
outputNames: ['Kept', 'Discarded'],
hints: [
{
message: 'The dedupe key set in “Value to Dedupe On” has no value',
displayCondition:
'={{ $parameter["operation"] === "removeItemsSeenInPreviousExecutions" && ($parameter["logic"] === "removeItemsWithAlreadySeenKeyValues" && $parameter["dedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredIncrementalKey" && $parameter["incrementalDedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredDate" && $parameter["dateDedupeValue"] === undefined) }}',
whenToDisplay: 'beforeExecution',
location: 'outputPane',
},
],
properties: [...removeDuplicatesNodeFields],
};
export class RemoveDuplicatesV2 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const operation = this.getNodeParameter('operation', 0);
const returnData: INodeExecutionData[][] = [];
const DEFAULT_MAX_ENTRIES = 10000;
try {
switch (operation) {
case 'removeDuplicateInputItems': {
return removeDuplicateInputItems(this, items);
}
case 'removeItemsSeenInPreviousExecutions': {
const logic = this.getNodeParameter('logic', 0);
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
if (logic === 'removeItemsWithAlreadySeenKeyValues') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let checkValue: string;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
checkValue = this.getNodeParameter('dedupeValue', itemIndex, '')?.toString() ?? '';
if (itemMapping[checkValue]) {
itemMapping[checkValue].push(items[itemIndex]);
} else {
itemMapping[checkValue] = [items[itemIndex]];
}
}
const maxEntries = this.getNodeParameter(
'options.historySize',
0,
DEFAULT_MAX_ENTRIES,
) as number;
const maxEntriesNum = Number(maxEntries);
const currentProcessedDataCount = await this.helpers.getProcessedDataCount(scope, {
mode: 'entries',
maxEntries,
});
if (currentProcessedDataCount + items.length > maxEntriesNum) {
throw new NodeOperationError(
this.getNode(),
'The number of items to be processed exceeds the maximum history size. Please increase the history size or reduce the number of items to be processed.',
);
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'entries', maxEntries },
);
const processedDataCount = await this.helpers.getProcessedDataCount(scope, {
mode: 'entries',
maxEntries,
});
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
if (maxEntriesNum > 0 && processedDataCount / maxEntriesNum > 0.5) {
this.addExecutionHints({
message: `Some duplicates may be not be removed since you're approaching the maximum history size (${maxEntriesNum} items). You can raise this limit using the ‘history size’ option.`,
location: 'outputPane',
});
}
return returnData;
} else if (logic === 'removeItemsUpToStoredIncrementalKey') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let parsedIncrementalKey: number;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const incrementalKey = this.getNodeParameter('incrementalDedupeValue', itemIndex, '');
if (!incrementalKey?.toString()) {
throw new NodeOperationError(
this.getNode(),
'The `Value to Dedupe` On is empty. Please provide a value.',
);
}
parsedIncrementalKey = Number(incrementalKey);
if (isNaN(parsedIncrementalKey)) {
throw new NodeOperationError(
this.getNode(),
`The value '${incrementalKey}' is not a number. Please provide a number.`,
);
}
if (itemMapping[parsedIncrementalKey]) {
itemMapping[parsedIncrementalKey].push(items[itemIndex]);
} else {
itemMapping[parsedIncrementalKey] = [items[itemIndex]];
}
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'latestIncrementalKey' },
);
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
return returnData;
} else if (logic === 'removeItemsUpToStoredDate') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let checkValue: string;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
checkValue =
this.getNodeParameter('dateDedupeValue', itemIndex, '')?.toString() ?? '';
if (!checkValue) {
throw new NodeOperationError(
this.getNode(),
'The `Value to Dedupe` On is empty. Please provide a value.',
);
}
try {
tryToParseDateTime(checkValue);
} catch (error) {
throw new NodeOperationError(
this.getNode(),
`The value '${checkValue}' is not a valid date. Please provide a valid date.`,
);
}
if (itemMapping[checkValue]) {
itemMapping[checkValue].push(items[itemIndex]);
} else {
itemMapping[checkValue] = [items[itemIndex]];
}
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'latestDate' },
);
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
return returnData;
} else {
return [items];
}
}
case 'clearDeduplicationHistory': {
const mode = this.getNodeParameter('mode', 0) as string;
if (mode === 'updateKeyValuesInDatabase') {
} else if (mode === 'deleteKeyValuesFromDatabase') {
} else if (mode === 'cleanDatabase') {
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
await this.helpers.clearAllProcessedItems(scope, {
mode: 'entries',
});
}
return [items];
}
default: {
return [items];
}
}
} catch (error) {
if (this.continueOnFail()) {
returnData.push([{ json: this.getInputData(0)[0].json, error }]);
} else {
throw error;
}
}
return returnData;
}
}
packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/test/RemoveDuplicates.test.ts
+130
View File
@@ -0,0 +1,130 @@
import { mock } from 'jest-mock-extended';
import type { IExecuteFunctions, INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow';
import { RemoveDuplicatesV2 } from '../RemoveDuplicatesV2.node';
describe('RemoveDuplicatesV2', () => {
let node: RemoveDuplicatesV2;
let executeFunctions: IExecuteFunctions;
beforeEach(() => {
const baseDescription: INodeTypeBaseDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
description: 'Delete items with matching field values',
};
node = new RemoveDuplicatesV2(baseDescription);
executeFunctions = mock<IExecuteFunctions>();
executeFunctions.helpers = {
checkProcessedAndRecord: jest.fn(),
clearAllProcessedItems: jest.fn(),
} as any;
executeFunctions.getInputData = jest.fn();
executeFunctions.getNodeParameter = jest.fn();
});
it('should Remove items repeated within current input based on all fields', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 1, name: 'John' } },
];
(executeFunctions.getInputData as jest.Mock<any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'removeDuplicateInputItems';
if (paramName === 'compare') return 'allFields';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
it('should Remove items repeated within current input based on selected fields', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 1, name: 'Doe' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'removeDuplicateInputItems';
if (paramName === 'compare') return 'selectedFields';
if (paramName === 'fieldsToCompare') return 'id';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
it('should remove items seen in previous executions', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 3, name: 'Doe' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string, itemIndex: number) => {
if (paramName === 'operation') return 'removeItemsSeenInPreviousExecutions';
if (paramName === 'logic') return 'removeItemsWithAlreadySeenKeyValues';
if (paramName === 'dedupeValue' && itemIndex === 0) return 1;
if (paramName === 'dedupeValue' && itemIndex === 1) return 2;
if (paramName === 'dedupeValue' && itemIndex === 2) return 3;
if (paramName === 'options.scope') return 'node';
if (paramName === 'options.historySize') return 10;
},
);
executeFunctions.helpers.getProcessedDataCount = jest.fn().mockReturnValue(3);
(executeFunctions.helpers.checkProcessedAndRecord as jest.Mock).mockReturnValue({
new: [1, 3],
processed: [2],
});
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(2);
expect(result[0]).toHaveLength(2);
expect(result[1]).toHaveLength(1);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 3, name: 'Doe' });
});
it('should clean database when managing key values', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'clearDeduplicationHistory';
if (paramName === 'mode') return 'cleanDatabase';
if (paramName === 'options.scope') return 'node';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
});