first commit

packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/RemoveDuplicatesV2.description.ts

@@ -0,0 +1,278 @@
+import type { INodeProperties } from 'n8n-workflow';
+const operationOptions = [
+	{
+		name: 'Remove Items Repeated Within Current Input',
+		value: 'removeDuplicateInputItems',
+		description: 'Remove duplicates from incoming items',
+		action: 'Remove items repeated within current input',
+	},
+	{
+		name: 'Remove Items Processed in Previous Executions',
+		value: 'removeItemsSeenInPreviousExecutions',
+		description: 'Deduplicate items already seen in previous executions',
+		action: 'Remove items processed in previous executions',
+	},
+	{
+		name: 'Clear Deduplication History',
+		value: 'clearDeduplicationHistory',
+		description: 'Wipe the store of previous items',
+		action: 'Clear deduplication history',
+	},
+];
+const compareOptions = [
+	{
+		name: 'All Fields',
+		value: 'allFields',
+	},
+	{
+		name: 'All Fields Except',
+		value: 'allFieldsExcept',
+	},
+	{
+		name: 'Selected Fields',
+		value: 'selectedFields',
+	},
+];
+const logicOptions = [
+	{
+		name: 'Value Is New',
+		value: 'removeItemsWithAlreadySeenKeyValues',
+		description: 'Remove all input items with values matching those already processed',
+	},
+	{
+		name: 'Value Is Higher than Any Previous Value',
+		value: 'removeItemsUpToStoredIncrementalKey',
+		description:
+			'Works with incremental values, removes all input items with values up to the stored value',
+	},
+	{
+		name: 'Value Is a Date Later than Any Previous Date',
+		value: 'removeItemsUpToStoredDate',
+		description:
+			'Works with date values, removes all input items with values up to the stored date',
+	},
+];
+const manageDatabaseModeOptions = [
+	{
+		name: 'Clean Database',
+		value: 'cleanDatabase',
+		description: 'Clear all values stored for a key in the database',
+	},
+];
+
+export const removeDuplicatesNodeFields: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		noDataExpression: true,
+		options: operationOptions,
+		default: 'removeDuplicateInputItems',
+	},
+	{
+		displayName: 'Compare',
+		name: 'compare',
+		type: 'options',
+		options: compareOptions,
+		default: 'allFields',
+		description: 'The fields of the input items to compare to see if they are the same',
+		displayOptions: {
+			show: {
+				operation: ['removeDuplicateInputItems'],
+			},
+		},
+	},
+	{
+		displayName: 'Fields To Exclude',
+		name: 'fieldsToExclude',
+		type: 'string',
+		placeholder: 'e.g. email, name',
+		requiresDataPath: 'multiple',
+		description: 'Fields in the input to exclude from the comparison',
+		default: '',
+		displayOptions: {
+			show: {
+				compare: ['allFieldsExcept'],
+			},
+		},
+	},
+	{
+		displayName: 'Fields To Compare',
+		name: 'fieldsToCompare',
+		type: 'string',
+		placeholder: 'e.g. email, name',
+		requiresDataPath: 'multiple',
+		description: 'Fields in the input to add to the comparison',
+		default: '',
+		displayOptions: {
+			show: {
+				compare: ['selectedFields'],
+			},
+		},
+	},
+
+	// ----------------------------------
+	{
+		displayName: 'Keep Items Where',
+		name: 'logic',
+		type: 'options',
+		noDataExpression: true,
+		options: logicOptions,
+		default: 'removeItemsWithAlreadySeenKeyValues',
+		description:
+			'How to select input items to remove by comparing them with key values previously processed',
+		displayOptions: {
+			show: {
+				operation: ['removeItemsSeenInPreviousExecutions'],
+			},
+		},
+	},
+	{
+		displayName: 'Value to Dedupe On',
+		name: 'dedupeValue',
+		type: 'string',
+		default: '',
+		description: 'Use an input field (or a combination of fields) that has a unique ID value',
+		hint: 'The input field value to compare between items',
+		placeholder: 'e.g. ID',
+		required: true,
+		displayOptions: {
+			show: {
+				logic: ['removeItemsWithAlreadySeenKeyValues'],
+				'/operation': ['removeItemsSeenInPreviousExecutions'],
+			},
+		},
+	},
+	{
+		displayName: 'Value to Dedupe On',
+		name: 'incrementalDedupeValue',
+		type: 'number',
+		default: '',
+		description: 'Use an input field (or a combination of fields) that has an incremental value',
+		hint: 'The input field value to compare between items, an incremental value is expected',
+		placeholder: 'e.g. ID',
+		displayOptions: {
+			show: {
+				logic: ['removeItemsUpToStoredIncrementalKey'],
+				'/operation': ['removeItemsSeenInPreviousExecutions'],
+			},
+		},
+	},
+	{
+		displayName: 'Value to Dedupe On',
+		name: 'dateDedupeValue',
+		type: 'dateTime',
+		default: '',
+		description: 'Use an input field that has a date value in ISO format',
+		hint: 'The input field value to compare between items, a date is expected',
+		placeholder: ' e.g. 2024-08-09T13:44:16Z',
+		displayOptions: {
+			show: {
+				logic: ['removeItemsUpToStoredDate'],
+				'/operation': ['removeItemsSeenInPreviousExecutions'],
+			},
+		},
+	},
+	{
+		displayName: 'Mode',
+		name: 'mode',
+		type: 'options',
+		default: 'cleanDatabase',
+		description:
+			'How you want to modify the key values stored on the database. None of these modes removes input items.',
+		displayOptions: {
+			show: {
+				operation: ['clearDeduplicationHistory'],
+			},
+		},
+		options: manageDatabaseModeOptions,
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add Field',
+		default: {},
+		displayOptions: {
+			show: {
+				operation: [
+					'removeDuplicateInputItems',
+					'removeItemsSeenInPreviousExecutions',
+					'clearDeduplicationHistory',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Disable Dot Notation',
+				name: 'disableDotNotation',
+				type: 'boolean',
+				default: false,
+				displayOptions: {
+					show: {
+						'/operation': ['removeDuplicateInputItems'],
+					},
+					hide: {
+						'/compare': ['allFields'],
+					},
+				},
+				description:
+					'Whether to disallow referencing child fields using `parent.child` in the field name',
+			},
+			{
+				displayName: 'Remove Other Fields',
+				name: 'removeOtherFields',
+				type: 'boolean',
+				default: false,
+				displayOptions: {
+					show: {
+						'/operation': ['removeDuplicateInputItems'],
+					},
+					hide: {
+						'/compare': ['allFields'],
+					},
+				},
+				description:
+					'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
+			},
+			{
+				displayName: 'Scope',
+				name: 'scope',
+				type: 'options',
+				default: 'node',
+				displayOptions: {
+					show: {
+						'/operation': ['clearDeduplicationHistory', 'removeItemsSeenInPreviousExecutions'],
+					},
+				},
+				description:
+					'If set to ‘workflow,’ key values will be shared across all nodes in the workflow. If set to ‘node,’ key values will be specific to this node.',
+				options: [
+					{
+						name: 'Workflow',
+						value: 'workflow',
+						description: 'Deduplication info will be shared by all the nodes in the workflow',
+					},
+					{
+						name: 'Node',
+						value: 'node',
+						description: 'Deduplication info will be stored only for this node',
+					},
+				],
+			},
+			{
+				displayName: 'History Size',
+				name: 'historySize',
+				type: 'number',
+				default: 10000,
+				hint: 'The max number of past items to store for deduplication',
+				displayOptions: {
+					show: {
+						'/logic': ['removeItemsWithAlreadySeenKeyValues'],
+						'/operation': ['removeItemsSeenInPreviousExecutions'],
+					},
+				},
+			},
+		],
+	},
+];

packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/RemoveDuplicatesV2.node.ts

@@ -0,0 +1,271 @@
+import { NodeConnectionTypes, NodeOperationError, tryToParseDateTime } from 'n8n-workflow';
+import type {
+	INodeTypeBaseDescription,
+	IExecuteFunctions,
+	INodeExecutionData,
+	INodeType,
+	INodeTypeDescription,
+	DeduplicationScope,
+} from 'n8n-workflow';
+
+import { removeDuplicatesNodeFields } from './RemoveDuplicatesV2.description';
+import { removeDuplicateInputItems } from '../utils';
+
+const versionDescription: INodeTypeDescription = {
+	displayName: 'Remove Duplicates',
+	name: 'removeDuplicates',
+	icon: 'file:removeDuplicates.svg',
+	group: ['transform'],
+	subtitle: '',
+	version: [2],
+	description: 'Delete items with matching field values',
+	defaults: {
+		name: 'Remove Duplicates',
+	},
+	inputs: [NodeConnectionTypes.Main],
+	outputs: [NodeConnectionTypes.Main],
+	outputNames: ['Kept', 'Discarded'],
+	hints: [
+		{
+			message: 'The dedupe key set in “Value to Dedupe On” has no value',
+			displayCondition:
+				'={{ $parameter["operation"] === "removeItemsSeenInPreviousExecutions" && ($parameter["logic"] === "removeItemsWithAlreadySeenKeyValues" && $parameter["dedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredIncrementalKey" && $parameter["incrementalDedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredDate" && $parameter["dateDedupeValue"] === undefined) }}',
+			whenToDisplay: 'beforeExecution',
+			location: 'outputPane',
+		},
+	],
+	properties: [...removeDuplicatesNodeFields],
+};
+export class RemoveDuplicatesV2 implements INodeType {
+	description: INodeTypeDescription;
+
+	constructor(baseDescription: INodeTypeBaseDescription) {
+		this.description = {
+			...baseDescription,
+			...versionDescription,
+		};
+	}
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+		const operation = this.getNodeParameter('operation', 0);
+		const returnData: INodeExecutionData[][] = [];
+		const DEFAULT_MAX_ENTRIES = 10000;
+		try {
+			switch (operation) {
+				case 'removeDuplicateInputItems': {
+					return removeDuplicateInputItems(this, items);
+				}
+				case 'removeItemsSeenInPreviousExecutions': {
+					const logic = this.getNodeParameter('logic', 0);
+					const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
+
+					if (logic === 'removeItemsWithAlreadySeenKeyValues') {
+						if (!['node', 'workflow'].includes(scope)) {
+							throw new NodeOperationError(
+								this.getNode(),
+								`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
+							);
+						}
+
+						let checkValue: string;
+						const itemMapping: {
+							[key: string]: INodeExecutionData[];
+						} = {};
+						for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+							checkValue = this.getNodeParameter('dedupeValue', itemIndex, '')?.toString() ?? '';
+							if (itemMapping[checkValue]) {
+								itemMapping[checkValue].push(items[itemIndex]);
+							} else {
+								itemMapping[checkValue] = [items[itemIndex]];
+							}
+						}
+
+						const maxEntries = this.getNodeParameter(
+							'options.historySize',
+							0,
+							DEFAULT_MAX_ENTRIES,
+						) as number;
+						const maxEntriesNum = Number(maxEntries);
+
+						const currentProcessedDataCount = await this.helpers.getProcessedDataCount(scope, {
+							mode: 'entries',
+							maxEntries,
+						});
+						if (currentProcessedDataCount + items.length > maxEntriesNum) {
+							throw new NodeOperationError(
+								this.getNode(),
+								'The number of items to be processed exceeds the maximum history size. Please increase the history size or reduce the number of items to be processed.',
+							);
+						}
+						const itemsProcessed = await this.helpers.checkProcessedAndRecord(
+							Object.keys(itemMapping),
+							scope,
+							{ mode: 'entries', maxEntries },
+						);
+						const processedDataCount = await this.helpers.getProcessedDataCount(scope, {
+							mode: 'entries',
+							maxEntries,
+						});
+						returnData.push(
+							itemsProcessed.new
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+							itemsProcessed.processed
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+						);
+
+						if (maxEntriesNum > 0 && processedDataCount / maxEntriesNum > 0.5) {
+							this.addExecutionHints({
+								message: `Some duplicates may be not be removed since you're approaching the maximum history size (${maxEntriesNum} items). You can raise this limit using the ‘history size’ option.`,
+								location: 'outputPane',
+							});
+						}
+						return returnData;
+					} else if (logic === 'removeItemsUpToStoredIncrementalKey') {
+						if (!['node', 'workflow'].includes(scope)) {
+							throw new NodeOperationError(
+								this.getNode(),
+								`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
+							);
+						}
+
+						let parsedIncrementalKey: number;
+						const itemMapping: {
+							[key: string]: INodeExecutionData[];
+						} = {};
+
+						for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+							const incrementalKey = this.getNodeParameter('incrementalDedupeValue', itemIndex, '');
+							if (!incrementalKey?.toString()) {
+								throw new NodeOperationError(
+									this.getNode(),
+									'The `Value to Dedupe` On is empty. Please provide a value.',
+								);
+							}
+							parsedIncrementalKey = Number(incrementalKey);
+							if (isNaN(parsedIncrementalKey)) {
+								throw new NodeOperationError(
+									this.getNode(),
+									`The value '${incrementalKey}' is not a number. Please provide a number.`,
+								);
+							}
+							if (itemMapping[parsedIncrementalKey]) {
+								itemMapping[parsedIncrementalKey].push(items[itemIndex]);
+							} else {
+								itemMapping[parsedIncrementalKey] = [items[itemIndex]];
+							}
+						}
+
+						const itemsProcessed = await this.helpers.checkProcessedAndRecord(
+							Object.keys(itemMapping),
+							scope,
+							{ mode: 'latestIncrementalKey' },
+						);
+
+						returnData.push(
+							itemsProcessed.new
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+							itemsProcessed.processed
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+						);
+
+						return returnData;
+					} else if (logic === 'removeItemsUpToStoredDate') {
+						if (!['node', 'workflow'].includes(scope)) {
+							throw new NodeOperationError(
+								this.getNode(),
+								`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
+							);
+						}
+
+						let checkValue: string;
+						const itemMapping: {
+							[key: string]: INodeExecutionData[];
+						} = {};
+
+						for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+							checkValue =
+								this.getNodeParameter('dateDedupeValue', itemIndex, '')?.toString() ?? '';
+							if (!checkValue) {
+								throw new NodeOperationError(
+									this.getNode(),
+									'The `Value to Dedupe` On is empty. Please provide a value.',
+								);
+							}
+							try {
+								tryToParseDateTime(checkValue);
+							} catch (error) {
+								throw new NodeOperationError(
+									this.getNode(),
+									`The value '${checkValue}' is not a valid date. Please provide a valid date.`,
+								);
+							}
+							if (itemMapping[checkValue]) {
+								itemMapping[checkValue].push(items[itemIndex]);
+							} else {
+								itemMapping[checkValue] = [items[itemIndex]];
+							}
+						}
+						const itemsProcessed = await this.helpers.checkProcessedAndRecord(
+							Object.keys(itemMapping),
+							scope,
+							{ mode: 'latestDate' },
+						);
+
+						returnData.push(
+							itemsProcessed.new
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+							itemsProcessed.processed
+								.map((key) => {
+									return itemMapping[key];
+								})
+								.flat(),
+						);
+
+						return returnData;
+					} else {
+						return [items];
+					}
+				}
+				case 'clearDeduplicationHistory': {
+					const mode = this.getNodeParameter('mode', 0) as string;
+					if (mode === 'updateKeyValuesInDatabase') {
+					} else if (mode === 'deleteKeyValuesFromDatabase') {
+					} else if (mode === 'cleanDatabase') {
+						const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
+						await this.helpers.clearAllProcessedItems(scope, {
+							mode: 'entries',
+						});
+					}
+
+					return [items];
+				}
+				default: {
+					return [items];
+				}
+			}
+		} catch (error) {
+			if (this.continueOnFail()) {
+				returnData.push([{ json: this.getInputData(0)[0].json, error }]);
+			} else {
+				throw error;
+			}
+		}
+		return returnData;
+	}
+}

packages/nodes-base/nodes/Transform/RemoveDuplicates/v2/test/RemoveDuplicates.test.ts

@@ -0,0 +1,130 @@
+import { mock } from 'jest-mock-extended';
+import type { IExecuteFunctions, INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow';
+
+import { RemoveDuplicatesV2 } from '../RemoveDuplicatesV2.node';
+
+describe('RemoveDuplicatesV2', () => {
+	let node: RemoveDuplicatesV2;
+	let executeFunctions: IExecuteFunctions;
+
+	beforeEach(() => {
+		const baseDescription: INodeTypeBaseDescription = {
+			displayName: 'Remove Duplicates',
+			name: 'removeDuplicates',
+			icon: 'file:removeDuplicates.svg',
+			group: ['transform'],
+			description: 'Delete items with matching field values',
+		};
+		node = new RemoveDuplicatesV2(baseDescription);
+		executeFunctions = mock<IExecuteFunctions>();
+		executeFunctions.helpers = {
+			checkProcessedAndRecord: jest.fn(),
+			clearAllProcessedItems: jest.fn(),
+		} as any;
+		executeFunctions.getInputData = jest.fn();
+		executeFunctions.getNodeParameter = jest.fn();
+	});
+
+	it('should Remove items repeated within current input based on all fields', async () => {
+		const items: INodeExecutionData[] = [
+			{ json: { id: 1, name: 'John' } },
+			{ json: { id: 2, name: 'Jane' } },
+			{ json: { id: 1, name: 'John' } },
+		];
+
+		(executeFunctions.getInputData as jest.Mock<any>).mockReturnValue(items);
+		(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
+			(paramName: string) => {
+				if (paramName === 'operation') return 'removeDuplicateInputItems';
+				if (paramName === 'compare') return 'allFields';
+				return undefined;
+			},
+		);
+
+		const result = await node.execute.call(executeFunctions);
+		expect(result).toHaveLength(1);
+		expect(result[0]).toHaveLength(2);
+		expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
+		expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
+	});
+
+	it('should Remove items repeated within current input based on selected fields', async () => {
+		const items: INodeExecutionData[] = [
+			{ json: { id: 1, name: 'John' } },
+			{ json: { id: 2, name: 'Jane' } },
+			{ json: { id: 1, name: 'Doe' } },
+		];
+
+		(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
+		(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
+			(paramName: string) => {
+				if (paramName === 'operation') return 'removeDuplicateInputItems';
+				if (paramName === 'compare') return 'selectedFields';
+				if (paramName === 'fieldsToCompare') return 'id';
+				return undefined;
+			},
+		);
+
+		const result = await node.execute.call(executeFunctions);
+		expect(result).toHaveLength(1);
+		expect(result[0]).toHaveLength(2);
+		expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
+		expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
+	});
+
+	it('should remove items seen in previous executions', async () => {
+		const items: INodeExecutionData[] = [
+			{ json: { id: 1, name: 'John' } },
+			{ json: { id: 2, name: 'Jane' } },
+			{ json: { id: 3, name: 'Doe' } },
+		];
+
+		(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
+		(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
+			(paramName: string, itemIndex: number) => {
+				if (paramName === 'operation') return 'removeItemsSeenInPreviousExecutions';
+				if (paramName === 'logic') return 'removeItemsWithAlreadySeenKeyValues';
+				if (paramName === 'dedupeValue' && itemIndex === 0) return 1;
+				if (paramName === 'dedupeValue' && itemIndex === 1) return 2;
+				if (paramName === 'dedupeValue' && itemIndex === 2) return 3;
+				if (paramName === 'options.scope') return 'node';
+				if (paramName === 'options.historySize') return 10;
+			},
+		);
+		executeFunctions.helpers.getProcessedDataCount = jest.fn().mockReturnValue(3);
+		(executeFunctions.helpers.checkProcessedAndRecord as jest.Mock).mockReturnValue({
+			new: [1, 3],
+			processed: [2],
+		});
+
+		const result = await node.execute.call(executeFunctions);
+		expect(result).toHaveLength(2);
+		expect(result[0]).toHaveLength(2);
+		expect(result[1]).toHaveLength(1);
+		expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
+		expect(result[0][1].json).toEqual({ id: 3, name: 'Doe' });
+	});
+
+	it('should clean database when managing key values', async () => {
+		const items: INodeExecutionData[] = [
+			{ json: { id: 1, name: 'John' } },
+			{ json: { id: 2, name: 'Jane' } },
+		];
+
+		(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
+		(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
+			(paramName: string) => {
+				if (paramName === 'operation') return 'clearDeduplicationHistory';
+				if (paramName === 'mode') return 'cleanDatabase';
+				if (paramName === 'options.scope') return 'node';
+				return undefined;
+			},
+		);
+
+		const result = await node.execute.call(executeFunctions);
+		expect(result).toHaveLength(1);
+		expect(result[0]).toHaveLength(2);
+		expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
+		expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
+	});
+});