first commit

packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json

@@ -0,0 +1,19 @@
+{
+	"node": "n8n-nodes-base.aggregate",
+	"nodeVersion": "1.0",
+	"codexVersion": "1.0",
+	"details": "",
+	"categories": ["Core Nodes"],
+	"resources": {
+		"primaryDocumentation": [
+			{
+				"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.aggregate/"
+			}
+		],
+		"generic": []
+	},
+	"alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"],
+	"subcategories": {
+		"Core Nodes": ["Data Transformation"]
+	}
+}

packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts

@@ -0,0 +1,455 @@
+import get from 'lodash/get';
+import isEmpty from 'lodash/isEmpty';
+import set from 'lodash/set';
+import {
+	NodeOperationError,
+	type IDataObject,
+	type IExecuteFunctions,
+	type INodeExecutionData,
+	type INodeType,
+	type INodeTypeDescription,
+	type IPairedItemData,
+	NodeConnectionTypes,
+	type NodeExecutionHint,
+} from 'n8n-workflow';
+
+import { addBinariesToItem } from './utils';
+import { prepareFieldsArray } from '../utils/utils';
+
+export class Aggregate implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'Aggregate',
+		name: 'aggregate',
+		icon: 'file:aggregate.svg',
+		group: ['transform'],
+		subtitle: '',
+		version: 1,
+		description: 'Combine a field from many items into a list in a single item',
+		defaults: {
+			name: 'Aggregate',
+		},
+		inputs: [NodeConnectionTypes.Main],
+		outputs: [NodeConnectionTypes.Main],
+		builderHint: {
+			message:
+				'Need to combine items from multiple branches? Use merge node. This nodes combines all items from one branch into one item.',
+			relatedNodes: [
+				{
+					nodeType: 'n8n-nodes-base.merge',
+					relationHint: 'For multiple branches',
+				},
+				{
+					nodeType: 'n8n-nodes-base.splitOut',
+					relationHint: 'Reverse operation',
+				},
+			],
+		},
+		properties: [
+			{
+				displayName: 'Aggregate',
+				name: 'aggregate',
+				type: 'options',
+				default: 'aggregateIndividualFields',
+				options: [
+					{
+						name: 'Individual Fields',
+						value: 'aggregateIndividualFields',
+					},
+					{
+						name: 'All Item Data (Into a Single List)',
+						value: 'aggregateAllItemData',
+					},
+				],
+			},
+			{
+				displayName: 'Fields To Aggregate',
+				name: 'fieldsToAggregate',
+				type: 'fixedCollection',
+				typeOptions: {
+					multipleValues: true,
+				},
+				placeholder: 'Add Field To Aggregate',
+				default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] },
+				displayOptions: {
+					show: {
+						aggregate: ['aggregateIndividualFields'],
+					},
+				},
+				options: [
+					{
+						displayName: '',
+						name: 'fieldToAggregate',
+						values: [
+							{
+								displayName: 'Input Field Name',
+								name: 'fieldToAggregate',
+								type: 'string',
+								default: '',
+								description: 'The name of a field in the input items to aggregate together',
+								// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
+								placeholder: 'e.g. id',
+								hint: ' Enter the field name as text',
+								requiresDataPath: 'single',
+							},
+							{
+								displayName: 'Rename Field',
+								name: 'renameField',
+								type: 'boolean',
+								default: false,
+								description: 'Whether to give the field a different name in the output',
+							},
+							{
+								displayName: 'Output Field Name',
+								name: 'outputFieldName',
+								displayOptions: {
+									show: {
+										renameField: [true],
+									},
+								},
+								type: 'string',
+								default: '',
+								description:
+									'The name of the field to put the aggregated data in. Leave blank to use the input field name.',
+								requiresDataPath: 'single',
+							},
+						],
+					},
+				],
+			},
+			{
+				displayName: 'Put Output in Field',
+				name: 'destinationFieldName',
+				type: 'string',
+				displayOptions: {
+					show: {
+						aggregate: ['aggregateAllItemData'],
+					},
+				},
+				default: 'data',
+				description: 'The name of the output field to put the data in',
+			},
+			{
+				displayName: 'Include',
+				name: 'include',
+				type: 'options',
+				default: 'allFields',
+				options: [
+					{
+						name: 'All Fields',
+						value: 'allFields',
+					},
+					{
+						name: 'Specified Fields',
+						value: 'specifiedFields',
+					},
+					{
+						name: 'All Fields Except',
+						value: 'allFieldsExcept',
+					},
+				],
+				displayOptions: {
+					show: {
+						aggregate: ['aggregateAllItemData'],
+					},
+				},
+			},
+			{
+				displayName: 'Fields To Exclude',
+				name: 'fieldsToExclude',
+				type: 'string',
+				placeholder: 'e.g. email, name',
+				default: '',
+				requiresDataPath: 'multiple',
+				displayOptions: {
+					show: {
+						aggregate: ['aggregateAllItemData'],
+						include: ['allFieldsExcept'],
+					},
+				},
+			},
+			{
+				displayName: 'Fields To Include',
+				name: 'fieldsToInclude',
+				type: 'string',
+				placeholder: 'e.g. email, name',
+				default: '',
+				requiresDataPath: 'multiple',
+				displayOptions: {
+					show: {
+						aggregate: ['aggregateAllItemData'],
+						include: ['specifiedFields'],
+					},
+				},
+			},
+			{
+				displayName: 'Options',
+				name: 'options',
+				type: 'collection',
+				placeholder: 'Add Field',
+				default: {},
+				options: [
+					{
+						displayName: 'Disable Dot Notation',
+						name: 'disableDotNotation',
+						type: 'boolean',
+						default: false,
+						description:
+							'Whether to disallow referencing child fields using `parent.child` in the field name',
+						displayOptions: {
+							hide: {
+								'/aggregate': ['aggregateAllItemData'],
+							},
+						},
+					},
+					{
+						displayName: 'Merge Lists',
+						name: 'mergeLists',
+						type: 'boolean',
+						default: false,
+						description:
+							'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
+						displayOptions: {
+							hide: {
+								'/aggregate': ['aggregateAllItemData'],
+							},
+						},
+					},
+					{
+						displayName: 'Include Binaries',
+						name: 'includeBinaries',
+						type: 'boolean',
+						default: false,
+						description: 'Whether to include the binary data in the new item',
+					},
+					{
+						displayName: 'Keep Only Unique Binaries',
+						name: 'keepOnlyUnique',
+						type: 'boolean',
+						default: false,
+						description:
+							'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
+						displayOptions: {
+							show: {
+								includeBinaries: [true],
+							},
+						},
+					},
+					{
+						displayName: 'Keep Missing And Null Values',
+						name: 'keepMissing',
+						type: 'boolean',
+						default: false,
+						description:
+							'Whether to add a null entry to the aggregated list when there is a missing or null value',
+						displayOptions: {
+							hide: {
+								'/aggregate': ['aggregateAllItemData'],
+							},
+						},
+					},
+				],
+			},
+		],
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		let returnData: INodeExecutionData = { json: {}, pairedItem: [] };
+		const items = this.getInputData();
+		const notFoundedFields: { [key: string]: boolean[] } = {};
+
+		const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
+
+		if (aggregate === 'aggregateIndividualFields') {
+			const disableDotNotation = this.getNodeParameter(
+				'options.disableDotNotation',
+				0,
+				false,
+			) as boolean;
+			const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean;
+			const fieldsToAggregate = this.getNodeParameter(
+				'fieldsToAggregate.fieldToAggregate',
+				0,
+				[],
+			) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }];
+			const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean;
+
+			if (!fieldsToAggregate.length) {
+				throw new NodeOperationError(this.getNode(), 'No fields specified', {
+					description: 'Please add a field to aggregate',
+				});
+			}
+
+			const newItem: INodeExecutionData = {
+				json: {},
+				pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => {
+					return {
+						item: index,
+					};
+				}),
+			};
+
+			const values: { [key: string]: any } = {};
+			const outputFields: string[] = [];
+
+			for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) {
+				const field = renameField ? outputFieldName : fieldToAggregate;
+
+				if (outputFields.includes(field)) {
+					throw new NodeOperationError(
+						this.getNode(),
+						`The '${field}' output field is used more than once`,
+						{ description: 'Please make sure each output field name is unique' },
+					);
+				} else {
+					outputFields.push(field);
+				}
+
+				const getFieldToAggregate = () =>
+					!disableDotNotation && fieldToAggregate.includes('.')
+						? fieldToAggregate.split('.').pop()
+						: fieldToAggregate;
+
+				const _outputFieldName = outputFieldName
+					? outputFieldName
+					: (getFieldToAggregate() as string);
+
+				if (fieldToAggregate !== '') {
+					values[_outputFieldName] = [];
+					for (let i = 0; i < items.length; i++) {
+						if (notFoundedFields[fieldToAggregate] === undefined) {
+							notFoundedFields[fieldToAggregate] = [];
+						}
+
+						if (!disableDotNotation) {
+							let value = get(items[i].json, fieldToAggregate);
+							notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
+
+							if (!keepMissing) {
+								if (Array.isArray(value)) {
+									value = value.filter((entry) => entry !== null);
+								} else if (value === null || value === undefined) {
+									continue;
+								}
+							}
+
+							if (Array.isArray(value) && mergeLists) {
+								values[_outputFieldName].push(...value);
+							} else {
+								values[_outputFieldName].push(value);
+							}
+						} else {
+							let value = items[i].json[fieldToAggregate];
+							notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
+
+							if (!keepMissing) {
+								if (Array.isArray(value)) {
+									value = value.filter((entry) => entry !== null);
+								} else if (value === null || value === undefined) {
+									continue;
+								}
+							}
+
+							if (Array.isArray(value) && mergeLists) {
+								values[_outputFieldName].push(...value);
+							} else {
+								values[_outputFieldName].push(value);
+							}
+						}
+					}
+				}
+			}
+
+			for (const key of Object.keys(values)) {
+				if (!disableDotNotation) {
+					set(newItem.json, key, values[key]);
+				} else {
+					newItem.json[key] = values[key];
+				}
+			}
+
+			returnData = newItem;
+		} else {
+			let newItems: IDataObject[] = items.map((item) => item.json);
+			let pairedItem: IPairedItemData[] = [];
+			const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string;
+
+			const fieldsToExclude = prepareFieldsArray(
+				this.getNodeParameter('fieldsToExclude', 0, '') as string,
+				'Fields To Exclude',
+			);
+
+			const fieldsToInclude = prepareFieldsArray(
+				this.getNodeParameter('fieldsToInclude', 0, '') as string,
+				'Fields To Include',
+			);
+
+			if (fieldsToExclude.length || fieldsToInclude.length) {
+				newItems = newItems.reduce((acc, item, index) => {
+					const newItem: IDataObject = {};
+					let outputFields = Object.keys(item);
+
+					if (fieldsToExclude.length) {
+						outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key));
+					}
+					if (fieldsToInclude.length) {
+						outputFields = outputFields.filter((key) =>
+							fieldsToInclude.length ? fieldsToInclude.includes(key) : true,
+						);
+					}
+
+					outputFields.forEach((key) => {
+						newItem[key] = item[key];
+					});
+
+					if (isEmpty(newItem)) {
+						return acc;
+					}
+
+					pairedItem.push({ item: index });
+					return acc.concat([newItem]);
+				}, [] as IDataObject[]);
+			} else {
+				pairedItem = Array.from({ length: newItems.length }, (_, item) => ({
+					item,
+				}));
+			}
+
+			const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
+
+			returnData = output;
+		}
+
+		const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
+
+		if (includeBinaries) {
+			const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];
+
+			const aggregatedItems = pairedItems.map((item) => {
+				return items[item.item];
+			});
+
+			const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
+
+			addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
+		}
+
+		if (Object.keys(notFoundedFields).length) {
+			const hints: NodeExecutionHint[] = [];
+
+			for (const [field, values] of Object.entries(notFoundedFields)) {
+				if (values.every((value) => !value)) {
+					hints.push({
+						message: `The field '${field}' wasn't found in any input item`,
+						location: 'outputPane',
+					});
+				}
+			}
+
+			if (hints.length) {
+				this.addExecutionHints(...hints);
+			}
+		}
+
+		return [[returnData]];
+	}
+}

packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#FF6D5A" clip-path="url(#a)"><path fill-rule="evenodd" d="M32 148c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/><path d="M74 76c0 6.627 5.373 12 12 12h116.217c17.673 0 32 14.327 32 32v56c0 26.978 10.272 51.557 27.119 70.039 5.055 5.545 5.055 14.377 0 19.922-16.847 18.482-27.119 43.061-27.119 70.039v56c0 17.673-14.327 32-32 32H86c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h116.217c44.183 0 80-35.817 80-80v-56c0-30.928 25.072-56 56-56a5.783 5.783 0 0 0 5.783-5.783v-36.434a5.783 5.783 0 0 0-5.783-5.783c-30.928 0-56-25.072-56-56v-56c0-44.183-35.817-80-80-80H86c-6.627 0-12 5.373-12 12z"/><path fill-rule="evenodd" d="M376 244c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H388c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>

packages/nodes-base/nodes/Transform/Aggregate/test/Aggregate.test.ts

@@ -0,0 +1,5 @@
+import { NodeTestHarness } from '@nodes-testing/node-test-harness';
+
+describe('Test Aggregate Node', () => {
+	new NodeTestHarness().setupTests();
+});

packages/nodes-base/nodes/Transform/Aggregate/test/workflow.aggregate.json

@@ -0,0 +1,228 @@
+{
+	"name": "itemLists test",
+	"nodes": [
+		{
+			"parameters": {},
+			"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
+			"name": "When clicking \"Execute Workflow\"",
+			"type": "n8n-nodes-base.manualTrigger",
+			"typeVersion": 1,
+			"position": [-440, 260]
+		},
+		{
+			"parameters": {
+				"data": [
+					{
+						"id": 1,
+						"char": "a"
+					},
+					{
+						"id": 2,
+						"char": "b"
+					},
+					{
+						"id": 3,
+						"char": "c"
+					},
+					{
+						"id": 4,
+						"char": "d"
+					},
+					{
+						"id": 5,
+						"char": "e"
+					}
+				]
+			},
+			"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
+			"name": "Code",
+			"type": "n8n-nodes-testing.testData",
+			"typeVersion": 1,
+			"position": [-180, 260]
+		},
+		{
+			"parameters": {
+				"fieldsToAggregate": {
+					"fieldToAggregate": [
+						{
+							"fieldToAggregate": "id",
+							"renameField": true,
+							"outputFieldName": "data"
+						}
+					]
+				},
+				"options": {}
+			},
+			"id": "d95ca3a3-fb43-4037-846e-b87103dec1a3",
+			"name": "fields aggregate and rename",
+			"type": "n8n-nodes-base.aggregate",
+			"typeVersion": 1,
+			"position": [80, 0]
+		},
+		{
+			"parameters": {
+				"aggregate": "aggregateAllItemData"
+			},
+			"id": "4c1bc7be-7611-418d-aad5-8642b1cc0781",
+			"name": "aggregate all fields into list",
+			"type": "n8n-nodes-base.aggregate",
+			"typeVersion": 1,
+			"position": [80, 320]
+		},
+		{
+			"parameters": {
+				"aggregate": "aggregateAllItemData",
+				"include": "specifiedFields",
+				"fieldsToInclude": ["id"]
+			},
+			"id": "951de23c-2018-437b-961e-8ae7d7fd1a82",
+			"name": "aggregate selected fields into list",
+			"type": "n8n-nodes-base.aggregate",
+			"typeVersion": 1,
+			"position": [80, 500]
+		},
+		{
+			"parameters": {
+				"aggregate": "aggregateAllItemData",
+				"destinationFieldName": "output",
+				"include": "allFieldsExcept",
+				"fieldsToExclude": ["char"]
+			},
+			"id": "b62c02ee-5edb-473d-a755-7fb8700641fa",
+			"name": "aggregate all fields except selected into list",
+			"type": "n8n-nodes-base.aggregate",
+			"typeVersion": 1,
+			"position": [80, 700]
+		}
+	],
+	"pinData": {
+		"fields aggregate and rename": [
+			{
+				"json": {
+					"data": [1, 2, 3, 4, 5]
+				}
+			}
+		],
+		"aggregate all fields into list": [
+			{
+				"json": {
+					"data": [
+						{
+							"id": 1,
+							"char": "a"
+						},
+						{
+							"id": 2,
+							"char": "b"
+						},
+						{
+							"id": 3,
+							"char": "c"
+						},
+						{
+							"id": 4,
+							"char": "d"
+						},
+						{
+							"id": 5,
+							"char": "e"
+						}
+					]
+				}
+			}
+		],
+		"aggregate selected fields into list": [
+			{
+				"json": {
+					"data": [
+						{
+							"id": 1
+						},
+						{
+							"id": 2
+						},
+						{
+							"id": 3
+						},
+						{
+							"id": 4
+						},
+						{
+							"id": 5
+						}
+					]
+				}
+			}
+		],
+		"aggregate all fields except selected into list": [
+			{
+				"json": {
+					"output": [
+						{
+							"id": 1
+						},
+						{
+							"id": 2
+						},
+						{
+							"id": 3
+						},
+						{
+							"id": 4
+						},
+						{
+							"id": 5
+						}
+					]
+				}
+			}
+		]
+	},
+	"connections": {
+		"When clicking \"Execute Workflow\"": {
+			"main": [
+				[
+					{
+						"node": "Code",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Code": {
+			"main": [
+				[
+					{
+						"node": "fields aggregate and rename",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "aggregate all fields into list",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "aggregate selected fields into list",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "aggregate all fields except selected into list",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"active": false,
+	"settings": {},
+	"versionId": "9bf7c52b-b118-4dad-bfef-7db41828393b",
+	"id": "105",
+	"meta": {
+		"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
+	},
+	"tags": []
+}

packages/nodes-base/nodes/Transform/Aggregate/utils.ts

@@ -0,0 +1,60 @@
+import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
+
+type PartialBinaryData = Omit<IBinaryData, 'data'>;
+const isBinaryUniqueSetup = () => {
+	const binaries: PartialBinaryData[] = [];
+	return (binary: IBinaryData) => {
+		for (const existingBinary of binaries) {
+			if (
+				existingBinary.mimeType === binary.mimeType &&
+				existingBinary.fileType === binary.fileType &&
+				existingBinary.fileSize === binary.fileSize &&
+				existingBinary.fileExtension === binary.fileExtension
+			) {
+				return false;
+			}
+		}
+
+		binaries.push({
+			mimeType: binary.mimeType,
+			fileType: binary.fileType,
+			fileSize: binary.fileSize,
+			fileExtension: binary.fileExtension,
+		});
+
+		return true;
+	};
+};
+
+export function addBinariesToItem(
+	newItem: INodeExecutionData,
+	items: INodeExecutionData[],
+	uniqueOnly?: boolean,
+) {
+	const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;
+
+	for (const item of items) {
+		if (item.binary === undefined) continue;
+
+		for (const key of Object.keys(item.binary)) {
+			if (!newItem.binary) newItem.binary = {};
+			let binaryKey = key;
+			const binary = item.binary[key];
+
+			if (isBinaryUnique && !isBinaryUnique(binary)) {
+				continue;
+			}
+
+			// If the binary key already exists add a suffix to it
+			let i = 1;
+			while (newItem.binary[binaryKey] !== undefined) {
+				binaryKey = `${key}_${i}`;
+				i++;
+			}
+
+			newItem.binary[binaryKey] = binary;
+		}
+	}
+
+	return newItem;
+}