first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled

This commit is contained in:
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
@@ -0,0 +1,19 @@
{
"node": "n8n-nodes-base.aggregate",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.aggregate/"
}
],
"generic": []
},
"alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,455 @@
import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import set from 'lodash/set';
import {
NodeOperationError,
type IDataObject,
type IExecuteFunctions,
type INodeExecutionData,
type INodeType,
type INodeTypeDescription,
type IPairedItemData,
NodeConnectionTypes,
type NodeExecutionHint,
} from 'n8n-workflow';
import { addBinariesToItem } from './utils';
import { prepareFieldsArray } from '../utils/utils';
export class Aggregate implements INodeType {
description: INodeTypeDescription = {
displayName: 'Aggregate',
name: 'aggregate',
icon: 'file:aggregate.svg',
group: ['transform'],
subtitle: '',
version: 1,
description: 'Combine a field from many items into a list in a single item',
defaults: {
name: 'Aggregate',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
builderHint: {
message:
'Need to combine items from multiple branches? Use merge node. This nodes combines all items from one branch into one item.',
relatedNodes: [
{
nodeType: 'n8n-nodes-base.merge',
relationHint: 'For multiple branches',
},
{
nodeType: 'n8n-nodes-base.splitOut',
relationHint: 'Reverse operation',
},
],
},
properties: [
{
displayName: 'Aggregate',
name: 'aggregate',
type: 'options',
default: 'aggregateIndividualFields',
options: [
{
name: 'Individual Fields',
value: 'aggregateIndividualFields',
},
{
name: 'All Item Data (Into a Single List)',
value: 'aggregateAllItemData',
},
],
},
{
displayName: 'Fields To Aggregate',
name: 'fieldsToAggregate',
type: 'fixedCollection',
typeOptions: {
multipleValues: true,
},
placeholder: 'Add Field To Aggregate',
default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] },
displayOptions: {
show: {
aggregate: ['aggregateIndividualFields'],
},
},
options: [
{
displayName: '',
name: 'fieldToAggregate',
values: [
{
displayName: 'Input Field Name',
name: 'fieldToAggregate',
type: 'string',
default: '',
description: 'The name of a field in the input items to aggregate together',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: ' Enter the field name as text',
requiresDataPath: 'single',
},
{
displayName: 'Rename Field',
name: 'renameField',
type: 'boolean',
default: false,
description: 'Whether to give the field a different name in the output',
},
{
displayName: 'Output Field Name',
name: 'outputFieldName',
displayOptions: {
show: {
renameField: [true],
},
},
type: 'string',
default: '',
description:
'The name of the field to put the aggregated data in. Leave blank to use the input field name.',
requiresDataPath: 'single',
},
],
},
],
},
{
displayName: 'Put Output in Field',
name: 'destinationFieldName',
type: 'string',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
},
},
default: 'data',
description: 'The name of the output field to put the data in',
},
{
displayName: 'Include',
name: 'include',
type: 'options',
default: 'allFields',
options: [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'Specified Fields',
value: 'specifiedFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
],
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
},
},
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
default: '',
requiresDataPath: 'multiple',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
include: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Include',
name: 'fieldsToInclude',
type: 'string',
placeholder: 'e.g. email, name',
default: '',
requiresDataPath: 'multiple',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
include: ['specifiedFields'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Merge Lists',
name: 'mergeLists',
type: 'boolean',
default: false,
description:
'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Include Binaries',
name: 'includeBinaries',
type: 'boolean',
default: false,
description: 'Whether to include the binary data in the new item',
},
{
displayName: 'Keep Only Unique Binaries',
name: 'keepOnlyUnique',
type: 'boolean',
default: false,
description:
'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
displayOptions: {
show: {
includeBinaries: [true],
},
},
},
{
displayName: 'Keep Missing And Null Values',
name: 'keepMissing',
type: 'boolean',
default: false,
description:
'Whether to add a null entry to the aggregated list when there is a missing or null value',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
let returnData: INodeExecutionData = { json: {}, pairedItem: [] };
const items = this.getInputData();
const notFoundedFields: { [key: string]: boolean[] } = {};
const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
if (aggregate === 'aggregateIndividualFields') {
const disableDotNotation = this.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean;
const fieldsToAggregate = this.getNodeParameter(
'fieldsToAggregate.fieldToAggregate',
0,
[],
) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }];
const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean;
if (!fieldsToAggregate.length) {
throw new NodeOperationError(this.getNode(), 'No fields specified', {
description: 'Please add a field to aggregate',
});
}
const newItem: INodeExecutionData = {
json: {},
pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => {
return {
item: index,
};
}),
};
const values: { [key: string]: any } = {};
const outputFields: string[] = [];
for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) {
const field = renameField ? outputFieldName : fieldToAggregate;
if (outputFields.includes(field)) {
throw new NodeOperationError(
this.getNode(),
`The '${field}' output field is used more than once`,
{ description: 'Please make sure each output field name is unique' },
);
} else {
outputFields.push(field);
}
const getFieldToAggregate = () =>
!disableDotNotation && fieldToAggregate.includes('.')
? fieldToAggregate.split('.').pop()
: fieldToAggregate;
const _outputFieldName = outputFieldName
? outputFieldName
: (getFieldToAggregate() as string);
if (fieldToAggregate !== '') {
values[_outputFieldName] = [];
for (let i = 0; i < items.length; i++) {
if (notFoundedFields[fieldToAggregate] === undefined) {
notFoundedFields[fieldToAggregate] = [];
}
if (!disableDotNotation) {
let value = get(items[i].json, fieldToAggregate);
notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
if (!keepMissing) {
if (Array.isArray(value)) {
value = value.filter((entry) => entry !== null);
} else if (value === null || value === undefined) {
continue;
}
}
if (Array.isArray(value) && mergeLists) {
values[_outputFieldName].push(...value);
} else {
values[_outputFieldName].push(value);
}
} else {
let value = items[i].json[fieldToAggregate];
notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
if (!keepMissing) {
if (Array.isArray(value)) {
value = value.filter((entry) => entry !== null);
} else if (value === null || value === undefined) {
continue;
}
}
if (Array.isArray(value) && mergeLists) {
values[_outputFieldName].push(...value);
} else {
values[_outputFieldName].push(value);
}
}
}
}
}
for (const key of Object.keys(values)) {
if (!disableDotNotation) {
set(newItem.json, key, values[key]);
} else {
newItem.json[key] = values[key];
}
}
returnData = newItem;
} else {
let newItems: IDataObject[] = items.map((item) => item.json);
let pairedItem: IPairedItemData[] = [];
const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string;
const fieldsToExclude = prepareFieldsArray(
this.getNodeParameter('fieldsToExclude', 0, '') as string,
'Fields To Exclude',
);
const fieldsToInclude = prepareFieldsArray(
this.getNodeParameter('fieldsToInclude', 0, '') as string,
'Fields To Include',
);
if (fieldsToExclude.length || fieldsToInclude.length) {
newItems = newItems.reduce((acc, item, index) => {
const newItem: IDataObject = {};
let outputFields = Object.keys(item);
if (fieldsToExclude.length) {
outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key));
}
if (fieldsToInclude.length) {
outputFields = outputFields.filter((key) =>
fieldsToInclude.length ? fieldsToInclude.includes(key) : true,
);
}
outputFields.forEach((key) => {
newItem[key] = item[key];
});
if (isEmpty(newItem)) {
return acc;
}
pairedItem.push({ item: index });
return acc.concat([newItem]);
}, [] as IDataObject[]);
} else {
pairedItem = Array.from({ length: newItems.length }, (_, item) => ({
item,
}));
}
const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
returnData = output;
}
const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
if (includeBinaries) {
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];
const aggregatedItems = pairedItems.map((item) => {
return items[item.item];
});
const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
}
if (Object.keys(notFoundedFields).length) {
const hints: NodeExecutionHint[] = [];
for (const [field, values] of Object.entries(notFoundedFields)) {
if (values.every((value) => !value)) {
hints.push({
message: `The field '${field}' wasn't found in any input item`,
location: 'outputPane',
});
}
}
if (hints.length) {
this.addExecutionHints(...hints);
}
}
return [[returnData]];
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#FF6D5A" clip-path="url(#a)"><path fill-rule="evenodd" d="M32 148c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/><path d="M74 76c0 6.627 5.373 12 12 12h116.217c17.673 0 32 14.327 32 32v56c0 26.978 10.272 51.557 27.119 70.039 5.055 5.545 5.055 14.377 0 19.922-16.847 18.482-27.119 43.061-27.119 70.039v56c0 17.673-14.327 32-32 32H86c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h116.217c44.183 0 80-35.817 80-80v-56c0-30.928 25.072-56 56-56a5.783 5.783 0 0 0 5.783-5.783v-36.434a5.783 5.783 0 0 0-5.783-5.783c-30.928 0-56-25.072-56-56v-56c0-44.183-35.817-80-80-80H86c-6.627 0-12 5.373-12 12z"/><path fill-rule="evenodd" d="M376 244c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H388c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 1.2 KiB

@@ -0,0 +1,5 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
describe('Test Aggregate Node', () => {
new NodeTestHarness().setupTests();
});
@@ -0,0 +1,228 @@
{
"name": "itemLists test",
"nodes": [
{
"parameters": {},
"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-440, 260]
},
{
"parameters": {
"data": [
{
"id": 1,
"char": "a"
},
{
"id": 2,
"char": "b"
},
{
"id": 3,
"char": "c"
},
{
"id": 4,
"char": "d"
},
{
"id": 5,
"char": "e"
}
]
},
"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [-180, 260]
},
{
"parameters": {
"fieldsToAggregate": {
"fieldToAggregate": [
{
"fieldToAggregate": "id",
"renameField": true,
"outputFieldName": "data"
}
]
},
"options": {}
},
"id": "d95ca3a3-fb43-4037-846e-b87103dec1a3",
"name": "fields aggregate and rename",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 0]
},
{
"parameters": {
"aggregate": "aggregateAllItemData"
},
"id": "4c1bc7be-7611-418d-aad5-8642b1cc0781",
"name": "aggregate all fields into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 320]
},
{
"parameters": {
"aggregate": "aggregateAllItemData",
"include": "specifiedFields",
"fieldsToInclude": ["id"]
},
"id": "951de23c-2018-437b-961e-8ae7d7fd1a82",
"name": "aggregate selected fields into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 500]
},
{
"parameters": {
"aggregate": "aggregateAllItemData",
"destinationFieldName": "output",
"include": "allFieldsExcept",
"fieldsToExclude": ["char"]
},
"id": "b62c02ee-5edb-473d-a755-7fb8700641fa",
"name": "aggregate all fields except selected into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 700]
}
],
"pinData": {
"fields aggregate and rename": [
{
"json": {
"data": [1, 2, 3, 4, 5]
}
}
],
"aggregate all fields into list": [
{
"json": {
"data": [
{
"id": 1,
"char": "a"
},
{
"id": 2,
"char": "b"
},
{
"id": 3,
"char": "c"
},
{
"id": 4,
"char": "d"
},
{
"id": 5,
"char": "e"
}
]
}
}
],
"aggregate selected fields into list": [
{
"json": {
"data": [
{
"id": 1
},
{
"id": 2
},
{
"id": 3
},
{
"id": 4
},
{
"id": 5
}
]
}
}
],
"aggregate all fields except selected into list": [
{
"json": {
"output": [
{
"id": 1
},
{
"id": 2
},
{
"id": 3
},
{
"id": 4
},
{
"id": 5
}
]
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "fields aggregate and rename",
"type": "main",
"index": 0
},
{
"node": "aggregate all fields into list",
"type": "main",
"index": 0
},
{
"node": "aggregate selected fields into list",
"type": "main",
"index": 0
},
{
"node": "aggregate all fields except selected into list",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "9bf7c52b-b118-4dad-bfef-7db41828393b",
"id": "105",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
@@ -0,0 +1,60 @@
import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
type PartialBinaryData = Omit<IBinaryData, 'data'>;
const isBinaryUniqueSetup = () => {
const binaries: PartialBinaryData[] = [];
return (binary: IBinaryData) => {
for (const existingBinary of binaries) {
if (
existingBinary.mimeType === binary.mimeType &&
existingBinary.fileType === binary.fileType &&
existingBinary.fileSize === binary.fileSize &&
existingBinary.fileExtension === binary.fileExtension
) {
return false;
}
}
binaries.push({
mimeType: binary.mimeType,
fileType: binary.fileType,
fileSize: binary.fileSize,
fileExtension: binary.fileExtension,
});
return true;
};
};
export function addBinariesToItem(
newItem: INodeExecutionData,
items: INodeExecutionData[],
uniqueOnly?: boolean,
) {
const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;
for (const item of items) {
if (item.binary === undefined) continue;
for (const key of Object.keys(item.binary)) {
if (!newItem.binary) newItem.binary = {};
let binaryKey = key;
const binary = item.binary[key];
if (isBinaryUnique && !isBinaryUnique(binary)) {
continue;
}
// If the binary key already exists add a suffix to it
let i = 1;
while (newItem.binary[binaryKey] !== undefined) {
binaryKey = `${key}_${i}`;
i++;
}
newItem.binary[binaryKey] = binary;
}
}
return newItem;
}
@@ -0,0 +1,19 @@
{
"node": "n8n-nodes-base.limit",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.limit/"
}
],
"generic": []
},
"alias": ["Limit", "Remove", "Slice", "Transform", "Array", "List", "Item"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,71 @@
import type {
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import { NodeConnectionTypes } from 'n8n-workflow';
export class Limit implements INodeType {
description: INodeTypeDescription = {
displayName: 'Limit',
name: 'limit',
icon: 'file:limit.svg',
group: ['transform'],
subtitle: '',
version: 1,
description: 'Restrict the number of items',
defaults: {
name: 'Limit',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Max Items',
name: 'maxItems',
type: 'number',
typeOptions: {
minValue: 1,
},
default: 1,
description: 'If there are more items than this number, some are removed',
},
{
displayName: 'Keep',
name: 'keep',
type: 'options',
options: [
{
name: 'First Items',
value: 'firstItems',
},
{
name: 'Last Items',
value: 'lastItems',
},
],
default: 'firstItems',
description: 'When removing items, whether to keep the ones at the start or the ending',
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
let returnData = items;
const maxItems = this.getNodeParameter('maxItems', 0) as number;
const keep = this.getNodeParameter('keep', 0) as string;
if (maxItems > items.length) {
return [returnData];
}
if (keep === 'firstItems') {
returnData = items.slice(0, maxItems);
} else {
returnData = items.slice(items.length - maxItems, items.length);
}
return [returnData];
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#2FB67C" fill-rule="evenodd" clip-path="url(#a)" clip-rule="evenodd"><path d="M512 458c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12H12c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm152-222c-6.627 0-12-5.373-12-12V30c0-6.627 5.373-12 12-12h24c6.627 0 12 5.373 12 12v194c0 6.627-5.373 12-12 12z"/><path d="M149.577 146.982c9.398-9.346 24.594-9.304 33.941.095L256 219.964l72.482-72.887c9.347-9.399 24.543-9.441 33.941-.095s9.441 24.543.095 33.941l-89.5 90a24 24 0 0 1-34.036 0l-89.5-90c-9.346-9.398-9.304-24.594.095-33.941M0 350c0-6.627 5.373-12 12-12h488c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H12c-6.627 0-12-5.373-12-12z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 1.1 KiB

@@ -0,0 +1,5 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
describe('Test Limit Node', () => {
new NodeTestHarness().setupTests();
});
@@ -0,0 +1,113 @@
{
"name": "itemLists test",
"nodes": [
{
"parameters": {},
"id": "bd7af0bb-de39-44b4-ac11-eb1d22f5e8d7",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [260, 180]
},
{
"parameters": {
"data": [
{
"entry": 1
},
{
"entry": 2
},
{
"entry": 3
},
{
"entry": 4
},
{
"entry": 5
}
]
},
"id": "21185d7a-f0c1-49a0-9c2d-f0f198ceea7e",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [520, 180]
},
{
"parameters": {
"maxItems": 1
},
"id": "7cc02cc4-1f5f-489a-81e2-4c96b3bdf221",
"name": "Item Lists limit first",
"type": "n8n-nodes-base.limit",
"typeVersion": 1,
"position": [740, 80]
},
{
"parameters": {
"keep": "lastItems",
"maxItems": 1
},
"id": "2bf79d53-7a0b-4716-aa09-55ad43d306ae",
"name": "Item Lists limit last",
"type": "n8n-nodes-base.limit",
"typeVersion": 1,
"position": [740, 300]
}
],
"pinData": {
"Item Lists limit first": [
{
"json": {
"entry": 1
}
}
],
"Item Lists limit last": [
{
"json": {
"entry": 5
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Item Lists limit first",
"type": "main",
"index": 0
},
{
"node": "Item Lists limit last",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "5036d554-1ba4-4b5f-ba9f-1de6df09e807",
"id": "105",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
@@ -0,0 +1,29 @@
{
"node": "n8n-nodes-base.removeDuplicates",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/"
}
],
"generic": []
},
"alias": [
"Dedupe",
"Deduplicate",
"Duplicates",
"Remove",
"Unique",
"Transform",
"Array",
"List",
"Item"
],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,25 @@
import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow';
import { VersionedNodeType } from 'n8n-workflow';
import { RemoveDuplicatesV1 } from './v1/RemoveDuplicatesV1.node';
import { RemoveDuplicatesV2 } from './v2/RemoveDuplicatesV2.node';
export class RemoveDuplicates extends VersionedNodeType {
constructor() {
const baseDescription: INodeTypeBaseDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
defaultVersion: 2,
description: 'Delete items with matching field values',
};
const nodeVersions: IVersionedNodeType['nodeVersions'] = {
1: new RemoveDuplicatesV1(baseDescription),
1.1: new RemoveDuplicatesV1(baseDescription),
2: new RemoveDuplicatesV2(baseDescription),
};
super(nodeVersions, baseDescription);
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#54B8C9" clip-path="url(#a)"><path d="M134.097 111h38.829v32.508H138.16v34.635h-32.508v-38.699c0-15.709 12.735-28.444 28.445-28.444m77.658 32.508V111h77.657v32.508zm116.486 0V111h77.658v32.508zm116.487 0V111h38.829c15.71 0 28.445 12.735 28.445 28.444v38.699h-32.508v-34.635zm34.766 73.238h32.508v38.698c0 15.71-12.735 28.445-28.445 28.445h-38.829v-32.508h34.766zM0 244.537C0 229.329 12.735 217 28.444 217h349.461c15.709 0 28.444 12.329 28.444 27.537v129.815c0 15.208-12.735 27.537-28.444 27.537H28.445C12.734 401.889 0 389.56 0 374.352z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 715 B

@@ -0,0 +1,65 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import type { INode } from 'n8n-workflow';
import { validateInputData } from '../utils';
describe('Test Remove Duplicates Node', () => {
new NodeTestHarness().setupTests();
});
describe('Test Remove Duplicates Node, validateInputData util', () => {
test('Should throw error for version 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).toThrow("'country' isn't always the same type");
});
test('Should ignore null values and not throw error for version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).not.toThrow();
});
test('Should throw error for different types, version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[{ json: { id: 1 } }, { json: { id: '1' } }, { json: { id: 2 } }, { json: { id: null } }],
['id'],
false,
),
).toThrow("'id' isn't always the same type");
});
});
@@ -0,0 +1,316 @@
{
"name": "Remove Duplicates",
"nodes": [
{
"parameters": {},
"id": "a4da10da-991f-48ab-b873-9d633a11311f",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [760, 420]
},
{
"parameters": {
"data": [
{
"id": 1,
"name": "John Doe",
"age": 18
},
{
"id": 1,
"name": "John Doe",
"age": 18
},
{
"id": 1,
"name": "John Doe",
"age": 98
},
{
"id": 3,
"name": "Bob Johnson",
"age": 34
}
]
},
"id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [980, 420]
},
{
"parameters": {},
"id": "c336939c-062e-475e-ba7c-8601e3662e8c",
"name": "Remove Duplicates (All Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 260]
},
{
"parameters": {
"compare": "selectedFields",
"fieldsToCompare": "name",
"options": {}
},
"id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80",
"name": "Remove Duplicates (Selected Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 420]
},
{
"parameters": {
"compare": "allFieldsExcept",
"fieldsToExclude": "age",
"options": {}
},
"id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b",
"name": "Remove Duplicates (Except Fields)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 580]
},
{
"parameters": {},
"id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba",
"name": "All Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 260]
},
{
"parameters": {},
"id": "b5c5c946-2e96-451b-b9a6-78e478504d6c",
"name": "Selected Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 420]
},
{
"parameters": {},
"id": "afb92bc5-beba-4b0a-aefb-b47cc708a125",
"name": "Except Fields",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 580]
},
{
"parameters": {
"compare": "allFieldsExcept",
"fieldsToExclude": "age",
"options": {
"removeOtherFields": true
}
},
"id": "f92c5533-ac29-476c-aebb-4849ddd22110",
"name": "Remove Duplicates (Remove)",
"type": "n8n-nodes-base.removeDuplicates",
"typeVersion": 1,
"position": [1200, 760]
},
{
"parameters": {},
"id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89",
"name": "Remove",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 760]
}
],
"pinData": {
"Code": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 98
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"All Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 1,
"name": "John Doe",
"age": 98
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Selected Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Except Fields": [
{
"json": {
"id": 1,
"name": "John Doe",
"age": 18
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson",
"age": 34
}
}
],
"Remove": [
{
"json": {
"id": 1,
"name": "John Doe"
}
},
{
"json": {
"id": 3,
"name": "Bob Johnson"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Remove Duplicates (All Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Selected Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Except Fields)",
"type": "main",
"index": 0
},
{
"node": "Remove Duplicates (Remove)",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (All Fields)": {
"main": [
[
{
"node": "All Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Selected Fields)": {
"main": [
[
{
"node": "Selected Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Except Fields)": {
"main": [
[
{
"node": "Except Fields",
"type": "main",
"index": 0
}
]
]
},
"Remove Duplicates (Remove)": {
"main": [
[
{
"node": "Remove",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {
"executionOrder": "v1"
},
"versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3",
"id": "74gMYOHjjPArZg4q",
"meta": {
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
},
"tags": []
}
@@ -0,0 +1,170 @@
import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import lt from 'lodash/lt';
import pick from 'lodash/pick';
import { NodeOperationError } from 'n8n-workflow';
import type { IExecuteFunctions, INode, INodeExecutionData } from 'n8n-workflow';
import { compareItems, flattenKeys } from '@utils/utilities';
import { prepareFieldsArray } from '../utils/utils';
export const validateInputData = (
node: INode,
items: INodeExecutionData[],
keysToCompare: string[],
disableDotNotation: boolean,
) => {
for (const key of keysToCompare) {
let type: any = undefined;
for (const [i, item] of items.entries()) {
if (key === '') {
throw new NodeOperationError(node, 'Name of field to compare is blank');
}
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
if (value === null && node.typeVersion > 1) continue;
if (value === undefined && disableDotNotation && key.includes('.')) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
description:
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
});
} else if (value === undefined) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
}
if (type !== undefined && value !== undefined && type !== typeof value) {
const description =
'The type of this field varies between items' +
(node.typeVersion > 1
? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
: '');
throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
description,
});
} else {
type = typeof value;
}
}
}
};
export function removeDuplicateInputItems(context: IExecuteFunctions, items: INodeExecutionData[]) {
const compare = context.getNodeParameter('compare', 0) as string;
const disableDotNotation = context.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
const removeOtherFields = context.getNodeParameter(
'options.removeOtherFields',
0,
false,
) as boolean;
let keys = disableDotNotation
? Object.keys(items[0].json)
: Object.keys(flattenKeys(items[0].json));
for (const item of items) {
const itemKeys = disableDotNotation
? Object.keys(item.json)
: Object.keys(flattenKeys(item.json));
for (const key of itemKeys) {
if (!keys.includes(key)) {
keys.push(key);
}
}
}
if (compare === 'allFieldsExcept') {
const fieldsToExclude = prepareFieldsArray(
context.getNodeParameter('fieldsToExclude', 0, '') as string,
'Fields To Exclude',
);
if (!fieldsToExclude.length) {
throw new NodeOperationError(
context.getNode(),
'No fields specified. Please add a field to exclude from comparison',
);
}
if (!disableDotNotation) {
keys = Object.keys(flattenKeys(items[0].json));
}
keys = keys.filter((key) => !fieldsToExclude.includes(key));
}
if (compare === 'selectedFields') {
const fieldsToCompare = prepareFieldsArray(
context.getNodeParameter('fieldsToCompare', 0, '') as string,
'Fields To Compare',
);
if (!fieldsToCompare.length) {
throw new NodeOperationError(
context.getNode(),
'No fields specified. Please add a field to compare on',
);
}
if (!disableDotNotation) {
keys = Object.keys(flattenKeys(items[0].json));
}
keys = fieldsToCompare.map((key) => key.trim());
}
// This solution is O(nlogn)
// add original index to the items
const newItems = items.map(
(item, index) =>
({
json: { ...item.json, __INDEX: index },
pairedItem: { item: index },
}) as INodeExecutionData,
);
//sort items using the compare keys
newItems.sort((a, b) => {
let result = 0;
for (const key of keys) {
let equal;
if (!disableDotNotation) {
equal = isEqual(get(a.json, key), get(b.json, key));
} else {
equal = isEqual(a.json[key], b.json[key]);
}
if (!equal) {
let lessThan;
if (!disableDotNotation) {
lessThan = lt(get(a.json, key), get(b.json, key));
} else {
lessThan = lt(a.json[key], b.json[key]);
}
result = lessThan ? -1 : 1;
break;
}
}
return result;
});
validateInputData(context.getNode(), newItems, keys, disableDotNotation);
// collect the original indexes of items to be removed
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];
}
}
let updatedItems: INodeExecutionData[] = items.filter(
(_, index) => !removedIndexes.includes(index),
);
if (removeOtherFields) {
updatedItems = updatedItems.map((item, index) => ({
json: pick(item.json, ...keys),
pairedItem: { item: index },
}));
}
return [updatedItems];
}
@@ -0,0 +1,121 @@
import { NodeConnectionTypes } from 'n8n-workflow';
import type {
INodeTypeBaseDescription,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import { removeDuplicateInputItems } from '../utils';
const versionDescription: INodeTypeDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
subtitle: '',
version: [1, 1.1],
description: 'Delete items with matching field values',
defaults: {
name: 'Remove Duplicates',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Compare',
name: 'compare',
type: 'options',
options: [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
{
name: 'Selected Fields',
value: 'selectedFields',
},
],
default: 'allFields',
description: 'The fields of the input items to compare to see if they are the same',
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to exclude from the comparison',
default: '',
displayOptions: {
show: {
compare: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Compare',
name: 'fieldsToCompare',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to add to the comparison',
default: '',
displayOptions: {
show: {
compare: ['selectedFields'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
show: {
compare: ['allFieldsExcept', 'selectedFields'],
},
},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Remove Other Fields',
name: 'removeOtherFields',
type: 'boolean',
default: false,
description:
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
},
],
},
],
};
export class RemoveDuplicatesV1 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
return removeDuplicateInputItems(this, items);
}
}
@@ -0,0 +1,278 @@
import type { INodeProperties } from 'n8n-workflow';
const operationOptions = [
{
name: 'Remove Items Repeated Within Current Input',
value: 'removeDuplicateInputItems',
description: 'Remove duplicates from incoming items',
action: 'Remove items repeated within current input',
},
{
name: 'Remove Items Processed in Previous Executions',
value: 'removeItemsSeenInPreviousExecutions',
description: 'Deduplicate items already seen in previous executions',
action: 'Remove items processed in previous executions',
},
{
name: 'Clear Deduplication History',
value: 'clearDeduplicationHistory',
description: 'Wipe the store of previous items',
action: 'Clear deduplication history',
},
];
const compareOptions = [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
{
name: 'Selected Fields',
value: 'selectedFields',
},
];
const logicOptions = [
{
name: 'Value Is New',
value: 'removeItemsWithAlreadySeenKeyValues',
description: 'Remove all input items with values matching those already processed',
},
{
name: 'Value Is Higher than Any Previous Value',
value: 'removeItemsUpToStoredIncrementalKey',
description:
'Works with incremental values, removes all input items with values up to the stored value',
},
{
name: 'Value Is a Date Later than Any Previous Date',
value: 'removeItemsUpToStoredDate',
description:
'Works with date values, removes all input items with values up to the stored date',
},
];
const manageDatabaseModeOptions = [
{
name: 'Clean Database',
value: 'cleanDatabase',
description: 'Clear all values stored for a key in the database',
},
];
export const removeDuplicatesNodeFields: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: operationOptions,
default: 'removeDuplicateInputItems',
},
{
displayName: 'Compare',
name: 'compare',
type: 'options',
options: compareOptions,
default: 'allFields',
description: 'The fields of the input items to compare to see if they are the same',
displayOptions: {
show: {
operation: ['removeDuplicateInputItems'],
},
},
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to exclude from the comparison',
default: '',
displayOptions: {
show: {
compare: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Compare',
name: 'fieldsToCompare',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input to add to the comparison',
default: '',
displayOptions: {
show: {
compare: ['selectedFields'],
},
},
},
// ----------------------------------
{
displayName: 'Keep Items Where',
name: 'logic',
type: 'options',
noDataExpression: true,
options: logicOptions,
default: 'removeItemsWithAlreadySeenKeyValues',
description:
'How to select input items to remove by comparing them with key values previously processed',
displayOptions: {
show: {
operation: ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'dedupeValue',
type: 'string',
default: '',
description: 'Use an input field (or a combination of fields) that has a unique ID value',
hint: 'The input field value to compare between items',
placeholder: 'e.g. ID',
required: true,
displayOptions: {
show: {
logic: ['removeItemsWithAlreadySeenKeyValues'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'incrementalDedupeValue',
type: 'number',
default: '',
description: 'Use an input field (or a combination of fields) that has an incremental value',
hint: 'The input field value to compare between items, an incremental value is expected',
placeholder: 'e.g. ID',
displayOptions: {
show: {
logic: ['removeItemsUpToStoredIncrementalKey'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Value to Dedupe On',
name: 'dateDedupeValue',
type: 'dateTime',
default: '',
description: 'Use an input field that has a date value in ISO format',
hint: 'The input field value to compare between items, a date is expected',
placeholder: ' e.g. 2024-08-09T13:44:16Z',
displayOptions: {
show: {
logic: ['removeItemsUpToStoredDate'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
{
displayName: 'Mode',
name: 'mode',
type: 'options',
default: 'cleanDatabase',
description:
'How you want to modify the key values stored on the database. None of these modes removes input items.',
displayOptions: {
show: {
operation: ['clearDeduplicationHistory'],
},
},
options: manageDatabaseModeOptions,
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
show: {
operation: [
'removeDuplicateInputItems',
'removeItemsSeenInPreviousExecutions',
'clearDeduplicationHistory',
],
},
},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
displayOptions: {
show: {
'/operation': ['removeDuplicateInputItems'],
},
hide: {
'/compare': ['allFields'],
},
},
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Remove Other Fields',
name: 'removeOtherFields',
type: 'boolean',
default: false,
displayOptions: {
show: {
'/operation': ['removeDuplicateInputItems'],
},
hide: {
'/compare': ['allFields'],
},
},
description:
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
},
{
displayName: 'Scope',
name: 'scope',
type: 'options',
default: 'node',
displayOptions: {
show: {
'/operation': ['clearDeduplicationHistory', 'removeItemsSeenInPreviousExecutions'],
},
},
description:
'If set to workflow, key values will be shared across all nodes in the workflow. If set to node, key values will be specific to this node.',
options: [
{
name: 'Workflow',
value: 'workflow',
description: 'Deduplication info will be shared by all the nodes in the workflow',
},
{
name: 'Node',
value: 'node',
description: 'Deduplication info will be stored only for this node',
},
],
},
{
displayName: 'History Size',
name: 'historySize',
type: 'number',
default: 10000,
hint: 'The max number of past items to store for deduplication',
displayOptions: {
show: {
'/logic': ['removeItemsWithAlreadySeenKeyValues'],
'/operation': ['removeItemsSeenInPreviousExecutions'],
},
},
},
],
},
];
@@ -0,0 +1,271 @@
import { NodeConnectionTypes, NodeOperationError, tryToParseDateTime } from 'n8n-workflow';
import type {
INodeTypeBaseDescription,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
DeduplicationScope,
} from 'n8n-workflow';
import { removeDuplicatesNodeFields } from './RemoveDuplicatesV2.description';
import { removeDuplicateInputItems } from '../utils';
const versionDescription: INodeTypeDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
subtitle: '',
version: [2],
description: 'Delete items with matching field values',
defaults: {
name: 'Remove Duplicates',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
outputNames: ['Kept', 'Discarded'],
hints: [
{
message: 'The dedupe key set in “Value to Dedupe On” has no value',
displayCondition:
'={{ $parameter["operation"] === "removeItemsSeenInPreviousExecutions" && ($parameter["logic"] === "removeItemsWithAlreadySeenKeyValues" && $parameter["dedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredIncrementalKey" && $parameter["incrementalDedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredDate" && $parameter["dateDedupeValue"] === undefined) }}',
whenToDisplay: 'beforeExecution',
location: 'outputPane',
},
],
properties: [...removeDuplicatesNodeFields],
};
export class RemoveDuplicatesV2 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const operation = this.getNodeParameter('operation', 0);
const returnData: INodeExecutionData[][] = [];
const DEFAULT_MAX_ENTRIES = 10000;
try {
switch (operation) {
case 'removeDuplicateInputItems': {
return removeDuplicateInputItems(this, items);
}
case 'removeItemsSeenInPreviousExecutions': {
const logic = this.getNodeParameter('logic', 0);
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
if (logic === 'removeItemsWithAlreadySeenKeyValues') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let checkValue: string;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
checkValue = this.getNodeParameter('dedupeValue', itemIndex, '')?.toString() ?? '';
if (itemMapping[checkValue]) {
itemMapping[checkValue].push(items[itemIndex]);
} else {
itemMapping[checkValue] = [items[itemIndex]];
}
}
const maxEntries = this.getNodeParameter(
'options.historySize',
0,
DEFAULT_MAX_ENTRIES,
) as number;
const maxEntriesNum = Number(maxEntries);
const currentProcessedDataCount = await this.helpers.getProcessedDataCount(scope, {
mode: 'entries',
maxEntries,
});
if (currentProcessedDataCount + items.length > maxEntriesNum) {
throw new NodeOperationError(
this.getNode(),
'The number of items to be processed exceeds the maximum history size. Please increase the history size or reduce the number of items to be processed.',
);
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'entries', maxEntries },
);
const processedDataCount = await this.helpers.getProcessedDataCount(scope, {
mode: 'entries',
maxEntries,
});
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
if (maxEntriesNum > 0 && processedDataCount / maxEntriesNum > 0.5) {
this.addExecutionHints({
message: `Some duplicates may be not be removed since you're approaching the maximum history size (${maxEntriesNum} items). You can raise this limit using the history size option.`,
location: 'outputPane',
});
}
return returnData;
} else if (logic === 'removeItemsUpToStoredIncrementalKey') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let parsedIncrementalKey: number;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const incrementalKey = this.getNodeParameter('incrementalDedupeValue', itemIndex, '');
if (!incrementalKey?.toString()) {
throw new NodeOperationError(
this.getNode(),
'The `Value to Dedupe` On is empty. Please provide a value.',
);
}
parsedIncrementalKey = Number(incrementalKey);
if (isNaN(parsedIncrementalKey)) {
throw new NodeOperationError(
this.getNode(),
`The value '${incrementalKey}' is not a number. Please provide a number.`,
);
}
if (itemMapping[parsedIncrementalKey]) {
itemMapping[parsedIncrementalKey].push(items[itemIndex]);
} else {
itemMapping[parsedIncrementalKey] = [items[itemIndex]];
}
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'latestIncrementalKey' },
);
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
return returnData;
} else if (logic === 'removeItemsUpToStoredDate') {
if (!['node', 'workflow'].includes(scope)) {
throw new NodeOperationError(
this.getNode(),
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
);
}
let checkValue: string;
const itemMapping: {
[key: string]: INodeExecutionData[];
} = {};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
checkValue =
this.getNodeParameter('dateDedupeValue', itemIndex, '')?.toString() ?? '';
if (!checkValue) {
throw new NodeOperationError(
this.getNode(),
'The `Value to Dedupe` On is empty. Please provide a value.',
);
}
try {
tryToParseDateTime(checkValue);
} catch (error) {
throw new NodeOperationError(
this.getNode(),
`The value '${checkValue}' is not a valid date. Please provide a valid date.`,
);
}
if (itemMapping[checkValue]) {
itemMapping[checkValue].push(items[itemIndex]);
} else {
itemMapping[checkValue] = [items[itemIndex]];
}
}
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
Object.keys(itemMapping),
scope,
{ mode: 'latestDate' },
);
returnData.push(
itemsProcessed.new
.map((key) => {
return itemMapping[key];
})
.flat(),
itemsProcessed.processed
.map((key) => {
return itemMapping[key];
})
.flat(),
);
return returnData;
} else {
return [items];
}
}
case 'clearDeduplicationHistory': {
const mode = this.getNodeParameter('mode', 0) as string;
if (mode === 'updateKeyValuesInDatabase') {
} else if (mode === 'deleteKeyValuesFromDatabase') {
} else if (mode === 'cleanDatabase') {
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
await this.helpers.clearAllProcessedItems(scope, {
mode: 'entries',
});
}
return [items];
}
default: {
return [items];
}
}
} catch (error) {
if (this.continueOnFail()) {
returnData.push([{ json: this.getInputData(0)[0].json, error }]);
} else {
throw error;
}
}
return returnData;
}
}
@@ -0,0 +1,130 @@
import { mock } from 'jest-mock-extended';
import type { IExecuteFunctions, INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow';
import { RemoveDuplicatesV2 } from '../RemoveDuplicatesV2.node';
describe('RemoveDuplicatesV2', () => {
let node: RemoveDuplicatesV2;
let executeFunctions: IExecuteFunctions;
beforeEach(() => {
const baseDescription: INodeTypeBaseDescription = {
displayName: 'Remove Duplicates',
name: 'removeDuplicates',
icon: 'file:removeDuplicates.svg',
group: ['transform'],
description: 'Delete items with matching field values',
};
node = new RemoveDuplicatesV2(baseDescription);
executeFunctions = mock<IExecuteFunctions>();
executeFunctions.helpers = {
checkProcessedAndRecord: jest.fn(),
clearAllProcessedItems: jest.fn(),
} as any;
executeFunctions.getInputData = jest.fn();
executeFunctions.getNodeParameter = jest.fn();
});
it('should Remove items repeated within current input based on all fields', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 1, name: 'John' } },
];
(executeFunctions.getInputData as jest.Mock<any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'removeDuplicateInputItems';
if (paramName === 'compare') return 'allFields';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
it('should Remove items repeated within current input based on selected fields', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 1, name: 'Doe' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'removeDuplicateInputItems';
if (paramName === 'compare') return 'selectedFields';
if (paramName === 'fieldsToCompare') return 'id';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
it('should remove items seen in previous executions', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
{ json: { id: 3, name: 'Doe' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string, itemIndex: number) => {
if (paramName === 'operation') return 'removeItemsSeenInPreviousExecutions';
if (paramName === 'logic') return 'removeItemsWithAlreadySeenKeyValues';
if (paramName === 'dedupeValue' && itemIndex === 0) return 1;
if (paramName === 'dedupeValue' && itemIndex === 1) return 2;
if (paramName === 'dedupeValue' && itemIndex === 2) return 3;
if (paramName === 'options.scope') return 'node';
if (paramName === 'options.historySize') return 10;
},
);
executeFunctions.helpers.getProcessedDataCount = jest.fn().mockReturnValue(3);
(executeFunctions.helpers.checkProcessedAndRecord as jest.Mock).mockReturnValue({
new: [1, 3],
processed: [2],
});
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(2);
expect(result[0]).toHaveLength(2);
expect(result[1]).toHaveLength(1);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 3, name: 'Doe' });
});
it('should clean database when managing key values', async () => {
const items: INodeExecutionData[] = [
{ json: { id: 1, name: 'John' } },
{ json: { id: 2, name: 'Jane' } },
];
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
(paramName: string) => {
if (paramName === 'operation') return 'clearDeduplicationHistory';
if (paramName === 'mode') return 'cleanDatabase';
if (paramName === 'options.scope') return 'node';
return undefined;
},
);
const result = await node.execute.call(executeFunctions);
expect(result).toHaveLength(1);
expect(result[0]).toHaveLength(2);
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
});
});
@@ -0,0 +1,19 @@
{
"node": "n8n-nodes-base.sort",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.sort/"
}
],
"generic": []
},
"alias": ["Sort", "Order", "Transform", "Array", "List", "Item", "Random"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,289 @@
import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import lt from 'lodash/lt';
import {
NodeOperationError,
type IDataObject,
type IExecuteFunctions,
type INodeExecutionData,
type INodeType,
type INodeTypeDescription,
NodeConnectionTypes,
} from 'n8n-workflow';
import { shuffleArray } from '@utils/utilities';
import { sortByCode } from './utils';
export class Sort implements INodeType {
description: INodeTypeDescription = {
displayName: 'Sort',
name: 'sort',
icon: 'file:sort.svg',
group: ['transform'],
subtitle: '',
version: 1,
description: 'Change items order',
defaults: {
name: 'Sort',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Type',
name: 'type',
type: 'options',
options: [
{
name: 'Simple',
value: 'simple',
},
{
name: 'Random',
value: 'random',
},
{
name: 'Code',
value: 'code',
},
],
default: 'simple',
description: 'The type of sorting to perform',
},
{
displayName: 'Fields To Sort By',
name: 'sortFieldsUi',
type: 'fixedCollection',
typeOptions: {
multipleValues: true,
},
placeholder: 'Add Field To Sort By',
options: [
{
displayName: '',
name: 'sortField',
values: [
{
displayName: 'Field Name',
name: 'fieldName',
type: 'string',
required: true,
default: '',
description: 'The field to sort by',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: ' Enter the field name as text',
requiresDataPath: 'single',
},
{
displayName: 'Order',
name: 'order',
type: 'options',
options: [
{
name: 'Ascending',
value: 'ascending',
},
{
name: 'Descending',
value: 'descending',
},
],
default: 'ascending',
description: 'The order to sort by',
},
],
},
],
default: {},
description: 'The fields of the input items to sort by',
displayOptions: {
show: {
type: ['simple'],
},
},
},
{
displayName: 'Code',
name: 'code',
type: 'string',
typeOptions: {
alwaysOpenEditWindow: true,
editor: 'jsEditor',
rows: 10,
},
default: `// The two items to compare are in the variables a and b
// Access the fields in a.json and b.json
// Return -1 if a should go before b
// Return 1 if b should go before a
// Return 0 if there's no difference
fieldName = 'myField';
if (a.json[fieldName] < b.json[fieldName]) {
return -1;
}
if (a.json[fieldName] > b.json[fieldName]) {
return 1;
}
return 0;`,
description: 'Javascript code to determine the order of any two items',
displayOptions: {
show: {
type: ['code'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
show: {
type: ['simple'],
},
},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
let returnData = [...items];
const type = this.getNodeParameter('type', 0) as string;
const disableDotNotation = this.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
if (type === 'random') {
shuffleArray(returnData);
return [returnData];
}
if (type === 'simple') {
const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject;
const sortFields = sortFieldsUi.sortField as Array<{
fieldName: string;
order: 'ascending' | 'descending';
}>;
if (!sortFields?.length) {
throw new NodeOperationError(
this.getNode(),
'No sorting specified. Please add a field to sort by',
);
}
for (const { fieldName } of sortFields) {
let found = false;
for (const item of items) {
if (!disableDotNotation) {
if (get(item.json, fieldName) !== undefined) {
found = true;
}
} else if (item.json.hasOwnProperty(fieldName)) {
found = true;
}
}
if (!found && disableDotNotation && fieldName.includes('.')) {
throw new NodeOperationError(
this.getNode(),
`Couldn't find the field '${fieldName}' in the input data`,
{
description:
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
},
);
} else if (!found) {
throw new NodeOperationError(
this.getNode(),
`Couldn't find the field '${fieldName}' in the input data`,
);
}
}
const sortFieldsWithDirection = sortFields.map((field) => ({
name: field.fieldName,
dir: field.order === 'ascending' ? 1 : -1,
}));
returnData.sort((a, b) => {
let result = 0;
for (const field of sortFieldsWithDirection) {
let equal;
if (!disableDotNotation) {
const _a =
typeof get(a.json, field.name) === 'string'
? (get(a.json, field.name) as string).toLowerCase()
: get(a.json, field.name);
const _b =
typeof get(b.json, field.name) === 'string'
? (get(b.json, field.name) as string).toLowerCase()
: get(b.json, field.name);
equal = isEqual(_a, _b);
} else {
const _a =
typeof a.json[field.name] === 'string'
? (a.json[field.name] as string).toLowerCase()
: a.json[field.name];
const _b =
typeof b.json[field.name] === 'string'
? (b.json[field.name] as string).toLowerCase()
: b.json[field.name];
equal = isEqual(_a, _b);
}
if (!equal) {
let lessThan;
if (!disableDotNotation) {
const _a =
typeof get(a.json, field.name) === 'string'
? (get(a.json, field.name) as string).toLowerCase()
: get(a.json, field.name);
const _b =
typeof get(b.json, field.name) === 'string'
? (get(b.json, field.name) as string).toLowerCase()
: get(b.json, field.name);
lessThan = lt(_a, _b);
} else {
const _a =
typeof a.json[field.name] === 'string'
? (a.json[field.name] as string).toLowerCase()
: a.json[field.name];
const _b =
typeof b.json[field.name] === 'string'
? (b.json[field.name] as string).toLowerCase()
: b.json[field.name];
lessThan = lt(_a, _b);
}
if (lessThan) {
result = -1 * field.dir;
} else {
result = 1 * field.dir;
}
break;
}
}
return result;
});
} else {
returnData = await sortByCode.call(this);
}
return [returnData];
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><path fill="#8287EB" fill-rule="evenodd" d="M130.5 422.5c-6.627 0-12-5.373-12-12v-351c0-6.627 5.373-12 12-12h24c6.627 0 12 5.373 12 12v351c0 6.627-5.373 12-12 12z" clip-rule="evenodd"/><path fill="#8287EB" fill-rule="evenodd" d="M36.077 333.482c9.398-9.346 24.594-9.304 33.94.095l72.483 72.887 72.482-72.887c9.347-9.399 24.543-9.441 33.941-.095s9.441 24.543.095 33.941l-89.5 90a24 24 0 0 1-34.036 0l-89.5-90c-9.346-9.398-9.304-24.594.095-33.941M381.5 89.5c6.627 0 12 5.373 12 12v351c0 6.627-5.373 12-12 12h-24c-6.627 0-12-5.373-12-12v-351c0-6.627 5.373-12 12-12z" clip-rule="evenodd"/><path fill="#8287EB" fill-rule="evenodd" d="M475.923 178.518c-9.398 9.346-24.594 9.304-33.941-.095L369.5 105.536l-72.482 72.887c-9.347 9.399-24.543 9.441-33.941.095s-9.441-24.543-.095-33.941l89.5-90a24 24 0 0 1 34.036 0l89.5 90c9.346 9.398 9.304 24.594-.095 33.941" clip-rule="evenodd"/></svg>

After

Width:  |  Height:  |  Size: 955 B

@@ -0,0 +1,25 @@
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { JsTaskRunnerSandbox } from '../../Code/JsTaskRunnerSandbox';
const returnRegExp = /\breturn\b/;
export async function sortByCode(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
const userCode = this.getNodeParameter('code', 0) as string;
if (!returnRegExp.test(userCode)) {
throw new NodeOperationError(
this.getNode(),
"Sort code doesn't return. Please add a 'return' statement to your code",
);
}
const mode = this.getMode();
const items = this.getInputData();
const code = `return items.sort((a, b) => { ${userCode} })`;
const chunkSize = undefined;
const sandbox = new JsTaskRunnerSandbox(mode, this, chunkSize, { items });
const executionResult = await sandbox.runCode<INodeExecutionData[]>(code);
return executionResult;
}
@@ -0,0 +1,19 @@
{
"node": "n8n-nodes-base.splitOut",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.splitout/"
}
],
"generic": []
},
"alias": ["Split", "Nested", "Transform", "Array", "List", "Item"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,282 @@
import get from 'lodash/get';
import unset from 'lodash/unset';
import { NodeOperationError, deepCopy, NodeConnectionTypes } from 'n8n-workflow';
import type {
IBinaryData,
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import { prepareFieldsArray } from '../utils/utils';
import { FieldsTracker } from './utils';
export class SplitOut implements INodeType {
description: INodeTypeDescription = {
displayName: 'Split Out',
name: 'splitOut',
icon: 'file:splitOut.svg',
group: ['transform'],
subtitle: '',
version: 1,
description: 'Turn a list inside item(s) into separate items',
defaults: {
name: 'Split Out',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
builderHint: {
relatedNodes: [
{
nodeType: 'n8n-nodes-base.aggregate',
relationHint: 'Reverse operation - combine items back',
},
],
},
properties: [
{
displayName: 'Fields To Split Out',
name: 'fieldToSplitOut',
type: 'string',
default: '',
required: true,
placeholder: 'Drag fields from the left or type their names',
description:
'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.',
requiresDataPath: 'multiple',
hint: 'Use $binary to split out the input item by binary data',
},
{
displayName: 'Include',
name: 'include',
type: 'options',
options: [
{
name: 'No Other Fields',
value: 'noOtherFields',
},
{
name: 'All Other Fields',
value: 'allOtherFields',
},
{
name: 'Selected Other Fields',
value: 'selectedOtherFields',
},
],
default: 'noOtherFields',
description: 'Whether to copy any other fields into the new items',
},
{
displayName: 'Fields To Include',
name: 'fieldsToInclude',
type: 'string',
placeholder: 'e.g. email, name',
requiresDataPath: 'multiple',
description: 'Fields in the input items to aggregate together',
default: '',
displayOptions: {
show: {
include: ['selectedOtherFields'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Destination Field Name',
name: 'destinationFieldName',
type: 'string',
requiresDataPath: 'multiple',
default: '',
description: 'The field in the output under which to put the split field contents',
},
{
displayName: 'Include Binary',
name: 'includeBinary',
type: 'boolean',
default: false,
description: 'Whether to include the binary data in the new items',
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const returnData: INodeExecutionData[] = [];
const items = this.getInputData();
const fieldsTracker = new FieldsTracker();
for (let i = 0; i < items.length; i++) {
const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string)
.split(',')
.map((field) => field.trim().replace(/^\$json\./, ''));
const options = this.getNodeParameter('options', i, {});
const disableDotNotation = options.disableDotNotation as boolean;
const destinationFields = ((options.destinationFieldName as string) || '')
.split(',')
.filter((field) => field.trim() !== '')
.map((field) => field.trim());
if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) {
throw new NodeOperationError(
this.getNode(),
'If multiple fields to split out are given, the same number of destination fields must be given',
);
}
const include = this.getNodeParameter('include', i) as
| 'selectedOtherFields'
| 'allOtherFields'
| 'noOtherFields';
const multiSplit = fieldsToSplitOut.length > 1;
const item = { ...items[i].json };
const splited: INodeExecutionData[] = [];
for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) {
const destinationFieldName = destinationFields[entryIndex] || '';
let entityToSplit: IDataObject[] = [];
if (fieldToSplitOut === '$binary') {
entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({
[key]: value,
}));
} else {
if (!disableDotNotation) {
entityToSplit = get(item, fieldToSplitOut) as IDataObject[];
} else {
entityToSplit = item[fieldToSplitOut] as IDataObject[];
}
fieldsTracker.add(fieldToSplitOut);
const entryExists = entityToSplit !== undefined;
if (!entryExists) {
entityToSplit = [];
}
fieldsTracker.update(fieldToSplitOut, entryExists);
if (typeof entityToSplit !== 'object' || entityToSplit === null) {
entityToSplit = [entityToSplit] as unknown as IDataObject[];
}
if (!Array.isArray(entityToSplit)) {
entityToSplit = Object.values(entityToSplit);
}
}
for (const [elementIndex, element] of entityToSplit.entries()) {
if (splited[elementIndex] === undefined) {
splited[elementIndex] = { json: {}, pairedItem: { item: i } };
}
const fieldName = destinationFieldName || fieldToSplitOut;
if (fieldToSplitOut === '$binary') {
if (splited[elementIndex].binary === undefined) {
splited[elementIndex].binary = {};
}
splited[elementIndex].binary[Object.keys(element)[0]] = Object.values(
element,
)[0] as IBinaryData;
continue;
}
if (typeof element === 'object' && element !== null && include === 'noOtherFields') {
if (destinationFieldName === '' && !multiSplit) {
splited[elementIndex] = {
json: { ...splited[elementIndex].json, ...element },
pairedItem: { item: i },
};
} else {
splited[elementIndex].json[fieldName] = element;
}
} else {
splited[elementIndex].json[fieldName] = element;
}
}
}
for (const splitEntry of splited) {
let newItem: INodeExecutionData = splitEntry;
if (include === 'allOtherFields') {
const itemCopy = deepCopy(item);
for (const fieldToSplitOut of fieldsToSplitOut) {
if (!disableDotNotation) {
unset(itemCopy, fieldToSplitOut);
} else {
delete itemCopy[fieldToSplitOut];
}
}
newItem.json = { ...itemCopy, ...splitEntry.json };
}
if (include === 'selectedOtherFields') {
const fieldsToInclude = prepareFieldsArray(
this.getNodeParameter('fieldsToInclude', i, '') as string,
'Fields To Include',
);
if (!fieldsToInclude.length) {
throw new NodeOperationError(this.getNode(), 'No fields specified', {
description: 'Please add a field to include',
});
}
for (const field of fieldsToInclude) {
if (!disableDotNotation) {
splitEntry.json[field] = get(item, field);
} else {
splitEntry.json[field] = item[field];
}
}
newItem = splitEntry;
}
const includeBinary = options.includeBinary as boolean;
if (includeBinary) {
if (items[i].binary && !newItem.binary) {
newItem.binary = items[i].binary;
}
}
returnData.push(newItem);
}
}
const hints = fieldsTracker.getHints();
if (hints.length) {
this.addExecutionHints(...hints);
}
return [returnData];
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#9B6DD5" clip-path="url(#a)"><path fill-rule="evenodd" d="M480 148c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12zm0 96c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12zm0 96c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12z" clip-rule="evenodd"/><path d="M438 76c0 6.627-5.373 12-12 12H309.783c-17.673 0-32 14.327-32 32v56c0 26.978-10.272 51.557-27.119 70.039-5.055 5.545-5.055 14.377 0 19.922 16.847 18.482 27.119 43.061 27.119 70.039v56c0 17.673 14.327 32 32 32H426c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H309.783c-44.183 0-80-35.817-80-80v-56c0-30.928-25.072-56-56-56a5.783 5.783 0 0 1-5.783-5.783v-36.434a5.783 5.783 0 0 1 5.783-5.783c30.928 0 56-25.072 56-56v-56c0-44.183 35.817-80 80-80H426c6.627 0 12 5.373 12 12z"/><path fill-rule="evenodd" d="M136 244c0-6.627-5.373-12-12-12H12c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h112c6.627 0 12-5.373 12-12z" clip-rule="evenodd"/></g><defs><clipPath id="a"><path fill="#fff" d="M512 0H0v512h512z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 1.2 KiB

@@ -0,0 +1,5 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
describe('Test Split Out Node', () => {
new NodeTestHarness().setupTests();
});
@@ -0,0 +1,68 @@
import { FieldsTracker } from '../utils';
describe('FieldsTracker', () => {
let fieldsTracker: FieldsTracker;
beforeEach(() => {
fieldsTracker = new FieldsTracker();
});
describe('add', () => {
it('should add field with false value', () => {
fieldsTracker.add('testField');
expect(fieldsTracker.fields.testField).toBe(false);
});
it('should not overwrite existing field', () => {
fieldsTracker.add('testField');
fieldsTracker.fields.testField = true;
fieldsTracker.add('testField');
expect(fieldsTracker.fields.testField).toBe(true);
});
});
describe('update', () => {
it('should update field from false to true', () => {
fieldsTracker.add('testField');
fieldsTracker.update('testField', true);
expect(fieldsTracker.fields.testField).toBe(true);
});
it('should not update field from true to false', () => {
fieldsTracker.add('testField');
fieldsTracker.fields.testField = true;
fieldsTracker.update('testField', false);
expect(fieldsTracker.fields.testField).toBe(true);
});
});
describe('getHints', () => {
it('should return empty array when no fields tracked', () => {
expect(fieldsTracker.getHints()).toEqual([]);
});
it('should return hint for missing field', () => {
fieldsTracker.add('missingField');
const hints = fieldsTracker.getHints();
expect(hints).toEqual([
{
message: "The field 'missingField' wasn't found in any input item",
location: 'outputPane',
},
]);
});
it('should not return hint for found field', () => {
fieldsTracker.add('foundField');
fieldsTracker.update('foundField', true);
expect(fieldsTracker.getHints()).toEqual([]);
});
});
});
@@ -0,0 +1,388 @@
{
"name": "splitOut test",
"nodes": [
{
"parameters": {},
"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-400, 400]
},
{
"parameters": {
"data": {
"data": [
{
"id": 3,
"char": "c"
},
{
"id": 4,
"char": "d"
},
{
"id": 5,
"char": "e"
},
{
"id": 1,
"char": "a"
},
{
"id": 2,
"char": "b"
}
],
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
]
}
},
"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [-180, 400]
},
{
"parameters": {
"fieldToSplitOut": "data",
"options": {}
},
"id": "e7eac465-8fe6-498c-9942-ebd47df537c1",
"name": "Item Lists",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [80, 160]
},
{
"parameters": {
"fieldToSplitOut": "data",
"include": "allOtherFields",
"options": {}
},
"id": "09b7fe15-dbad-4ca6-bf1e-3093139d14e5",
"name": "Item Lists1",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [80, 320]
},
{
"parameters": {
"fieldToSplitOut": "data",
"include": "selectedOtherFields",
"fieldsToInclude": ["data3"],
"options": {}
},
"id": "7ea63dc7-8141-4233-af47-9894919c7fe4",
"name": "Item Lists2",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [80, 480]
},
{
"parameters": {
"fieldToSplitOut": "data",
"options": {
"destinationFieldName": "output"
}
},
"id": "89c3c1b4-9577-480a-931f-3b34450b23cb",
"name": "Item Lists3",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [80, 660]
}
],
"pinData": {
"Item Lists": [
{
"json": {
"id": 3,
"char": "c"
}
},
{
"json": {
"id": 4,
"char": "d"
}
},
{
"json": {
"id": 5,
"char": "e"
}
},
{
"json": {
"id": 1,
"char": "a"
}
},
{
"json": {
"id": 2,
"char": "b"
}
}
],
"Item Lists1": [
{
"json": {
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 3,
"char": "c"
}
}
},
{
"json": {
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 4,
"char": "d"
}
}
},
{
"json": {
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 5,
"char": "e"
}
}
},
{
"json": {
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 1,
"char": "a"
}
}
},
{
"json": {
"data2": [
{
"text": "foo"
}
],
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 2,
"char": "b"
}
}
}
],
"Item Lists2": [
{
"json": {
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 3,
"char": "c"
}
}
},
{
"json": {
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 4,
"char": "d"
}
}
},
{
"json": {
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 5,
"char": "e"
}
}
},
{
"json": {
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 1,
"char": "a"
}
}
},
{
"json": {
"data3": [
{
"text": "bar"
}
],
"data": {
"id": 2,
"char": "b"
}
}
}
],
"Item Lists3": [
{
"json": {
"output": {
"id": 3,
"char": "c"
}
}
},
{
"json": {
"output": {
"id": 4,
"char": "d"
}
}
},
{
"json": {
"output": {
"id": 5,
"char": "e"
}
}
},
{
"json": {
"output": {
"id": 1,
"char": "a"
}
}
},
{
"json": {
"output": {
"id": 2,
"char": "b"
}
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Item Lists",
"type": "main",
"index": 0
},
{
"node": "Item Lists1",
"type": "main",
"index": 0
},
{
"node": "Item Lists2",
"type": "main",
"index": 0
},
{
"node": "Item Lists3",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "9230f580-6f41-47c9-9949-bf258fc3fa47",
"id": "105",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
@@ -0,0 +1,409 @@
{
"name": "itemList split Object",
"nodes": [
{
"parameters": {},
"id": "ade46a75-ab57-48c6-886b-0c118f5ef1c6",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [520, 800]
},
{
"parameters": {
"fieldToSplitOut": "data",
"include": "selectedOtherFields",
"fieldsToInclude": ["tag"],
"options": {}
},
"id": "45e1d7a3-d6e8-4b69-a68a-1038db13be4c",
"name": "Item Lists1",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [1120, 340]
},
{
"parameters": {
"data": {
"data": {
"entry1": {
"id": 1,
"info": "some info 1"
},
"entry2": {
"id": 2,
"info": "some info 2"
},
"entry3": {
"id": 3,
"info": "some info 3"
}
},
"data2": ["a", "b", "c"],
"data3": {
"a": 1,
"b": 2,
"c": 3
},
"data4": null,
"tag": "bar"
}
},
"id": "faa78fac-468d-42b8-96e9-0fb62c312da3",
"name": "Code1",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [760, 800]
},
{
"parameters": {},
"id": "5baaf321-7e89-473d-a313-7cb90b3f13b3",
"name": "No Operation, do nothing",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 340]
},
{
"parameters": {
"fieldToSplitOut": "data3",
"include": "allOtherFields",
"options": {
"destinationFieldName": "extracted"
}
},
"id": "a786bea9-eb29-4c6d-aea6-a22aee622bc6",
"name": "Item Lists",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [1120, 720]
},
{
"parameters": {},
"id": "0521a24b-c74a-48fa-ae50-48a242b97806",
"name": "No Operation, do nothing1",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 720]
},
{
"parameters": {
"fieldToSplitOut": "data3",
"options": {}
},
"id": "0c1c8827-72ab-4738-918c-d529e66505c6",
"name": "Item Lists2",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [1120, 540]
},
{
"parameters": {},
"id": "4c0dca36-c2ae-4d40-8952-0e728ac93fa3",
"name": "No Operation, do nothing2",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 540]
},
{
"parameters": {
"fieldToSplitOut": "data2",
"options": {}
},
"id": "b2031380-b2a8-426d-8f7a-ab072d23b979",
"name": "Item Lists3",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [1120, 920]
},
{
"parameters": {},
"id": "617f7259-beee-42f1-bba2-4e75a83fe369",
"name": "No Operation, do nothing3",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 920]
},
{
"parameters": {
"fieldToSplitOut": "data4",
"include": "allOtherFields",
"options": {}
},
"id": "8909b8eb-e5a9-4436-8e62-09d8c9670ac1",
"name": "Item Lists4",
"type": "n8n-nodes-base.splitOut",
"typeVersion": 1,
"position": [1120, 1140],
"continueOnFail": true
},
{
"parameters": {},
"id": "a9278f90-8ad9-42dc-85b6-28bf1b6764b7",
"name": "No Operation, do nothing4",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [1380, 1140]
}
],
"pinData": {
"No Operation, do nothing": [
{
"json": {
"tag": "bar",
"data": {
"id": 1,
"info": "some info 1"
}
}
},
{
"json": {
"tag": "bar",
"data": {
"id": 2,
"info": "some info 2"
}
}
},
{
"json": {
"tag": "bar",
"data": {
"id": 3,
"info": "some info 3"
}
}
}
],
"No Operation, do nothing2": [
{
"json": {
"data3": 1
}
},
{
"json": {
"data3": 2
}
},
{
"json": {
"data3": 3
}
}
],
"No Operation, do nothing1": [
{
"json": {
"data": {
"entry1": {
"id": 1,
"info": "some info 1"
},
"entry2": {
"id": 2,
"info": "some info 2"
},
"entry3": {
"id": 3,
"info": "some info 3"
}
},
"data2": ["a", "b", "c"],
"data4": null,
"tag": "bar",
"extracted": 1
}
},
{
"json": {
"data": {
"entry1": {
"id": 1,
"info": "some info 1"
},
"entry2": {
"id": 2,
"info": "some info 2"
},
"entry3": {
"id": 3,
"info": "some info 3"
}
},
"data2": ["a", "b", "c"],
"data4": null,
"tag": "bar",
"extracted": 2
}
},
{
"json": {
"data": {
"entry1": {
"id": 1,
"info": "some info 1"
},
"entry2": {
"id": 2,
"info": "some info 2"
},
"entry3": {
"id": 3,
"info": "some info 3"
}
},
"data2": ["a", "b", "c"],
"data4": null,
"tag": "bar",
"extracted": 3
}
}
],
"No Operation, do nothing3": [
{
"json": {
"data2": "a"
}
},
{
"json": {
"data2": "b"
}
},
{
"json": {
"data2": "c"
}
}
],
"No Operation, do nothing4": [
{
"json": {
"data": {
"entry1": {
"id": 1,
"info": "some info 1"
},
"entry2": {
"id": 2,
"info": "some info 2"
},
"entry3": {
"id": 3,
"info": "some info 3"
}
},
"data2": ["a", "b", "c"],
"data3": {
"a": 1,
"b": 2,
"c": 3
},
"data4": null,
"tag": "bar"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code1",
"type": "main",
"index": 0
}
]
]
},
"Code1": {
"main": [
[
{
"node": "Item Lists1",
"type": "main",
"index": 0
},
{
"node": "Item Lists2",
"type": "main",
"index": 0
},
{
"node": "Item Lists",
"type": "main",
"index": 0
},
{
"node": "Item Lists3",
"type": "main",
"index": 0
},
{
"node": "Item Lists4",
"type": "main",
"index": 0
}
]
]
},
"Item Lists1": {
"main": [
[
{
"node": "No Operation, do nothing",
"type": "main",
"index": 0
}
]
]
},
"Item Lists": {
"main": [
[
{
"node": "No Operation, do nothing1",
"type": "main",
"index": 0
}
]
]
},
"Item Lists2": {
"main": [
[
{
"node": "No Operation, do nothing2",
"type": "main",
"index": 0
}
]
]
},
"Item Lists3": {
"main": [
[
{
"node": "No Operation, do nothing3",
"type": "main",
"index": 0
}
]
]
},
"Item Lists4": {
"main": [
[
{
"node": "No Operation, do nothing4",
"type": "main",
"index": 0
}
]
]
}
},
"active": false
}
@@ -0,0 +1,32 @@
import type { NodeExecutionHint } from 'n8n-workflow';
export class FieldsTracker {
fields: { [key: string]: boolean } = {};
add(key: string) {
if (this.fields[key] === undefined) {
this.fields[key] = false;
}
}
update(key: string, value: boolean) {
if (!this.fields[key] && value) {
this.fields[key] = true;
}
}
getHints() {
const hints: NodeExecutionHint[] = [];
for (const [field, value] of Object.entries(this.fields)) {
if (!value) {
hints.push({
message: `The field '${field}' wasn't found in any input item`,
location: 'outputPane',
});
}
}
return hints;
}
}
@@ -0,0 +1,36 @@
{
"node": "n8n-nodes-base.summarize",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.summarize/"
}
],
"generic": []
},
"alias": [
"Append",
"Array",
"Average",
"Concatenate",
"Count",
"Group",
"Item",
"List",
"Max",
"Min",
"Pivot",
"Sum",
"Summarise",
"Summarize",
"Transform",
"Unique"
],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}
@@ -0,0 +1,379 @@
import {
type IExecuteFunctions,
type INodeExecutionData,
type INodeType,
type INodeTypeDescription,
NodeConnectionTypes,
type NodeExecutionHint,
NodeOperationError,
} from 'n8n-workflow';
import {
type Aggregations,
NUMERICAL_AGGREGATIONS,
type SummarizeOptions,
aggregateAndSplitData,
checkIfFieldExists,
fieldValueGetter,
flattenAggregationResultToArray,
flattenAggregationResultToObject,
} from './utils';
export class Summarize implements INodeType {
description: INodeTypeDescription = {
displayName: 'Summarize',
name: 'summarize',
icon: 'file:summarize.svg',
group: ['transform'],
subtitle: '',
version: [1, 1.1],
description: 'Sum, count, max, etc. across items',
defaults: {
name: 'Summarize',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Fields to Summarize',
name: 'fieldsToSummarize',
type: 'fixedCollection',
placeholder: 'Add Field',
default: { values: [{ aggregation: 'count', field: '' }] },
typeOptions: {
multipleValues: true,
},
options: [
{
displayName: '',
name: 'values',
values: [
{
displayName: 'Aggregation',
name: 'aggregation',
type: 'options',
options: [
{
name: 'Append',
value: 'append',
},
{
name: 'Average',
value: 'average',
},
{
name: 'Concatenate',
value: 'concatenate',
},
{
name: 'Count',
value: 'count',
},
{
name: 'Count Unique',
value: 'countUnique',
},
{
name: 'Max',
value: 'max',
},
{
name: 'Min',
value: 'min',
},
{
name: 'Sum',
value: 'sum',
},
],
default: 'count',
description: 'How to combine the values of the field you want to summarize',
},
//field repeated to have different descriptions for different aggregations --------------------------------
{
displayName: 'Field',
name: 'field',
type: 'string',
default: '',
description: 'The name of an input field that you want to summarize',
placeholder: 'e.g. cost',
hint: ' Enter the field name as text',
displayOptions: {
hide: {
aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'],
},
},
requiresDataPath: 'single',
},
{
displayName: 'Field',
name: 'field',
type: 'string',
default: '',
description:
'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.',
placeholder: 'e.g. cost',
hint: ' Enter the field name as text',
displayOptions: {
show: {
aggregation: NUMERICAL_AGGREGATIONS,
},
},
requiresDataPath: 'single',
},
{
displayName: 'Field',
name: 'field',
type: 'string',
default: '',
description:
'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored',
placeholder: 'e.g. cost',
hint: ' Enter the field name as text',
displayOptions: {
show: {
aggregation: ['countUnique', 'count', 'max', 'min'],
},
},
requiresDataPath: 'single',
},
// ----------------------------------------------------------------------------------------------------------
{
displayName: 'Include Empty Values',
name: 'includeEmpty',
type: 'boolean',
default: false,
displayOptions: {
show: {
aggregation: ['append', 'concatenate', 'count', 'countUnique'],
},
},
},
{
displayName: 'Separator',
name: 'separateBy',
type: 'options',
default: ',',
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
options: [
{
name: 'Comma',
value: ',',
},
{
name: 'Comma and Space',
value: ', ',
},
{
name: 'New Line',
value: '\n',
},
{
name: 'None',
value: '',
},
{
name: 'Space',
value: ' ',
},
{
name: 'Other',
value: 'other',
},
],
hint: 'What to insert between values',
displayOptions: {
show: {
aggregation: ['concatenate'],
},
},
},
{
displayName: 'Custom Separator',
name: 'customSeparator',
type: 'string',
default: '',
displayOptions: {
show: {
aggregation: ['concatenate'],
separateBy: ['other'],
},
},
},
],
},
],
},
// fieldsToSplitBy repeated to have different displayName for singleItem and separateItems -----------------------------
{
displayName: 'Fields to Split By',
name: 'fieldsToSplitBy',
type: 'string',
placeholder: 'e.g. country, city',
default: '',
description: 'The name of the input fields that you want to split the summary by',
hint: 'Enter the name of the fields as text (separated by commas)',
displayOptions: {
hide: {
'/options.outputFormat': ['singleItem'],
},
},
requiresDataPath: 'multiple',
},
{
displayName: 'Fields to Group By',
name: 'fieldsToSplitBy',
type: 'string',
placeholder: 'e.g. country, city',
default: '',
description: 'The name of the input fields that you want to split the summary by',
hint: 'Enter the name of the fields as text (separated by commas)',
displayOptions: {
show: {
'/options.outputFormat': ['singleItem'],
},
},
requiresDataPath: 'multiple',
},
// ----------------------------------------------------------------------------------------------------------
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add option',
default: {},
options: [
{
displayName: 'Continue if Field Not Found',
name: 'continueIfFieldNotFound',
type: 'boolean',
default: false,
description:
"Whether to continue if field to summarize can't be found in any items and return single empty item, otherwise an error would be thrown",
displayOptions: {
hide: {
'@version': [{ _cnd: { gte: 1.1 } }],
},
},
},
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Output Format',
name: 'outputFormat',
type: 'options',
default: 'separateItems',
options: [
{
name: 'Each Split in a Separate Item',
value: 'separateItems',
},
{
name: 'All Splits in a Single Item',
value: 'singleItem',
},
],
},
{
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
displayName: 'Ignore items without valid fields to group by',
name: 'skipEmptySplitFields',
type: 'boolean',
default: false,
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i }));
const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions;
const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string)
.split(',')
.map((field) => field.trim())
.filter((field) => field);
const fieldsToSummarize = this.getNodeParameter(
'fieldsToSummarize.values',
0,
[],
) as Aggregations;
if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) {
throw new NodeOperationError(
this.getNode(),
"You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'",
);
}
const getValue = fieldValueGetter(options.disableDotNotation);
const nodeVersion = this.getNode().typeVersion;
const aggregationResult = aggregateAndSplitData({
splitKeys: fieldsToSplitBy,
inputItems: newItems,
fieldsToSummarize,
options,
getValue,
convertKeysToString: nodeVersion === 1,
});
const fieldsNotFound: NodeExecutionHint[] = [];
try {
checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue);
} catch (error) {
if (nodeVersion > 1 || options.continueIfFieldNotFound) {
const fieldNotFoundHint: NodeExecutionHint = {
message: error instanceof Error ? error.message : String(error),
location: 'outputPane',
};
fieldsNotFound.push(fieldNotFoundHint);
} else {
throw error;
}
}
if (fieldsNotFound.length) {
this.addExecutionHints(...fieldsNotFound);
}
if (options.outputFormat === 'singleItem') {
const executionData: INodeExecutionData = {
json: flattenAggregationResultToObject(aggregationResult),
pairedItem: newItems.map((_v, index) => ({
item: index,
})),
};
return [[executionData]];
} else {
if (!fieldsToSplitBy.length && 'pairedItems' in aggregationResult) {
const { pairedItems, returnData } = aggregationResult;
const executionData: INodeExecutionData = {
json: returnData,
pairedItem: (pairedItems ?? []).map((index) => ({ item: index })),
};
return [[executionData]];
}
const flatAggregationResults = flattenAggregationResultToArray(aggregationResult);
const executionData = flatAggregationResults.map((item) => {
const { pairedItems, returnData } = item;
return {
json: returnData,
pairedItem: (pairedItems ?? []).map((index) => ({ item: index })),
};
});
return [executionData];
}
}
}
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><path fill="#F92" fill-rule="evenodd" d="M132 91c-15.464 0-28 12.536-28 28v317c0 15.464 12.536 28 28 28h248c15.464 0 28-12.536 28-28V119c0-15.464-12.536-28-28-28h-6a6 6 0 0 1-6-6V49a6 6 0 0 1 6-6h6c41.974 0 76 34.026 76 76v317c0 41.974-34.026 76-76 76H132c-41.974 0-76-34.026-76-76V119c0-41.974 34.026-76 76-76h6a6 6 0 0 1 6 6v36a6 6 0 0 1-6 6z" clip-rule="evenodd"/><path fill="#F92" fill-rule="evenodd" d="M256 0c-27.232 0-50.227 18.142-57.558 43H182a6 6 0 0 0-6 6v70a6 6 0 0 0 6 6h148a6 6 0 0 0 6-6V49a6 6 0 0 0-6-6h-16.442C306.227 18.142 283.232 0 256 0m0 40a19.9 19.9 0 0 0-10.541 3C239.781 46.528 236 52.823 236 60c0 11.046 8.954 20 20 20s20-8.954 20-20c0-7.177-3.781-13.472-9.459-17A19.9 19.9 0 0 0 256 40m101 179c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 207c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/><path fill="#F92" fill-rule="evenodd" d="M357 395c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 383c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/><path fill="#F92" fill-rule="evenodd" d="M357 307c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 295c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/></svg>

After

Width:  |  Height:  |  Size: 1.6 KiB

@@ -0,0 +1,5 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
describe('Test Summarize Node', () => {
new NodeTestHarness().setupTests();
});
@@ -0,0 +1,524 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is false: split-field-with-spaces-array 1`] = `
[
{
"pairedItems": [
0,
],
"returnData": {
"Product": "Widget A",
"Warehouse": "WH1",
"appended_Warehouse": [
"WH1",
],
},
},
{
"pairedItems": [
2,
],
"returnData": {
"Product": "Widget A",
"Warehouse": null,
"appended_Warehouse": [
null,
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Product": null,
"Warehouse": "WH2",
"appended_Warehouse": [
"WH2",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is false: split-field-with-spaces-result 1`] = `
{
"fieldName": "Product",
"splits": Map {
"Widget A" => {
"fieldName": "Warehouse",
"splits": Map {
"WH1" => {
"pairedItems": [
0,
],
"returnData": {
"appended_Warehouse": [
"WH1",
],
},
},
null => {
"pairedItems": [
2,
],
"returnData": {
"appended_Warehouse": [
null,
],
},
},
},
},
null => {
"fieldName": "Warehouse",
"splits": Map {
"WH2" => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"WH2",
],
},
},
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is true: split-field-with-spaces-array 1`] = `
[
{
"pairedItems": [
0,
],
"returnData": {
"Product": "Widget A",
"Warehouse": "WH1",
"appended_Warehouse": [
"WH1",
],
},
},
{
"pairedItems": [
2,
],
"returnData": {
"Product": "Widget A",
"Warehouse": "null",
"appended_Warehouse": [
null,
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Product": "null",
"Warehouse": "WH2",
"appended_Warehouse": [
"WH2",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is true: split-field-with-spaces-result 1`] = `
{
"fieldName": "Product",
"splits": Map {
"Widget A" => {
"fieldName": "Warehouse",
"splits": Map {
"WH1" => {
"pairedItems": [
0,
],
"returnData": {
"appended_Warehouse": [
"WH1",
],
},
},
"null" => {
"pairedItems": [
2,
],
"returnData": {
"appended_Warehouse": [
null,
],
},
},
},
},
"null" => {
"fieldName": "Warehouse",
"splits": Map {
"WH2" => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"WH2",
],
},
},
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is not set: split-field-with-spaces-array 1`] = `
[
{
"pairedItems": [
0,
2,
],
"returnData": {
"Product": "Widget A",
"appended_Warehouse": [
"WH1",
"WH3",
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Product": "Widget B",
"appended_Warehouse": [
"WH2",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is not set: split-field-with-spaces-result 1`] = `
{
"fieldName": "Product",
"splits": Map {
"Widget A" => {
"pairedItems": [
0,
2,
],
"returnData": {
"appended_Warehouse": [
"WH1",
"WH3",
],
},
},
"Widget B" => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"WH2",
],
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is true: split-field-with-spaces-array 1`] = `
[
{
"pairedItems": [
0,
2,
],
"returnData": {
"Product": "Widget A",
"appended_Warehouse": [
"WH1",
"WH3",
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Product": "Widget B",
"appended_Warehouse": [
"WH2",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is true: split-field-with-spaces-result 1`] = `
{
"fieldName": "Product",
"splits": Map {
"Widget A" => {
"pairedItems": [
0,
2,
],
"returnData": {
"appended_Warehouse": [
"WH1",
"WH3",
],
},
},
"Widget B" => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"WH2",
],
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData should not convert numbers to strings: array 1`] = `
[
{
"pairedItems": [
0,
],
"returnData": {
"Qty": 1,
"Sku": 12345,
"appended_Warehouse": [
"BER_0G",
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Qty": 2,
"Sku": 12345,
"appended_Warehouse": [
"BER_0L",
],
},
},
{
"pairedItems": [
2,
],
"returnData": {
"Qty": 1,
"Sku": 6534563534,
"appended_Warehouse": [
"BER_0L",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should not convert numbers to strings: result 1`] = `
{
"fieldName": "Sku",
"splits": Map {
12345 => {
"fieldName": "Qty",
"splits": Map {
1 => {
"pairedItems": [
0,
],
"returnData": {
"appended_Warehouse": [
"BER_0G",
],
},
},
2 => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"BER_0L",
],
},
},
},
},
6534563534 => {
"fieldName": "Qty",
"splits": Map {
1 => {
"pairedItems": [
2,
],
"returnData": {
"appended_Warehouse": [
"BER_0L",
],
},
},
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData should not convert strings to numbers: array 1`] = `
[
{
"pairedItems": [
0,
],
"returnData": {
"Qty": "1",
"Sku": "012345",
"appended_Warehouse": [
"BER_0G",
],
},
},
{
"pairedItems": [
1,
],
"returnData": {
"Qty": "2",
"Sku": "012345",
"appended_Warehouse": [
"BER_0L",
],
},
},
{
"pairedItems": [
2,
],
"returnData": {
"Qty": "1",
"Sku": "06534563534",
"appended_Warehouse": [
"BER_0L",
],
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData should not convert strings to numbers: result 1`] = `
{
"fieldName": "Sku",
"splits": Map {
"012345" => {
"fieldName": "Qty",
"splits": Map {
"1" => {
"pairedItems": [
0,
],
"returnData": {
"appended_Warehouse": [
"BER_0G",
],
},
},
"2" => {
"pairedItems": [
1,
],
"returnData": {
"appended_Warehouse": [
"BER_0L",
],
},
},
},
},
"06534563534" => {
"fieldName": "Qty",
"splits": Map {
"1" => {
"pairedItems": [
2,
],
"returnData": {
"appended_Warehouse": [
"BER_0L",
],
},
},
},
},
},
}
`;
exports[`Test Summarize Node, aggregateAndSplitData with skipEmptySplitFields=true should skip empty split fields: array 1`] = `
[
{
"pairedItems": [
0,
3,
],
"returnData": {
"Sku": 12345,
"concatenated_Warehouse": "BER_0G//{"name":"BER_0G3"}",
},
},
{
"pairedItems": [
2,
],
"returnData": {
"Sku": "{}",
"concatenated_Warehouse": "BER_0L",
},
},
]
`;
exports[`Test Summarize Node, aggregateAndSplitData with skipEmptySplitFields=true should skip empty split fields: result 1`] = `
{
"fieldName": "Sku",
"splits": Map {
12345 => {
"pairedItems": [
0,
3,
],
"returnData": {
"concatenated_Warehouse": "BER_0G//{"name":"BER_0G3"}",
},
},
"{}" => {
"pairedItems": [
2,
],
"returnData": {
"concatenated_Warehouse": "BER_0L",
},
},
},
}
`;
@@ -0,0 +1,53 @@
import { NodeOperationError, type IExecuteFunctions, type IDataObject } from 'n8n-workflow';
import { checkIfFieldExists, type Aggregations } from '../../utils';
describe('Test Summarize Node, checkIfFieldExists', () => {
let mockExecuteFunctions: IExecuteFunctions;
beforeEach(() => {
mockExecuteFunctions = {
getNode: jest.fn().mockReturnValue({ name: 'test-node' }),
} as unknown as IExecuteFunctions;
});
const items = [{ a: 1 }, { b: 2 }, { c: 3 }];
it('should not throw error if all fields exist', () => {
const aggregations: Aggregations = [
{ aggregation: 'sum', field: 'a' },
{ aggregation: 'count', field: 'c' },
];
const getValue = (item: IDataObject, field: string) => item[field];
expect(() => {
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
}).not.toThrow();
});
it('should throw NodeOperationError if any field does not exist', () => {
const aggregations: Aggregations = [
{ aggregation: 'sum', field: 'b' },
{ aggregation: 'count', field: 'd' },
];
const getValue = (item: IDataObject, field: string) => item[field];
expect(() => {
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
}).toThrow(NodeOperationError);
});
it("should throw NodeOperationError with error message containing the field name that doesn't exist", () => {
const aggregations: Aggregations = [{ aggregation: 'count', field: 'D' }];
const getValue = (item: IDataObject, field: string) => item[field];
expect(() => {
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
}).toThrow("The field 'D' does not exist in any items");
});
it('should not throw error if field is empty string', () => {
const aggregations: Aggregations = [{ aggregation: 'count', field: '' }];
const getValue = (item: IDataObject, field: string) => item[field];
expect(() => {
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
}).not.toThrow();
});
});
@@ -0,0 +1,77 @@
import type { MockProxy } from 'jest-mock-extended';
import { mock } from 'jest-mock-extended';
import type { IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { Summarize } from '../../Summarize.node';
import type { Aggregations } from '../../utils';
let summarizeNode: Summarize;
let mockExecuteFunctions: MockProxy<IExecuteFunctions>;
describe('Test Summarize Node, execute', () => {
beforeEach(() => {
summarizeNode = new Summarize();
mockExecuteFunctions = mock<IExecuteFunctions>({
getNode: jest.fn().mockReturnValue({ name: 'test-node' }),
getNodeParameter: jest.fn(),
getInputData: jest.fn(),
helpers: {
constructExecutionMetaData: jest.fn().mockReturnValue([]),
},
});
});
afterEach(() => {
jest.clearAllMocks();
});
it('should handle field not found with hints if version > 1', async () => {
mockExecuteFunctions.getInputData.mockReturnValue([{ json: { someField: 1 } }]);
mockExecuteFunctions.getNode.mockReturnValue({
id: '1',
name: 'test-node',
type: 'test-type',
position: [0, 0],
parameters: {},
typeVersion: 1.1,
});
mockExecuteFunctions.getNodeParameter
.mockReturnValueOnce({}) // options
.mockReturnValueOnce('') // fieldsToSplitBy
.mockReturnValueOnce([{ field: 'nonexistentField', aggregation: 'sum' }]); // fieldsToSummarize
const result = await summarizeNode.execute.call(mockExecuteFunctions);
expect(result).toEqual([[{ json: { sum_nonexistentField: 0 }, pairedItem: [{ item: 0 }] }]]);
expect(mockExecuteFunctions.addExecutionHints).toHaveBeenCalledWith({
location: 'outputPane',
message: "The field 'nonexistentField' does not exist in any items",
});
});
it('should throw error if node version is < 1.1 and fields not found', async () => {
const items = [{ json: { a: 1, b: 2, c: 3 } }];
const aggregations: Aggregations = [
{ aggregation: 'sum', field: 'b' },
{ aggregation: 'count', field: 'd' },
];
mockExecuteFunctions.getNode.mockReturnValue({
id: '1',
name: 'test-node',
type: 'test-type',
position: [0, 0],
parameters: {},
typeVersion: 1,
});
mockExecuteFunctions.getInputData.mockReturnValue(items);
mockExecuteFunctions.getNodeParameter
.mockReturnValueOnce({}) // options
.mockReturnValueOnce('') // fieldsToSplitBy
.mockReturnValueOnce(aggregations); // fieldsToSummarize
await expect(async () => {
await summarizeNode.execute.bind(mockExecuteFunctions)();
}).rejects.toThrow(NodeOperationError);
});
});
@@ -0,0 +1,314 @@
import {
fieldValueGetter,
aggregateAndSplitData,
flattenAggregationResultToArray,
type Aggregations,
} from '../../utils';
describe('Test Summarize Node, aggregateAndSplitData', () => {
test('should not convert strings to numbers', () => {
const data = [
{
Sku: '012345',
Warehouse: 'BER_0G',
Qty: '1',
_itemIndex: 0,
},
{
Sku: '012345',
Warehouse: 'BER_0L',
Qty: '2',
_itemIndex: 1,
},
{
Sku: '06534563534',
Warehouse: 'BER_0L',
Qty: '1',
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Sku', 'Qty'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
});
expect(result).toMatchSnapshot('result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
});
test('should not convert numbers to strings', () => {
const data = [
{
Sku: 12345,
Warehouse: 'BER_0G',
Qty: 1,
_itemIndex: 0,
},
{
Sku: 12345,
Warehouse: 'BER_0L',
Qty: 2,
_itemIndex: 1,
},
{
Sku: 6534563534,
Warehouse: 'BER_0L',
Qty: 1,
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Sku', 'Qty'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
});
expect(result).toMatchSnapshot('result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
});
test('should handle split field values containing spaces when convertKeysToString is not set', () => {
const data = [
{
Product: 'Widget A',
Warehouse: 'WH1',
Qty: '5',
_itemIndex: 0,
},
{
Product: 'Widget B',
Warehouse: 'WH2',
Qty: '3',
_itemIndex: 1,
},
{
Product: 'Widget A',
Warehouse: 'WH3',
Qty: '2',
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Product'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
});
expect(result).toMatchSnapshot('split-field-with-spaces-result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
'split-field-with-spaces-array',
);
});
test('should handle split field values containing spaces when convertKeysToString is true', () => {
const data = [
{
Product: 'Widget A',
Warehouse: 'WH1',
Qty: '5',
_itemIndex: 0,
},
{
Product: 'Widget B',
Warehouse: 'WH2',
Qty: '3',
_itemIndex: 1,
},
{
Product: 'Widget A',
Warehouse: 'WH3',
Qty: '2',
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Product'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
convertKeysToString: true,
});
expect(result).toMatchSnapshot('split-field-with-spaces-result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
'split-field-with-spaces-array',
);
});
test('should handle multiple split field values containing null when convertKeysToString is true', () => {
const data = [
{
Product: 'Widget A',
Warehouse: 'WH1',
Qty: '5',
_itemIndex: 0,
},
{
Product: null,
Warehouse: 'WH2',
Qty: '3',
_itemIndex: 1,
},
{
Product: 'Widget A',
Warehouse: null,
Qty: '2',
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Product', 'Warehouse'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
convertKeysToString: true,
});
expect(result).toMatchSnapshot('split-field-with-spaces-result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
'split-field-with-spaces-array',
);
});
test('should handle multiple split field values containing null when convertKeysToString is false', () => {
const data = [
{
Product: 'Widget A',
Warehouse: 'WH1',
Qty: '5',
_itemIndex: 0,
},
{
Product: null,
Warehouse: 'WH2',
Qty: '3',
_itemIndex: 1,
},
{
Product: 'Widget A',
Warehouse: null,
Qty: '2',
_itemIndex: 2,
},
];
const aggregations: Aggregations = [
{
aggregation: 'append',
field: 'Warehouse',
includeEmpty: true,
},
];
const result = aggregateAndSplitData({
splitKeys: ['Product', 'Warehouse'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true },
getValue: fieldValueGetter(),
convertKeysToString: false,
});
expect(result).toMatchSnapshot('split-field-with-spaces-result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
'split-field-with-spaces-array',
);
});
describe('with skipEmptySplitFields=true', () => {
test('should skip empty split fields', () => {
const data = [
{
Sku: 12345,
Warehouse: 'BER_0G',
_itemIndex: 0,
},
{
Warehouse: 'BER_0L',
_itemIndex: 1,
},
{
Sku: {},
Warehouse: 'BER_0L',
_itemIndex: 2,
},
{
Sku: 12345,
Warehouse: { name: 'BER_0G3' },
_itemIndex: 3,
},
];
const aggregations: Aggregations = [
{
aggregation: 'concatenate',
field: 'Warehouse',
separateBy: 'other',
customSeparator: '//',
},
];
const result = aggregateAndSplitData({
splitKeys: ['Sku'],
inputItems: data,
fieldsToSummarize: aggregations,
options: { continueIfFieldNotFound: true, skipEmptySplitFields: true },
getValue: fieldValueGetter(),
});
expect(result).toMatchSnapshot('result');
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
});
});
});
@@ -0,0 +1,416 @@
{
"name": "summarize test",
"nodes": [
{
"parameters": {},
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-160, 340],
"id": "0cc16e22-72e3-4f2c-a920-b1613fd4bcaf",
"name": "When clicking Execute workflow"
},
{
"parameters": {
"data": [
{
"category": "red",
"text": "foo",
"char": "a",
"value": 1,
"params.interval": "1 day",
"params": {
"interval": "1 day"
}
},
{
"category": "blue",
"text": "spam",
"char": "b",
"value": 2,
"params": {
"interval": "1 hour"
}
},
{
"category": "green",
"text": "bar",
"char": "c",
"value": 3
},
{
"category": "red",
"text": "foo",
"char": "a",
"value": 4
},
{
"category": "red",
"text": "bar",
"char": "a",
"value": 5,
"params": {
"interval": "1 minute"
}
},
{
"category": "blue",
"text": "foo",
"char": "a",
"value": 6
},
{
"category": "blue",
"text": "foo",
"char": "a",
"value": 7,
"params": {
"interval": "1 second"
}
}
]
},
"id": "5a71d6f2-96fb-4616-94b7-5644c71e3bfb",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [60, 340]
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "char"
},
{
"field": "char"
},
{
"aggregation": "countUnique",
"field": "char"
},
{
"aggregation": "concatenate",
"field": "char",
"separateBy": ", "
}
]
},
"fieldsToSplitBy": "category, text",
"options": {}
},
"id": "281b21e0-5737-4150-bce8-8331a3f83366",
"name": "Summarize1",
"type": "n8n-nodes-base.summarize",
"typeVersion": 1,
"position": [280, -60]
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "char"
},
{
"field": "char"
},
{
"aggregation": "countUnique",
"field": "char"
},
{
"aggregation": "concatenate",
"field": "char",
"separateBy": ", "
}
]
},
"fieldsToSplitBy": "category, text",
"options": {
"outputFormat": "singleItem"
}
},
"id": "04bf538c-24d4-46d8-9c43-76ebc4fe7368",
"name": "Summarize2",
"type": "n8n-nodes-base.summarize",
"typeVersion": 1,
"position": [280, 140]
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "average",
"field": "value"
},
{
"aggregation": "max",
"field": "value"
},
{
"aggregation": "min",
"field": "value"
},
{
"aggregation": "max",
"field": "value"
},
{
"aggregation": "sum",
"field": "value"
},
{
"aggregation": "append",
"field": "value"
}
]
},
"fieldsToSplitBy": "category",
"options": {}
},
"id": "e6f750f2-15c1-4de9-80a1-a255649d6e99",
"name": "Summarize3",
"type": "n8n-nodes-base.summarize",
"typeVersion": 1,
"position": [280, 340]
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "params.interval"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [280, 540],
"id": "bfe0521a-c060-454f-99bb-b682aab44f63",
"name": "Dot notation"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "params.interval"
}
]
},
"options": {
"disableDotNotation": true
}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [280, 740],
"id": "b2c05f23-449f-49aa-80a3-4e1cd9156aa1",
"name": "Dot notation (disabled)"
}
],
"pinData": {
"Summarize1": [
{
"json": {
"category": "red",
"text": "foo",
"appended_char": ["a", "a"],
"count_char": 2,
"unique_count_char": 1,
"concatenated_char": "a, a"
}
},
{
"json": {
"category": "red",
"text": "bar",
"appended_char": ["a"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "a"
}
},
{
"json": {
"category": "blue",
"text": "spam",
"appended_char": ["b"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "b"
}
},
{
"json": {
"category": "blue",
"text": "foo",
"appended_char": ["a", "a"],
"count_char": 2,
"unique_count_char": 1,
"concatenated_char": "a, a"
}
},
{
"json": {
"category": "green",
"text": "bar",
"appended_char": ["c"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "c"
}
}
],
"Summarize2": [
{
"json": {
"red": {
"foo": {
"appended_char": ["a", "a"],
"count_char": 2,
"unique_count_char": 1,
"concatenated_char": "a, a"
},
"bar": {
"appended_char": ["a"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "a"
}
},
"blue": {
"spam": {
"appended_char": ["b"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "b"
},
"foo": {
"appended_char": ["a", "a"],
"count_char": 2,
"unique_count_char": 1,
"concatenated_char": "a, a"
}
},
"green": {
"bar": {
"appended_char": ["c"],
"count_char": 1,
"unique_count_char": 1,
"concatenated_char": "c"
}
}
}
}
],
"Summarize3": [
{
"json": {
"category": "red",
"average_value": 3.3333333333333335,
"max_value": 5,
"min_value": 1,
"sum_value": 10,
"appended_value": [1, 4, 5]
}
},
{
"json": {
"category": "blue",
"average_value": 5,
"max_value": 7,
"min_value": 2,
"sum_value": 15,
"appended_value": [2, 6, 7]
}
},
{
"json": {
"category": "green",
"average_value": 3,
"max_value": 3,
"min_value": 3,
"sum_value": 3,
"appended_value": [3]
}
}
],
"Dot notation": [
{
"json": {
"appended_params_interval": ["1 day", "1 hour", "1 minute", "1 second"]
}
}
],
"Dot notation (disabled)": [
{
"json": {
"appended_params_interval": ["1 day"]
}
}
]
},
"connections": {
"When clicking Execute workflow": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Summarize1",
"type": "main",
"index": 0
},
{
"node": "Summarize2",
"type": "main",
"index": 0
},
{
"node": "Summarize3",
"type": "main",
"index": 0
},
{
"node": "Dot notation",
"type": "main",
"index": 0
},
{
"node": "Dot notation (disabled)",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {
"executionOrder": "v1"
},
"versionId": "60e793d6-8c99-4bd2-aeb8-8ff350fc3101",
"meta": {
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
},
"id": "0HNQZNduHDmRCBBW",
"tags": []
}
@@ -0,0 +1,366 @@
{
"name": "Summarize Test",
"nodes": [
{
"parameters": {},
"id": "e46eed6e-4a6b-4e1f-bd7a-96cf579dd2ee",
"name": "When clicking \"Execute workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-900, 260]
},
{
"parameters": {
"data": [
{ "review": "Great product!" },
{ "review": "" },
{ "review": null },
{ "review": null },
{ "review": "Bad product" },
{},
{}
]
},
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [-520, -80],
"id": "80f3462d-7ea5-4af1-bd13-f3f46b0ee43f",
"name": "sample data1"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"field": "review"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-280, -320],
"id": "98ff6686-bf9f-436a-8a64-3b3b1e648184",
"name": "count non-empty"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "countUnique",
"field": "review"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-280, -160],
"id": "aa094d4c-5e00-49a4-9799-e7692b49f204",
"name": "countUnique non-empty"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"field": "review",
"includeEmpty": true
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-280, -40],
"id": "c4a439c9-4d25-4ebb-b36c-c72d0d738049",
"name": "count with Empty"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "countUnique",
"field": "review",
"includeEmpty": true
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-280, 100],
"id": "e8abc054-5285-4dcc-980b-4bba4ef45ae0",
"name": "countUnique non-empty1"
},
{
"parameters": {
"data": [
{
"region": "North",
"sales": 100
},
{
"region": "North",
"sales": 200
},
{
"region": "South",
"sales": null
},
{
"region": "",
"sales": 300
},
{
"region": "East"
},
{
"region": "West",
"sales": 400
}
]
},
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [-520, 500],
"id": "0de1d232-09c1-412f-9481-0120db0792bf",
"name": "sample data2"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"field": "region"
},
{
"field": "sales"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-260, 300],
"id": "b81e9c48-828c-4ac8-b7ca-818022384ca6",
"name": "count non-empty1"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "countUnique",
"field": "region"
},
{
"field": "sales"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-260, 440],
"id": "22bd90da-9b57-4b38-9cc3-356901a6f666",
"name": "countUnique non-empty2"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"field": "region",
"includeEmpty": true
},
{
"field": "sales",
"includeEmpty": true
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-260, 580],
"id": "02bfe12f-eb71-483b-bef5-6007af739ace",
"name": "count with Empty1"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "countUnique",
"field": "region",
"includeEmpty": true
},
{
"field": "sales",
"includeEmpty": true
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [-260, 720],
"id": "e67cd4b5-47ae-4d73-b933-905b04183db8",
"name": "countUnique non-empty3"
}
],
"pinData": {
"count non-empty": [
{
"json": {
"count_review": 2
}
}
],
"countUnique non-empty": [
{
"json": {
"unique_count_review": 2
}
}
],
"count with Empty": [
{
"json": {
"count_review": 7
}
}
],
"countUnique non-empty1": [
{
"json": {
"unique_count_review": 5
}
}
],
"count non-empty1": [
{
"json": {
"count_region": 5,
"count_sales": 4
}
}
],
"countUnique non-empty2": [
{
"json": {
"unique_count_region": 4,
"count_sales": 4
}
}
],
"count with Empty1": [
{
"json": {
"count_region": 6,
"count_sales": 6
}
}
],
"countUnique non-empty3": [
{
"json": {
"unique_count_region": 5,
"count_sales": 6
}
}
]
},
"connections": {
"When clicking \"Execute workflow\"": {
"main": [
[
{
"node": "sample data1",
"type": "main",
"index": 0
},
{
"node": "sample data2",
"type": "main",
"index": 0
}
]
]
},
"sample data1": {
"main": [
[
{
"node": "count non-empty",
"type": "main",
"index": 0
},
{
"node": "countUnique non-empty",
"type": "main",
"index": 0
},
{
"node": "count with Empty",
"type": "main",
"index": 0
},
{
"node": "countUnique non-empty1",
"type": "main",
"index": 0
}
]
]
},
"sample data2": {
"main": [
[
{
"node": "count non-empty1",
"type": "main",
"index": 0
},
{
"node": "countUnique non-empty2",
"type": "main",
"index": 0
},
{
"node": "count with Empty1",
"type": "main",
"index": 0
},
{
"node": "countUnique non-empty3",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {
"executionOrder": "v1"
},
"versionId": "bb837da6-9160-4c61-b3a4-e7ef192bd5c6",
"meta": {
"templateCredsSetupCompleted": true,
"instanceId": "b0d1a7f453aa2195078ce658269afa7743e3dc59edb733395b6c75bde57da7ff"
},
"id": "8ja9H2443AhLdBsS",
"tags": []
}
@@ -0,0 +1,267 @@
{
"name": "Summarize Test",
"nodes": [
{
"parameters": {},
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-320, -80],
"id": "509c50eb-a5c3-4172-a553-82cce7ca0455",
"name": "When clicking Execute workflow"
},
{
"parameters": {
"data": [
{
"uid": "a0e797c2-ffcf-495b-96f3-982f7bc4eae5",
"email": "Alfredo83@gmail.com",
"firstname": "Charlie",
"lastname": "Jacobi",
"password": "34~XLKdGit",
"Group": 0
},
{
"uid": "5713682e-4f32-47dc-ae89-ae4a8ac28a46",
"email": "Zachary43@yahoo.com",
"firstname": "Leonard",
"lastname": "Blick",
"password": "4sh5z!R",
"Group": 0
},
{
"uid": "1bb2a432-d2fb-48e4-a7b0-39030ef0d784",
"email": "Cory_Wilderman@yahoo.com",
"firstname": "Debra",
"lastname": "Corkery",
"password": "-8X1CKMFh",
"Group": 1
},
{
"uid": "4edb11bc-9256-4f26-9be6-f8b04e90ecb2",
"email": "Mindy_Murazik86@gmail.com",
"firstname": "Wendy",
"lastname": "Schiller",
"password": "WF373t,fe",
"Group": 2
},
{
"uid": "ec6efd3d-508c-4f1e-acae-c9467b74f5a6",
"email": "Darin_Vandervort@yahoo.com",
"firstname": "Dolores",
"lastname": "Walter",
"password": ".45KCBk",
"Group": 2
},
{
"uid": "173ee780-4e4f-4164-aa39-41a0402cd2ad",
"email": "Claude.Grady6@gmail.com",
"firstname": "Leonard",
"lastname": "Krajcik",
"password": "6x6-MN8vch",
"Group": null
},
{
"uid": "b1f191d8-4f95-49f6-8533-042048b4c162",
"email": "Ismael_Volkman@hotmail.com",
"firstname": "Terri",
"lastname": "Kuphal",
"password": "62+BNNCk",
"Group": null
},
{
"uid": "ee8c6d18-1e77-435d-8c9a-5215141d7697",
"email": "Paulette.Hudson@hotmail.com",
"firstname": "Clinton",
"lastname": "Murphy",
"password": "2+R8IRxHL",
"Group": null
},
{
"uid": "feeb4357-0e20-4b15-9199-f161c23da1a2",
"email": "Guillermo.Kub33@gmail.com",
"firstname": "Edward",
"lastname": "Schaden",
"password": "Kv23S1_H",
"Group": 0
},
{
"uid": "676003ec-8a40-4105-8237-ba1d9eb51fc2",
"email": "Cindy.White@hotmail.com",
"firstname": "Esther",
"lastname": "Monahan",
"password": "5q$RyBhp",
"Group": 2
}
]
},
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [-100, -80],
"id": "691f2744-8689-4b5c-91e7-2bb378691ecf",
"name": "sample data"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "email",
"includeEmpty": true
}
]
},
"fieldsToSplitBy": "Group",
"options": {
"outputFormat": "separateItems"
}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [160, 0],
"id": "d1267da5-0cab-453c-9450-1bbd78346bd6",
"name": "Summarize v1.1"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"aggregation": "append",
"field": "email",
"includeEmpty": true
}
]
},
"fieldsToSplitBy": "Group",
"options": {
"continueIfFieldNotFound": false,
"outputFormat": "separateItems"
}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1,
"position": [160, -160],
"id": "680b7269-c920-4cce-b9f5-b2189955848d",
"name": "Summarize v1"
}
],
"pinData": {
"Summarize v1": [
{
"json": {
"Group": "0",
"appended_email": [
"Alfredo83@gmail.com",
"Zachary43@yahoo.com",
"Guillermo.Kub33@gmail.com"
]
}
},
{
"json": {
"Group": "1",
"appended_email": ["Cory_Wilderman@yahoo.com"]
}
},
{
"json": {
"Group": "2",
"appended_email": [
"Mindy_Murazik86@gmail.com",
"Darin_Vandervort@yahoo.com",
"Cindy.White@hotmail.com"
]
}
},
{
"json": {
"Group": "null",
"appended_email": [
"Claude.Grady6@gmail.com",
"Ismael_Volkman@hotmail.com",
"Paulette.Hudson@hotmail.com"
]
}
}
],
"Summarize v1.1": [
{
"json": {
"Group": 0,
"appended_email": [
"Alfredo83@gmail.com",
"Zachary43@yahoo.com",
"Guillermo.Kub33@gmail.com"
]
}
},
{
"json": {
"Group": 1,
"appended_email": ["Cory_Wilderman@yahoo.com"]
}
},
{
"json": {
"Group": 2,
"appended_email": [
"Mindy_Murazik86@gmail.com",
"Darin_Vandervort@yahoo.com",
"Cindy.White@hotmail.com"
]
}
},
{
"json": {
"Group": null,
"appended_email": [
"Claude.Grady6@gmail.com",
"Ismael_Volkman@hotmail.com",
"Paulette.Hudson@hotmail.com"
]
}
}
]
},
"connections": {
"When clicking Execute workflow": {
"main": [
[
{
"node": "sample data",
"type": "main",
"index": 0
}
]
]
},
"sample data": {
"main": [
[
{
"node": "Summarize v1",
"type": "main",
"index": 0
},
{
"node": "Summarize v1.1",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {
"executionOrder": "v1"
},
"versionId": "7304eae2-3bae-4934-8d1f-1135a0ff8ab8",
"meta": {
"templateCredsSetupCompleted": true,
"instanceId": "b0d1a7f453aa2195078ce658269afa7743e3dc59edb733395b6c75bde57da7ff"
},
"id": "8ja9H2443AhLdBsS",
"tags": []
}
@@ -0,0 +1,70 @@
{
"nodes": [
{
"parameters": {},
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [520, -80],
"id": "9ff1d8e0-ebe8-4c52-931a-e482e14ac911",
"name": "When clicking Execute workflow"
},
{
"parameters": {
"category": "randomData",
"randomDataSeed": "ria",
"randomDataCount": 5
},
"type": "n8n-nodes-base.debugHelper",
"typeVersion": 1,
"position": [700, -80],
"id": "50cea846-f8ed-4b5a-a14a-c11bf4ba0f5d",
"name": "DebugHelper2"
},
{
"parameters": {
"fieldsToSummarize": {
"values": [
{
"field": "passwor"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.summarize",
"typeVersion": 1.1,
"position": [900, -80],
"id": "d39fedc9-62e1-45bc-8c9a-d20f2b63b543",
"name": "Summarize1"
}
],
"connections": {
"When clicking Execute workflow": {
"main": [
[
{
"node": "DebugHelper2",
"type": "main",
"index": 0
}
]
]
},
"DebugHelper2": {
"main": [
[
{
"node": "Summarize1",
"type": "main",
"index": 0
}
]
]
}
},
"pinData": {},
"meta": {
"templateCredsSetupCompleted": true,
"instanceId": "ee90fdf8d57662f949e6c691dc07fa0fd2f66e1eee28ed82ef06658223e67255"
}
}
@@ -0,0 +1,288 @@
import get from 'lodash/get';
import {
type GenericValue,
type IDataObject,
type IExecuteFunctions,
NodeOperationError,
} from 'n8n-workflow';
type AggregationType =
| 'append'
| 'average'
| 'concatenate'
| 'count'
| 'countUnique'
| 'max'
| 'min'
| 'sum';
export type Aggregation = {
aggregation: AggregationType;
field: string;
includeEmpty?: boolean;
separateBy?: string;
customSeparator?: string;
};
export type Aggregations = Aggregation[];
const AggregationDisplayNames = {
append: 'appended_',
average: 'average_',
concatenate: 'concatenated_',
count: 'count_',
countUnique: 'unique_count_',
max: 'max_',
min: 'min_',
sum: 'sum_',
};
export const NUMERICAL_AGGREGATIONS = ['average', 'sum'];
export type SummarizeOptions = {
continueIfFieldNotFound: boolean;
disableDotNotation?: boolean;
outputFormat?: 'separateItems' | 'singleItem';
skipEmptySplitFields?: boolean;
};
export type ValueGetterFn = (
item: IDataObject,
field: string,
) => IDataObject | IDataObject[] | GenericValue | GenericValue[];
function isEmpty<T>(value: T) {
return value === undefined || value === null || value === '';
}
function normalizeFieldName(fieldName: string) {
return fieldName.replace(/[\]\["]/g, '').replace(/[ .]/g, '_');
}
export const fieldValueGetter = (disableDotNotation?: boolean) => {
return (item: IDataObject, field: string) =>
disableDotNotation ? item[field] : get(item, field);
};
export function checkIfFieldExists(
this: IExecuteFunctions,
items: IDataObject[],
aggregations: Aggregations,
getValue: ValueGetterFn,
) {
for (const aggregation of aggregations) {
if (aggregation.field === '') {
continue;
}
const exist = items.some((item) => getValue(item, aggregation.field) !== undefined);
if (!exist) {
throw new NodeOperationError(
this.getNode(),
`The field '${aggregation.field}' does not exist in any items`,
);
}
}
}
function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) {
const { aggregation, field } = entry;
let data = [...items];
if (NUMERICAL_AGGREGATIONS.includes(aggregation)) {
data = data.filter(
(item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)),
);
}
switch (aggregation) {
//combine operations
case 'append':
if (!entry.includeEmpty) {
data = data.filter((item) => !isEmpty(getValue(item, field)));
}
return data.map((item) => getValue(item, field));
case 'concatenate':
const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy;
if (!entry.includeEmpty) {
data = data.filter((item) => !isEmpty(getValue(item, field)));
}
return data
.map((item) => {
let value = getValue(item, field);
if (typeof value === 'object') {
value = JSON.stringify(value);
}
if (typeof value === 'undefined') {
value = 'undefined';
}
return value;
})
.join(separateBy);
//numerical operations
case 'average':
return (
data.reduce((acc, item) => {
return acc + (getValue(item, field) as number);
}, 0) / data.length
);
case 'sum':
return data.reduce((acc, item) => {
return acc + (getValue(item, field) as number);
}, 0);
//comparison operations
case 'min':
let min;
for (const item of data) {
const value = getValue(item, field);
if (value !== undefined && value !== null && value !== '') {
if (min === undefined || value < min) {
min = value;
}
}
}
return min ?? null;
case 'max':
let max;
for (const item of data) {
const value = getValue(item, field);
if (value !== undefined && value !== null && value !== '') {
if (max === undefined || value > max) {
max = value;
}
}
}
return max ?? null;
//count operations
case 'countUnique':
if (!entry.includeEmpty) {
return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item)))
.size;
}
return new Set(data.map((item) => getValue(item, field))).size;
default:
//count by default
if (!entry.includeEmpty) {
return data.filter((item) => !isEmpty(getValue(item, field))).length;
}
return data.length;
}
}
function aggregateData(
data: IDataObject[],
fieldsToSummarize: Aggregations,
options: SummarizeOptions,
getValue: ValueGetterFn,
): { returnData: IDataObject; pairedItems?: number[] } {
const returnData = Object.fromEntries(
fieldsToSummarize.map((aggregation) => {
const key = normalizeFieldName(
`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`,
);
const result = aggregate(data, aggregation, getValue);
return [key, result];
}),
);
if (options.outputFormat === 'singleItem') {
return { returnData };
}
return { returnData, pairedItems: data.map((item) => item._itemIndex as number) };
}
type AggregationResult = { returnData: IDataObject; pairedItems?: number[] };
type NestedAggregationResult =
| AggregationResult
| { fieldName: string; splits: Map<unknown, NestedAggregationResult> };
// Using Map to preserve types
// With a plain JS object, keys are converted to string
export function aggregateAndSplitData({
splitKeys,
inputItems,
fieldsToSummarize,
options,
getValue,
convertKeysToString = false,
}: {
splitKeys: string[] | undefined;
inputItems: IDataObject[];
fieldsToSummarize: Aggregations;
options: SummarizeOptions;
getValue: ValueGetterFn;
convertKeysToString?: boolean; // Legacy option for v1
}): NestedAggregationResult {
if (!splitKeys?.length) {
return aggregateData(inputItems, fieldsToSummarize, options, getValue);
}
const [firstSplitKey, ...restSplitKeys] = splitKeys;
const groupedItems = new Map<unknown, IDataObject[]>();
for (const item of inputItems) {
let splitValue = getValue(item, firstSplitKey);
if (splitValue && typeof splitValue === 'object') {
splitValue = JSON.stringify(splitValue);
}
if (convertKeysToString) {
splitValue = String(splitValue);
}
if (options.skipEmptySplitFields && typeof splitValue !== 'number' && !splitValue) {
continue;
}
const group = groupedItems.get(splitValue) ?? [];
groupedItems.set(splitValue, group.concat([item]));
}
const splits = new Map(
Array.from(groupedItems.entries()).map(([groupKey, items]) => [
groupKey,
aggregateAndSplitData({
splitKeys: restSplitKeys,
inputItems: items,
fieldsToSummarize,
options,
getValue,
convertKeysToString,
}),
]),
);
return { fieldName: firstSplitKey, splits };
}
export function flattenAggregationResultToObject(result: NestedAggregationResult): IDataObject {
if ('splits' in result) {
return Object.fromEntries(
Array.from(result.splits.entries()).map(([key, value]) => [
key,
flattenAggregationResultToObject(value),
]),
);
}
return result.returnData;
}
export function flattenAggregationResultToArray(
result: NestedAggregationResult,
): AggregationResult[] {
if ('splits' in result) {
return Array.from(result.splits.entries()).flatMap(([value, innerResult]) =>
flattenAggregationResultToArray(innerResult).map((v) => {
v.returnData[normalizeFieldName(result.fieldName)] = value as IDataObject;
return v;
}),
);
}
return [result];
}
@@ -0,0 +1,17 @@
import { ApplicationError } from '@n8n/errors';
export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => {
if (typeof fields === 'string') {
return fields
.split(',')
.map((entry) => entry.trim())
.filter((entry) => entry !== '');
}
if (Array.isArray(fields)) {
return fields;
}
throw new ApplicationError(
`The \'${fieldName}\' parameter must be a string of fields separated by commas or an array of strings.`,
{ level: 'warning' },
);
};