first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.aggregate",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.aggregate/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
import get from 'lodash/get';
|
||||
import isEmpty from 'lodash/isEmpty';
|
||||
import set from 'lodash/set';
|
||||
import {
|
||||
NodeOperationError,
|
||||
type IDataObject,
|
||||
type IExecuteFunctions,
|
||||
type INodeExecutionData,
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
type IPairedItemData,
|
||||
NodeConnectionTypes,
|
||||
type NodeExecutionHint,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { addBinariesToItem } from './utils';
|
||||
import { prepareFieldsArray } from '../utils/utils';
|
||||
|
||||
export class Aggregate implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Aggregate',
|
||||
name: 'aggregate',
|
||||
icon: 'file:aggregate.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: 1,
|
||||
description: 'Combine a field from many items into a list in a single item',
|
||||
defaults: {
|
||||
name: 'Aggregate',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
builderHint: {
|
||||
message:
|
||||
'Need to combine items from multiple branches? Use merge node. This nodes combines all items from one branch into one item.',
|
||||
relatedNodes: [
|
||||
{
|
||||
nodeType: 'n8n-nodes-base.merge',
|
||||
relationHint: 'For multiple branches',
|
||||
},
|
||||
{
|
||||
nodeType: 'n8n-nodes-base.splitOut',
|
||||
relationHint: 'Reverse operation',
|
||||
},
|
||||
],
|
||||
},
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Aggregate',
|
||||
name: 'aggregate',
|
||||
type: 'options',
|
||||
default: 'aggregateIndividualFields',
|
||||
options: [
|
||||
{
|
||||
name: 'Individual Fields',
|
||||
value: 'aggregateIndividualFields',
|
||||
},
|
||||
{
|
||||
name: 'All Item Data (Into a Single List)',
|
||||
value: 'aggregateAllItemData',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Aggregate',
|
||||
name: 'fieldsToAggregate',
|
||||
type: 'fixedCollection',
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
placeholder: 'Add Field To Aggregate',
|
||||
default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] },
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregate: ['aggregateIndividualFields'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: '',
|
||||
name: 'fieldToAggregate',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Input Field Name',
|
||||
name: 'fieldToAggregate',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'The name of a field in the input items to aggregate together',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
|
||||
placeholder: 'e.g. id',
|
||||
hint: ' Enter the field name as text',
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
{
|
||||
displayName: 'Rename Field',
|
||||
name: 'renameField',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to give the field a different name in the output',
|
||||
},
|
||||
{
|
||||
displayName: 'Output Field Name',
|
||||
name: 'outputFieldName',
|
||||
displayOptions: {
|
||||
show: {
|
||||
renameField: [true],
|
||||
},
|
||||
},
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The name of the field to put the aggregated data in. Leave blank to use the input field name.',
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Put Output in Field',
|
||||
name: 'destinationFieldName',
|
||||
type: 'string',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregate: ['aggregateAllItemData'],
|
||||
},
|
||||
},
|
||||
default: 'data',
|
||||
description: 'The name of the output field to put the data in',
|
||||
},
|
||||
{
|
||||
displayName: 'Include',
|
||||
name: 'include',
|
||||
type: 'options',
|
||||
default: 'allFields',
|
||||
options: [
|
||||
{
|
||||
name: 'All Fields',
|
||||
value: 'allFields',
|
||||
},
|
||||
{
|
||||
name: 'Specified Fields',
|
||||
value: 'specifiedFields',
|
||||
},
|
||||
{
|
||||
name: 'All Fields Except',
|
||||
value: 'allFieldsExcept',
|
||||
},
|
||||
],
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregate: ['aggregateAllItemData'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Exclude',
|
||||
name: 'fieldsToExclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
default: '',
|
||||
requiresDataPath: 'multiple',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregate: ['aggregateAllItemData'],
|
||||
include: ['allFieldsExcept'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Include',
|
||||
name: 'fieldsToInclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
default: '',
|
||||
requiresDataPath: 'multiple',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregate: ['aggregateAllItemData'],
|
||||
include: ['specifiedFields'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/aggregate': ['aggregateAllItemData'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Merge Lists',
|
||||
name: 'mergeLists',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/aggregate': ['aggregateAllItemData'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Include Binaries',
|
||||
name: 'includeBinaries',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to include the binary data in the new item',
|
||||
},
|
||||
{
|
||||
displayName: 'Keep Only Unique Binaries',
|
||||
name: 'keepOnlyUnique',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
|
||||
displayOptions: {
|
||||
show: {
|
||||
includeBinaries: [true],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Keep Missing And Null Values',
|
||||
name: 'keepMissing',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to add a null entry to the aggregated list when there is a missing or null value',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/aggregate': ['aggregateAllItemData'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
let returnData: INodeExecutionData = { json: {}, pairedItem: [] };
|
||||
const items = this.getInputData();
|
||||
const notFoundedFields: { [key: string]: boolean[] } = {};
|
||||
|
||||
const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
|
||||
|
||||
if (aggregate === 'aggregateIndividualFields') {
|
||||
const disableDotNotation = this.getNodeParameter(
|
||||
'options.disableDotNotation',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean;
|
||||
const fieldsToAggregate = this.getNodeParameter(
|
||||
'fieldsToAggregate.fieldToAggregate',
|
||||
0,
|
||||
[],
|
||||
) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }];
|
||||
const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean;
|
||||
|
||||
if (!fieldsToAggregate.length) {
|
||||
throw new NodeOperationError(this.getNode(), 'No fields specified', {
|
||||
description: 'Please add a field to aggregate',
|
||||
});
|
||||
}
|
||||
|
||||
const newItem: INodeExecutionData = {
|
||||
json: {},
|
||||
pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => {
|
||||
return {
|
||||
item: index,
|
||||
};
|
||||
}),
|
||||
};
|
||||
|
||||
const values: { [key: string]: any } = {};
|
||||
const outputFields: string[] = [];
|
||||
|
||||
for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) {
|
||||
const field = renameField ? outputFieldName : fieldToAggregate;
|
||||
|
||||
if (outputFields.includes(field)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The '${field}' output field is used more than once`,
|
||||
{ description: 'Please make sure each output field name is unique' },
|
||||
);
|
||||
} else {
|
||||
outputFields.push(field);
|
||||
}
|
||||
|
||||
const getFieldToAggregate = () =>
|
||||
!disableDotNotation && fieldToAggregate.includes('.')
|
||||
? fieldToAggregate.split('.').pop()
|
||||
: fieldToAggregate;
|
||||
|
||||
const _outputFieldName = outputFieldName
|
||||
? outputFieldName
|
||||
: (getFieldToAggregate() as string);
|
||||
|
||||
if (fieldToAggregate !== '') {
|
||||
values[_outputFieldName] = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
if (notFoundedFields[fieldToAggregate] === undefined) {
|
||||
notFoundedFields[fieldToAggregate] = [];
|
||||
}
|
||||
|
||||
if (!disableDotNotation) {
|
||||
let value = get(items[i].json, fieldToAggregate);
|
||||
notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
|
||||
|
||||
if (!keepMissing) {
|
||||
if (Array.isArray(value)) {
|
||||
value = value.filter((entry) => entry !== null);
|
||||
} else if (value === null || value === undefined) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(value) && mergeLists) {
|
||||
values[_outputFieldName].push(...value);
|
||||
} else {
|
||||
values[_outputFieldName].push(value);
|
||||
}
|
||||
} else {
|
||||
let value = items[i].json[fieldToAggregate];
|
||||
notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);
|
||||
|
||||
if (!keepMissing) {
|
||||
if (Array.isArray(value)) {
|
||||
value = value.filter((entry) => entry !== null);
|
||||
} else if (value === null || value === undefined) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(value) && mergeLists) {
|
||||
values[_outputFieldName].push(...value);
|
||||
} else {
|
||||
values[_outputFieldName].push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of Object.keys(values)) {
|
||||
if (!disableDotNotation) {
|
||||
set(newItem.json, key, values[key]);
|
||||
} else {
|
||||
newItem.json[key] = values[key];
|
||||
}
|
||||
}
|
||||
|
||||
returnData = newItem;
|
||||
} else {
|
||||
let newItems: IDataObject[] = items.map((item) => item.json);
|
||||
let pairedItem: IPairedItemData[] = [];
|
||||
const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string;
|
||||
|
||||
const fieldsToExclude = prepareFieldsArray(
|
||||
this.getNodeParameter('fieldsToExclude', 0, '') as string,
|
||||
'Fields To Exclude',
|
||||
);
|
||||
|
||||
const fieldsToInclude = prepareFieldsArray(
|
||||
this.getNodeParameter('fieldsToInclude', 0, '') as string,
|
||||
'Fields To Include',
|
||||
);
|
||||
|
||||
if (fieldsToExclude.length || fieldsToInclude.length) {
|
||||
newItems = newItems.reduce((acc, item, index) => {
|
||||
const newItem: IDataObject = {};
|
||||
let outputFields = Object.keys(item);
|
||||
|
||||
if (fieldsToExclude.length) {
|
||||
outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key));
|
||||
}
|
||||
if (fieldsToInclude.length) {
|
||||
outputFields = outputFields.filter((key) =>
|
||||
fieldsToInclude.length ? fieldsToInclude.includes(key) : true,
|
||||
);
|
||||
}
|
||||
|
||||
outputFields.forEach((key) => {
|
||||
newItem[key] = item[key];
|
||||
});
|
||||
|
||||
if (isEmpty(newItem)) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
pairedItem.push({ item: index });
|
||||
return acc.concat([newItem]);
|
||||
}, [] as IDataObject[]);
|
||||
} else {
|
||||
pairedItem = Array.from({ length: newItems.length }, (_, item) => ({
|
||||
item,
|
||||
}));
|
||||
}
|
||||
|
||||
const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
|
||||
|
||||
returnData = output;
|
||||
}
|
||||
|
||||
const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
|
||||
|
||||
if (includeBinaries) {
|
||||
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];
|
||||
|
||||
const aggregatedItems = pairedItems.map((item) => {
|
||||
return items[item.item];
|
||||
});
|
||||
|
||||
const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
|
||||
|
||||
addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
|
||||
}
|
||||
|
||||
if (Object.keys(notFoundedFields).length) {
|
||||
const hints: NodeExecutionHint[] = [];
|
||||
|
||||
for (const [field, values] of Object.entries(notFoundedFields)) {
|
||||
if (values.every((value) => !value)) {
|
||||
hints.push({
|
||||
message: `The field '${field}' wasn't found in any input item`,
|
||||
location: 'outputPane',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (hints.length) {
|
||||
this.addExecutionHints(...hints);
|
||||
}
|
||||
}
|
||||
|
||||
return [[returnData]];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#FF6D5A" clip-path="url(#a)"><path fill-rule="evenodd" d="M32 148c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12zm0 96c0-6.627 5.373-12 12-12h146c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H44c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/><path d="M74 76c0 6.627 5.373 12 12 12h116.217c17.673 0 32 14.327 32 32v56c0 26.978 10.272 51.557 27.119 70.039 5.055 5.545 5.055 14.377 0 19.922-16.847 18.482-27.119 43.061-27.119 70.039v56c0 17.673-14.327 32-32 32H86c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h116.217c44.183 0 80-35.817 80-80v-56c0-30.928 25.072-56 56-56a5.783 5.783 0 0 0 5.783-5.783v-36.434a5.783 5.783 0 0 0-5.783-5.783c-30.928 0-56-25.072-56-56v-56c0-44.183-35.817-80-80-80H86c-6.627 0-12 5.373-12 12z"/><path fill-rule="evenodd" d="M376 244c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H388c-6.627 0-12-5.373-12-12z" clip-rule="evenodd"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
@@ -0,0 +1,5 @@
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
|
||||
describe('Test Aggregate Node', () => {
|
||||
new NodeTestHarness().setupTests();
|
||||
});
|
||||
@@ -0,0 +1,228 @@
|
||||
{
|
||||
"name": "itemLists test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [-440, 260]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [-180, 260]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToAggregate": {
|
||||
"fieldToAggregate": [
|
||||
{
|
||||
"fieldToAggregate": "id",
|
||||
"renameField": true,
|
||||
"outputFieldName": "data"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"id": "d95ca3a3-fb43-4037-846e-b87103dec1a3",
|
||||
"name": "fields aggregate and rename",
|
||||
"type": "n8n-nodes-base.aggregate",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 0]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"aggregate": "aggregateAllItemData"
|
||||
},
|
||||
"id": "4c1bc7be-7611-418d-aad5-8642b1cc0781",
|
||||
"name": "aggregate all fields into list",
|
||||
"type": "n8n-nodes-base.aggregate",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 320]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"aggregate": "aggregateAllItemData",
|
||||
"include": "specifiedFields",
|
||||
"fieldsToInclude": ["id"]
|
||||
},
|
||||
"id": "951de23c-2018-437b-961e-8ae7d7fd1a82",
|
||||
"name": "aggregate selected fields into list",
|
||||
"type": "n8n-nodes-base.aggregate",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 500]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"aggregate": "aggregateAllItemData",
|
||||
"destinationFieldName": "output",
|
||||
"include": "allFieldsExcept",
|
||||
"fieldsToExclude": ["char"]
|
||||
},
|
||||
"id": "b62c02ee-5edb-473d-a755-7fb8700641fa",
|
||||
"name": "aggregate all fields except selected into list",
|
||||
"type": "n8n-nodes-base.aggregate",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 700]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"fields aggregate and rename": [
|
||||
{
|
||||
"json": {
|
||||
"data": [1, 2, 3, 4, 5]
|
||||
}
|
||||
}
|
||||
],
|
||||
"aggregate all fields into list": [
|
||||
{
|
||||
"json": {
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"aggregate selected fields into list": [
|
||||
{
|
||||
"json": {
|
||||
"data": [
|
||||
{
|
||||
"id": 1
|
||||
},
|
||||
{
|
||||
"id": 2
|
||||
},
|
||||
{
|
||||
"id": 3
|
||||
},
|
||||
{
|
||||
"id": 4
|
||||
},
|
||||
{
|
||||
"id": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"aggregate all fields except selected into list": [
|
||||
{
|
||||
"json": {
|
||||
"output": [
|
||||
{
|
||||
"id": 1
|
||||
},
|
||||
{
|
||||
"id": 2
|
||||
},
|
||||
{
|
||||
"id": 3
|
||||
},
|
||||
{
|
||||
"id": 4
|
||||
},
|
||||
{
|
||||
"id": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "fields aggregate and rename",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "aggregate all fields into list",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "aggregate selected fields into list",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "aggregate all fields except selected into list",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "9bf7c52b-b118-4dad-bfef-7db41828393b",
|
||||
"id": "105",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
|
||||
|
||||
type PartialBinaryData = Omit<IBinaryData, 'data'>;
|
||||
const isBinaryUniqueSetup = () => {
|
||||
const binaries: PartialBinaryData[] = [];
|
||||
return (binary: IBinaryData) => {
|
||||
for (const existingBinary of binaries) {
|
||||
if (
|
||||
existingBinary.mimeType === binary.mimeType &&
|
||||
existingBinary.fileType === binary.fileType &&
|
||||
existingBinary.fileSize === binary.fileSize &&
|
||||
existingBinary.fileExtension === binary.fileExtension
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
binaries.push({
|
||||
mimeType: binary.mimeType,
|
||||
fileType: binary.fileType,
|
||||
fileSize: binary.fileSize,
|
||||
fileExtension: binary.fileExtension,
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
};
|
||||
|
||||
export function addBinariesToItem(
|
||||
newItem: INodeExecutionData,
|
||||
items: INodeExecutionData[],
|
||||
uniqueOnly?: boolean,
|
||||
) {
|
||||
const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;
|
||||
|
||||
for (const item of items) {
|
||||
if (item.binary === undefined) continue;
|
||||
|
||||
for (const key of Object.keys(item.binary)) {
|
||||
if (!newItem.binary) newItem.binary = {};
|
||||
let binaryKey = key;
|
||||
const binary = item.binary[key];
|
||||
|
||||
if (isBinaryUnique && !isBinaryUnique(binary)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If the binary key already exists add a suffix to it
|
||||
let i = 1;
|
||||
while (newItem.binary[binaryKey] !== undefined) {
|
||||
binaryKey = `${key}_${i}`;
|
||||
i++;
|
||||
}
|
||||
|
||||
newItem.binary[binaryKey] = binary;
|
||||
}
|
||||
}
|
||||
|
||||
return newItem;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.limit",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.limit/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": ["Limit", "Remove", "Slice", "Transform", "Array", "List", "Item"],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import type {
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeType,
|
||||
INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
|
||||
export class Limit implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Limit',
|
||||
name: 'limit',
|
||||
icon: 'file:limit.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: 1,
|
||||
description: 'Restrict the number of items',
|
||||
defaults: {
|
||||
name: 'Limit',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Max Items',
|
||||
name: 'maxItems',
|
||||
type: 'number',
|
||||
typeOptions: {
|
||||
minValue: 1,
|
||||
},
|
||||
default: 1,
|
||||
description: 'If there are more items than this number, some are removed',
|
||||
},
|
||||
{
|
||||
displayName: 'Keep',
|
||||
name: 'keep',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'First Items',
|
||||
value: 'firstItems',
|
||||
},
|
||||
{
|
||||
name: 'Last Items',
|
||||
value: 'lastItems',
|
||||
},
|
||||
],
|
||||
default: 'firstItems',
|
||||
description: 'When removing items, whether to keep the ones at the start or the ending',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
let returnData = items;
|
||||
const maxItems = this.getNodeParameter('maxItems', 0) as number;
|
||||
const keep = this.getNodeParameter('keep', 0) as string;
|
||||
|
||||
if (maxItems > items.length) {
|
||||
return [returnData];
|
||||
}
|
||||
|
||||
if (keep === 'firstItems') {
|
||||
returnData = items.slice(0, maxItems);
|
||||
} else {
|
||||
returnData = items.slice(items.length - maxItems, items.length);
|
||||
}
|
||||
return [returnData];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#2FB67C" fill-rule="evenodd" clip-path="url(#a)" clip-rule="evenodd"><path d="M512 458c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12h-68c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm-140 0c0-6.627-5.373-12-12-12H12c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h68c6.627 0 12-5.373 12-12zm152-222c-6.627 0-12-5.373-12-12V30c0-6.627 5.373-12 12-12h24c6.627 0 12 5.373 12 12v194c0 6.627-5.373 12-12 12z"/><path d="M149.577 146.982c9.398-9.346 24.594-9.304 33.941.095L256 219.964l72.482-72.887c9.347-9.399 24.543-9.441 33.941-.095s9.441 24.543.095 33.941l-89.5 90a24 24 0 0 1-34.036 0l-89.5-90c-9.346-9.398-9.304-24.594.095-33.941M0 350c0-6.627 5.373-12 12-12h488c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H12c-6.627 0-12-5.373-12-12z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.1 KiB |
@@ -0,0 +1,5 @@
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
|
||||
describe('Test Limit Node', () => {
|
||||
new NodeTestHarness().setupTests();
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"name": "itemLists test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "bd7af0bb-de39-44b4-ac11-eb1d22f5e8d7",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [260, 180]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"entry": 1
|
||||
},
|
||||
{
|
||||
"entry": 2
|
||||
},
|
||||
{
|
||||
"entry": 3
|
||||
},
|
||||
{
|
||||
"entry": 4
|
||||
},
|
||||
{
|
||||
"entry": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "21185d7a-f0c1-49a0-9c2d-f0f198ceea7e",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [520, 180]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"maxItems": 1
|
||||
},
|
||||
"id": "7cc02cc4-1f5f-489a-81e2-4c96b3bdf221",
|
||||
"name": "Item Lists limit first",
|
||||
"type": "n8n-nodes-base.limit",
|
||||
"typeVersion": 1,
|
||||
"position": [740, 80]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"keep": "lastItems",
|
||||
"maxItems": 1
|
||||
},
|
||||
"id": "2bf79d53-7a0b-4716-aa09-55ad43d306ae",
|
||||
"name": "Item Lists limit last",
|
||||
"type": "n8n-nodes-base.limit",
|
||||
"typeVersion": 1,
|
||||
"position": [740, 300]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Item Lists limit first": [
|
||||
{
|
||||
"json": {
|
||||
"entry": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"Item Lists limit last": [
|
||||
{
|
||||
"json": {
|
||||
"entry": 5
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Item Lists limit first",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists limit last",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "5036d554-1ba4-4b5f-ba9f-1de6df09e807",
|
||||
"id": "105",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.removeDuplicates",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": [
|
||||
"Dedupe",
|
||||
"Deduplicate",
|
||||
"Duplicates",
|
||||
"Remove",
|
||||
"Unique",
|
||||
"Transform",
|
||||
"Array",
|
||||
"List",
|
||||
"Item"
|
||||
],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow';
|
||||
import { VersionedNodeType } from 'n8n-workflow';
|
||||
|
||||
import { RemoveDuplicatesV1 } from './v1/RemoveDuplicatesV1.node';
|
||||
import { RemoveDuplicatesV2 } from './v2/RemoveDuplicatesV2.node';
|
||||
export class RemoveDuplicates extends VersionedNodeType {
|
||||
constructor() {
|
||||
const baseDescription: INodeTypeBaseDescription = {
|
||||
displayName: 'Remove Duplicates',
|
||||
name: 'removeDuplicates',
|
||||
icon: 'file:removeDuplicates.svg',
|
||||
group: ['transform'],
|
||||
defaultVersion: 2,
|
||||
description: 'Delete items with matching field values',
|
||||
};
|
||||
|
||||
const nodeVersions: IVersionedNodeType['nodeVersions'] = {
|
||||
1: new RemoveDuplicatesV1(baseDescription),
|
||||
1.1: new RemoveDuplicatesV1(baseDescription),
|
||||
2: new RemoveDuplicatesV2(baseDescription),
|
||||
};
|
||||
|
||||
super(nodeVersions, baseDescription);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#54B8C9" clip-path="url(#a)"><path d="M134.097 111h38.829v32.508H138.16v34.635h-32.508v-38.699c0-15.709 12.735-28.444 28.445-28.444m77.658 32.508V111h77.657v32.508zm116.486 0V111h77.658v32.508zm116.487 0V111h38.829c15.71 0 28.445 12.735 28.445 28.444v38.699h-32.508v-34.635zm34.766 73.238h32.508v38.698c0 15.71-12.735 28.445-28.445 28.445h-38.829v-32.508h34.766zM0 244.537C0 229.329 12.735 217 28.444 217h349.461c15.709 0 28.444 12.329 28.444 27.537v129.815c0 15.208-12.735 27.537-28.444 27.537H28.445C12.734 401.889 0 389.56 0 374.352z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h512v512H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 715 B |
@@ -0,0 +1,65 @@
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
import type { INode } from 'n8n-workflow';
|
||||
|
||||
import { validateInputData } from '../utils';
|
||||
|
||||
describe('Test Remove Duplicates Node', () => {
|
||||
new NodeTestHarness().setupTests();
|
||||
});
|
||||
|
||||
describe('Test Remove Duplicates Node, validateInputData util', () => {
|
||||
test('Should throw error for version 1', () => {
|
||||
expect(() =>
|
||||
validateInputData(
|
||||
{
|
||||
name: 'Remove Duplicates',
|
||||
type: 'n8n-nodes-base.removeDuplicates',
|
||||
typeVersion: 1,
|
||||
} as INode,
|
||||
[
|
||||
{ json: { country: 'uk' } },
|
||||
{ json: { country: 'us' } },
|
||||
{ json: { country: 'uk' } },
|
||||
{ json: { country: null } },
|
||||
],
|
||||
['country'],
|
||||
false,
|
||||
),
|
||||
).toThrow("'country' isn't always the same type");
|
||||
});
|
||||
|
||||
test('Should ignore null values and not throw error for version grater than 1', () => {
|
||||
expect(() =>
|
||||
validateInputData(
|
||||
{
|
||||
name: 'Remove Duplicates',
|
||||
type: 'n8n-nodes-base.removeDuplicates',
|
||||
typeVersion: 1.1,
|
||||
} as INode,
|
||||
[
|
||||
{ json: { country: 'uk' } },
|
||||
{ json: { country: 'us' } },
|
||||
{ json: { country: 'uk' } },
|
||||
{ json: { country: null } },
|
||||
],
|
||||
['country'],
|
||||
false,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
test('Should throw error for different types, version grater than 1', () => {
|
||||
expect(() =>
|
||||
validateInputData(
|
||||
{
|
||||
name: 'Remove Duplicates',
|
||||
type: 'n8n-nodes-base.removeDuplicates',
|
||||
typeVersion: 1.1,
|
||||
} as INode,
|
||||
[{ json: { id: 1 } }, { json: { id: '1' } }, { json: { id: 2 } }, { json: { id: null } }],
|
||||
['id'],
|
||||
false,
|
||||
),
|
||||
).toThrow("'id' isn't always the same type");
|
||||
});
|
||||
});
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
{
|
||||
"name": "Remove Duplicates",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "a4da10da-991f-48ab-b873-9d633a11311f",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [760, 420]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 98
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [980, 420]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "c336939c-062e-475e-ba7c-8601e3662e8c",
|
||||
"name": "Remove Duplicates (All Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [1200, 260]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "selectedFields",
|
||||
"fieldsToCompare": "name",
|
||||
"options": {}
|
||||
},
|
||||
"id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80",
|
||||
"name": "Remove Duplicates (Selected Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [1200, 420]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "allFieldsExcept",
|
||||
"fieldsToExclude": "age",
|
||||
"options": {}
|
||||
},
|
||||
"id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b",
|
||||
"name": "Remove Duplicates (Except Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [1200, 580]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba",
|
||||
"name": "All Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 260]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "b5c5c946-2e96-451b-b9a6-78e478504d6c",
|
||||
"name": "Selected Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 420]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "afb92bc5-beba-4b0a-aefb-b47cc708a125",
|
||||
"name": "Except Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 580]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "allFieldsExcept",
|
||||
"fieldsToExclude": "age",
|
||||
"options": {
|
||||
"removeOtherFields": true
|
||||
}
|
||||
},
|
||||
"id": "f92c5533-ac29-476c-aebb-4849ddd22110",
|
||||
"name": "Remove Duplicates (Remove)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [1200, 760]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89",
|
||||
"name": "Remove",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 760]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Code": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 98
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"All Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 98
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Selected Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Except Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Remove": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Remove Duplicates (All Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Selected Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Except Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Remove)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (All Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "All Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Selected Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Selected Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Except Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Except Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Remove)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Remove",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {
|
||||
"executionOrder": "v1"
|
||||
},
|
||||
"versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3",
|
||||
"id": "74gMYOHjjPArZg4q",
|
||||
"meta": {
|
||||
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import get from 'lodash/get';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import lt from 'lodash/lt';
|
||||
import pick from 'lodash/pick';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
import type { IExecuteFunctions, INode, INodeExecutionData } from 'n8n-workflow';
|
||||
|
||||
import { compareItems, flattenKeys } from '@utils/utilities';
|
||||
|
||||
import { prepareFieldsArray } from '../utils/utils';
|
||||
|
||||
export const validateInputData = (
|
||||
node: INode,
|
||||
items: INodeExecutionData[],
|
||||
keysToCompare: string[],
|
||||
disableDotNotation: boolean,
|
||||
) => {
|
||||
for (const key of keysToCompare) {
|
||||
let type: any = undefined;
|
||||
for (const [i, item] of items.entries()) {
|
||||
if (key === '') {
|
||||
throw new NodeOperationError(node, 'Name of field to compare is blank');
|
||||
}
|
||||
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
|
||||
if (value === null && node.typeVersion > 1) continue;
|
||||
|
||||
if (value === undefined && disableDotNotation && key.includes('.')) {
|
||||
throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
|
||||
description:
|
||||
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
|
||||
});
|
||||
} else if (value === undefined) {
|
||||
throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
|
||||
}
|
||||
if (type !== undefined && value !== undefined && type !== typeof value) {
|
||||
const description =
|
||||
'The type of this field varies between items' +
|
||||
(node.typeVersion > 1
|
||||
? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
|
||||
: '');
|
||||
throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
|
||||
description,
|
||||
});
|
||||
} else {
|
||||
type = typeof value;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export function removeDuplicateInputItems(context: IExecuteFunctions, items: INodeExecutionData[]) {
|
||||
const compare = context.getNodeParameter('compare', 0) as string;
|
||||
const disableDotNotation = context.getNodeParameter(
|
||||
'options.disableDotNotation',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
const removeOtherFields = context.getNodeParameter(
|
||||
'options.removeOtherFields',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
|
||||
let keys = disableDotNotation
|
||||
? Object.keys(items[0].json)
|
||||
: Object.keys(flattenKeys(items[0].json));
|
||||
|
||||
for (const item of items) {
|
||||
const itemKeys = disableDotNotation
|
||||
? Object.keys(item.json)
|
||||
: Object.keys(flattenKeys(item.json));
|
||||
for (const key of itemKeys) {
|
||||
if (!keys.includes(key)) {
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (compare === 'allFieldsExcept') {
|
||||
const fieldsToExclude = prepareFieldsArray(
|
||||
context.getNodeParameter('fieldsToExclude', 0, '') as string,
|
||||
'Fields To Exclude',
|
||||
);
|
||||
|
||||
if (!fieldsToExclude.length) {
|
||||
throw new NodeOperationError(
|
||||
context.getNode(),
|
||||
'No fields specified. Please add a field to exclude from comparison',
|
||||
);
|
||||
}
|
||||
if (!disableDotNotation) {
|
||||
keys = Object.keys(flattenKeys(items[0].json));
|
||||
}
|
||||
keys = keys.filter((key) => !fieldsToExclude.includes(key));
|
||||
}
|
||||
if (compare === 'selectedFields') {
|
||||
const fieldsToCompare = prepareFieldsArray(
|
||||
context.getNodeParameter('fieldsToCompare', 0, '') as string,
|
||||
'Fields To Compare',
|
||||
);
|
||||
if (!fieldsToCompare.length) {
|
||||
throw new NodeOperationError(
|
||||
context.getNode(),
|
||||
'No fields specified. Please add a field to compare on',
|
||||
);
|
||||
}
|
||||
if (!disableDotNotation) {
|
||||
keys = Object.keys(flattenKeys(items[0].json));
|
||||
}
|
||||
keys = fieldsToCompare.map((key) => key.trim());
|
||||
}
|
||||
|
||||
// This solution is O(nlogn)
|
||||
// add original index to the items
|
||||
const newItems = items.map(
|
||||
(item, index) =>
|
||||
({
|
||||
json: { ...item.json, __INDEX: index },
|
||||
pairedItem: { item: index },
|
||||
}) as INodeExecutionData,
|
||||
);
|
||||
//sort items using the compare keys
|
||||
newItems.sort((a, b) => {
|
||||
let result = 0;
|
||||
|
||||
for (const key of keys) {
|
||||
let equal;
|
||||
if (!disableDotNotation) {
|
||||
equal = isEqual(get(a.json, key), get(b.json, key));
|
||||
} else {
|
||||
equal = isEqual(a.json[key], b.json[key]);
|
||||
}
|
||||
if (!equal) {
|
||||
let lessThan;
|
||||
if (!disableDotNotation) {
|
||||
lessThan = lt(get(a.json, key), get(b.json, key));
|
||||
} else {
|
||||
lessThan = lt(a.json[key], b.json[key]);
|
||||
}
|
||||
result = lessThan ? -1 : 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
validateInputData(context.getNode(), newItems, keys, disableDotNotation);
|
||||
|
||||
// collect the original indexes of items to be removed
|
||||
const removedIndexes: number[] = [];
|
||||
let temp = newItems[0];
|
||||
for (let index = 1; index < newItems.length; index++) {
|
||||
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
|
||||
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
|
||||
} else {
|
||||
temp = newItems[index];
|
||||
}
|
||||
}
|
||||
let updatedItems: INodeExecutionData[] = items.filter(
|
||||
(_, index) => !removedIndexes.includes(index),
|
||||
);
|
||||
|
||||
if (removeOtherFields) {
|
||||
updatedItems = updatedItems.map((item, index) => ({
|
||||
json: pick(item.json, ...keys),
|
||||
pairedItem: { item: index },
|
||||
}));
|
||||
}
|
||||
return [updatedItems];
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
import type {
|
||||
INodeTypeBaseDescription,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeType,
|
||||
INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { removeDuplicateInputItems } from '../utils';
|
||||
|
||||
const versionDescription: INodeTypeDescription = {
|
||||
displayName: 'Remove Duplicates',
|
||||
name: 'removeDuplicates',
|
||||
icon: 'file:removeDuplicates.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: [1, 1.1],
|
||||
description: 'Delete items with matching field values',
|
||||
defaults: {
|
||||
name: 'Remove Duplicates',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Compare',
|
||||
name: 'compare',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'All Fields',
|
||||
value: 'allFields',
|
||||
},
|
||||
{
|
||||
name: 'All Fields Except',
|
||||
value: 'allFieldsExcept',
|
||||
},
|
||||
{
|
||||
name: 'Selected Fields',
|
||||
value: 'selectedFields',
|
||||
},
|
||||
],
|
||||
default: 'allFields',
|
||||
description: 'The fields of the input items to compare to see if they are the same',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Exclude',
|
||||
name: 'fieldsToExclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to exclude from the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['allFieldsExcept'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Compare',
|
||||
name: 'fieldsToCompare',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to add to the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['selectedFields'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['allFieldsExcept', 'selectedFields'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Remove Other Fields',
|
||||
name: 'removeOtherFields',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
export class RemoveDuplicatesV1 implements INodeType {
|
||||
description: INodeTypeDescription;
|
||||
|
||||
constructor(baseDescription: INodeTypeBaseDescription) {
|
||||
this.description = {
|
||||
...baseDescription,
|
||||
...versionDescription,
|
||||
};
|
||||
}
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
return removeDuplicateInputItems(this, items);
|
||||
}
|
||||
}
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
import type { INodeProperties } from 'n8n-workflow';
|
||||
const operationOptions = [
|
||||
{
|
||||
name: 'Remove Items Repeated Within Current Input',
|
||||
value: 'removeDuplicateInputItems',
|
||||
description: 'Remove duplicates from incoming items',
|
||||
action: 'Remove items repeated within current input',
|
||||
},
|
||||
{
|
||||
name: 'Remove Items Processed in Previous Executions',
|
||||
value: 'removeItemsSeenInPreviousExecutions',
|
||||
description: 'Deduplicate items already seen in previous executions',
|
||||
action: 'Remove items processed in previous executions',
|
||||
},
|
||||
{
|
||||
name: 'Clear Deduplication History',
|
||||
value: 'clearDeduplicationHistory',
|
||||
description: 'Wipe the store of previous items',
|
||||
action: 'Clear deduplication history',
|
||||
},
|
||||
];
|
||||
const compareOptions = [
|
||||
{
|
||||
name: 'All Fields',
|
||||
value: 'allFields',
|
||||
},
|
||||
{
|
||||
name: 'All Fields Except',
|
||||
value: 'allFieldsExcept',
|
||||
},
|
||||
{
|
||||
name: 'Selected Fields',
|
||||
value: 'selectedFields',
|
||||
},
|
||||
];
|
||||
const logicOptions = [
|
||||
{
|
||||
name: 'Value Is New',
|
||||
value: 'removeItemsWithAlreadySeenKeyValues',
|
||||
description: 'Remove all input items with values matching those already processed',
|
||||
},
|
||||
{
|
||||
name: 'Value Is Higher than Any Previous Value',
|
||||
value: 'removeItemsUpToStoredIncrementalKey',
|
||||
description:
|
||||
'Works with incremental values, removes all input items with values up to the stored value',
|
||||
},
|
||||
{
|
||||
name: 'Value Is a Date Later than Any Previous Date',
|
||||
value: 'removeItemsUpToStoredDate',
|
||||
description:
|
||||
'Works with date values, removes all input items with values up to the stored date',
|
||||
},
|
||||
];
|
||||
const manageDatabaseModeOptions = [
|
||||
{
|
||||
name: 'Clean Database',
|
||||
value: 'cleanDatabase',
|
||||
description: 'Clear all values stored for a key in the database',
|
||||
},
|
||||
];
|
||||
|
||||
export const removeDuplicatesNodeFields: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Operation',
|
||||
name: 'operation',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
options: operationOptions,
|
||||
default: 'removeDuplicateInputItems',
|
||||
},
|
||||
{
|
||||
displayName: 'Compare',
|
||||
name: 'compare',
|
||||
type: 'options',
|
||||
options: compareOptions,
|
||||
default: 'allFields',
|
||||
description: 'The fields of the input items to compare to see if they are the same',
|
||||
displayOptions: {
|
||||
show: {
|
||||
operation: ['removeDuplicateInputItems'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Exclude',
|
||||
name: 'fieldsToExclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to exclude from the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['allFieldsExcept'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Compare',
|
||||
name: 'fieldsToCompare',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to add to the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['selectedFields'],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// ----------------------------------
|
||||
{
|
||||
displayName: 'Keep Items Where',
|
||||
name: 'logic',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
options: logicOptions,
|
||||
default: 'removeItemsWithAlreadySeenKeyValues',
|
||||
description:
|
||||
'How to select input items to remove by comparing them with key values previously processed',
|
||||
displayOptions: {
|
||||
show: {
|
||||
operation: ['removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Value to Dedupe On',
|
||||
name: 'dedupeValue',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Use an input field (or a combination of fields) that has a unique ID value',
|
||||
hint: 'The input field value to compare between items',
|
||||
placeholder: 'e.g. ID',
|
||||
required: true,
|
||||
displayOptions: {
|
||||
show: {
|
||||
logic: ['removeItemsWithAlreadySeenKeyValues'],
|
||||
'/operation': ['removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Value to Dedupe On',
|
||||
name: 'incrementalDedupeValue',
|
||||
type: 'number',
|
||||
default: '',
|
||||
description: 'Use an input field (or a combination of fields) that has an incremental value',
|
||||
hint: 'The input field value to compare between items, an incremental value is expected',
|
||||
placeholder: 'e.g. ID',
|
||||
displayOptions: {
|
||||
show: {
|
||||
logic: ['removeItemsUpToStoredIncrementalKey'],
|
||||
'/operation': ['removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Value to Dedupe On',
|
||||
name: 'dateDedupeValue',
|
||||
type: 'dateTime',
|
||||
default: '',
|
||||
description: 'Use an input field that has a date value in ISO format',
|
||||
hint: 'The input field value to compare between items, a date is expected',
|
||||
placeholder: ' e.g. 2024-08-09T13:44:16Z',
|
||||
displayOptions: {
|
||||
show: {
|
||||
logic: ['removeItemsUpToStoredDate'],
|
||||
'/operation': ['removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Mode',
|
||||
name: 'mode',
|
||||
type: 'options',
|
||||
default: 'cleanDatabase',
|
||||
description:
|
||||
'How you want to modify the key values stored on the database. None of these modes removes input items.',
|
||||
displayOptions: {
|
||||
show: {
|
||||
operation: ['clearDeduplicationHistory'],
|
||||
},
|
||||
},
|
||||
options: manageDatabaseModeOptions,
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
displayOptions: {
|
||||
show: {
|
||||
operation: [
|
||||
'removeDuplicateInputItems',
|
||||
'removeItemsSeenInPreviousExecutions',
|
||||
'clearDeduplicationHistory',
|
||||
],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/operation': ['removeDuplicateInputItems'],
|
||||
},
|
||||
hide: {
|
||||
'/compare': ['allFields'],
|
||||
},
|
||||
},
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Remove Other Fields',
|
||||
name: 'removeOtherFields',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/operation': ['removeDuplicateInputItems'],
|
||||
},
|
||||
hide: {
|
||||
'/compare': ['allFields'],
|
||||
},
|
||||
},
|
||||
description:
|
||||
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
|
||||
},
|
||||
{
|
||||
displayName: 'Scope',
|
||||
name: 'scope',
|
||||
type: 'options',
|
||||
default: 'node',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/operation': ['clearDeduplicationHistory', 'removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
description:
|
||||
'If set to ‘workflow,’ key values will be shared across all nodes in the workflow. If set to ‘node,’ key values will be specific to this node.',
|
||||
options: [
|
||||
{
|
||||
name: 'Workflow',
|
||||
value: 'workflow',
|
||||
description: 'Deduplication info will be shared by all the nodes in the workflow',
|
||||
},
|
||||
{
|
||||
name: 'Node',
|
||||
value: 'node',
|
||||
description: 'Deduplication info will be stored only for this node',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'History Size',
|
||||
name: 'historySize',
|
||||
type: 'number',
|
||||
default: 10000,
|
||||
hint: 'The max number of past items to store for deduplication',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/logic': ['removeItemsWithAlreadySeenKeyValues'],
|
||||
'/operation': ['removeItemsSeenInPreviousExecutions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,271 @@
|
||||
import { NodeConnectionTypes, NodeOperationError, tryToParseDateTime } from 'n8n-workflow';
|
||||
import type {
|
||||
INodeTypeBaseDescription,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeType,
|
||||
INodeTypeDescription,
|
||||
DeduplicationScope,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { removeDuplicatesNodeFields } from './RemoveDuplicatesV2.description';
|
||||
import { removeDuplicateInputItems } from '../utils';
|
||||
|
||||
const versionDescription: INodeTypeDescription = {
|
||||
displayName: 'Remove Duplicates',
|
||||
name: 'removeDuplicates',
|
||||
icon: 'file:removeDuplicates.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: [2],
|
||||
description: 'Delete items with matching field values',
|
||||
defaults: {
|
||||
name: 'Remove Duplicates',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
outputNames: ['Kept', 'Discarded'],
|
||||
hints: [
|
||||
{
|
||||
message: 'The dedupe key set in “Value to Dedupe On” has no value',
|
||||
displayCondition:
|
||||
'={{ $parameter["operation"] === "removeItemsSeenInPreviousExecutions" && ($parameter["logic"] === "removeItemsWithAlreadySeenKeyValues" && $parameter["dedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredIncrementalKey" && $parameter["incrementalDedupeValue"] === undefined) || ($parameter["logic"] === "removeItemsUpToStoredDate" && $parameter["dateDedupeValue"] === undefined) }}',
|
||||
whenToDisplay: 'beforeExecution',
|
||||
location: 'outputPane',
|
||||
},
|
||||
],
|
||||
properties: [...removeDuplicatesNodeFields],
|
||||
};
|
||||
export class RemoveDuplicatesV2 implements INodeType {
|
||||
description: INodeTypeDescription;
|
||||
|
||||
constructor(baseDescription: INodeTypeBaseDescription) {
|
||||
this.description = {
|
||||
...baseDescription,
|
||||
...versionDescription,
|
||||
};
|
||||
}
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
const operation = this.getNodeParameter('operation', 0);
|
||||
const returnData: INodeExecutionData[][] = [];
|
||||
const DEFAULT_MAX_ENTRIES = 10000;
|
||||
try {
|
||||
switch (operation) {
|
||||
case 'removeDuplicateInputItems': {
|
||||
return removeDuplicateInputItems(this, items);
|
||||
}
|
||||
case 'removeItemsSeenInPreviousExecutions': {
|
||||
const logic = this.getNodeParameter('logic', 0);
|
||||
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
|
||||
|
||||
if (logic === 'removeItemsWithAlreadySeenKeyValues') {
|
||||
if (!['node', 'workflow'].includes(scope)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
|
||||
);
|
||||
}
|
||||
|
||||
let checkValue: string;
|
||||
const itemMapping: {
|
||||
[key: string]: INodeExecutionData[];
|
||||
} = {};
|
||||
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
||||
checkValue = this.getNodeParameter('dedupeValue', itemIndex, '')?.toString() ?? '';
|
||||
if (itemMapping[checkValue]) {
|
||||
itemMapping[checkValue].push(items[itemIndex]);
|
||||
} else {
|
||||
itemMapping[checkValue] = [items[itemIndex]];
|
||||
}
|
||||
}
|
||||
|
||||
const maxEntries = this.getNodeParameter(
|
||||
'options.historySize',
|
||||
0,
|
||||
DEFAULT_MAX_ENTRIES,
|
||||
) as number;
|
||||
const maxEntriesNum = Number(maxEntries);
|
||||
|
||||
const currentProcessedDataCount = await this.helpers.getProcessedDataCount(scope, {
|
||||
mode: 'entries',
|
||||
maxEntries,
|
||||
});
|
||||
if (currentProcessedDataCount + items.length > maxEntriesNum) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'The number of items to be processed exceeds the maximum history size. Please increase the history size or reduce the number of items to be processed.',
|
||||
);
|
||||
}
|
||||
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
|
||||
Object.keys(itemMapping),
|
||||
scope,
|
||||
{ mode: 'entries', maxEntries },
|
||||
);
|
||||
const processedDataCount = await this.helpers.getProcessedDataCount(scope, {
|
||||
mode: 'entries',
|
||||
maxEntries,
|
||||
});
|
||||
returnData.push(
|
||||
itemsProcessed.new
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
itemsProcessed.processed
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
);
|
||||
|
||||
if (maxEntriesNum > 0 && processedDataCount / maxEntriesNum > 0.5) {
|
||||
this.addExecutionHints({
|
||||
message: `Some duplicates may be not be removed since you're approaching the maximum history size (${maxEntriesNum} items). You can raise this limit using the ‘history size’ option.`,
|
||||
location: 'outputPane',
|
||||
});
|
||||
}
|
||||
return returnData;
|
||||
} else if (logic === 'removeItemsUpToStoredIncrementalKey') {
|
||||
if (!['node', 'workflow'].includes(scope)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
|
||||
);
|
||||
}
|
||||
|
||||
let parsedIncrementalKey: number;
|
||||
const itemMapping: {
|
||||
[key: string]: INodeExecutionData[];
|
||||
} = {};
|
||||
|
||||
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
||||
const incrementalKey = this.getNodeParameter('incrementalDedupeValue', itemIndex, '');
|
||||
if (!incrementalKey?.toString()) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'The `Value to Dedupe` On is empty. Please provide a value.',
|
||||
);
|
||||
}
|
||||
parsedIncrementalKey = Number(incrementalKey);
|
||||
if (isNaN(parsedIncrementalKey)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The value '${incrementalKey}' is not a number. Please provide a number.`,
|
||||
);
|
||||
}
|
||||
if (itemMapping[parsedIncrementalKey]) {
|
||||
itemMapping[parsedIncrementalKey].push(items[itemIndex]);
|
||||
} else {
|
||||
itemMapping[parsedIncrementalKey] = [items[itemIndex]];
|
||||
}
|
||||
}
|
||||
|
||||
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
|
||||
Object.keys(itemMapping),
|
||||
scope,
|
||||
{ mode: 'latestIncrementalKey' },
|
||||
);
|
||||
|
||||
returnData.push(
|
||||
itemsProcessed.new
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
itemsProcessed.processed
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
);
|
||||
|
||||
return returnData;
|
||||
} else if (logic === 'removeItemsUpToStoredDate') {
|
||||
if (!['node', 'workflow'].includes(scope)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The scope '${scope}' is not supported. Please select either "node" or "workflow".`,
|
||||
);
|
||||
}
|
||||
|
||||
let checkValue: string;
|
||||
const itemMapping: {
|
||||
[key: string]: INodeExecutionData[];
|
||||
} = {};
|
||||
|
||||
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
||||
checkValue =
|
||||
this.getNodeParameter('dateDedupeValue', itemIndex, '')?.toString() ?? '';
|
||||
if (!checkValue) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'The `Value to Dedupe` On is empty. Please provide a value.',
|
||||
);
|
||||
}
|
||||
try {
|
||||
tryToParseDateTime(checkValue);
|
||||
} catch (error) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The value '${checkValue}' is not a valid date. Please provide a valid date.`,
|
||||
);
|
||||
}
|
||||
if (itemMapping[checkValue]) {
|
||||
itemMapping[checkValue].push(items[itemIndex]);
|
||||
} else {
|
||||
itemMapping[checkValue] = [items[itemIndex]];
|
||||
}
|
||||
}
|
||||
const itemsProcessed = await this.helpers.checkProcessedAndRecord(
|
||||
Object.keys(itemMapping),
|
||||
scope,
|
||||
{ mode: 'latestDate' },
|
||||
);
|
||||
|
||||
returnData.push(
|
||||
itemsProcessed.new
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
itemsProcessed.processed
|
||||
.map((key) => {
|
||||
return itemMapping[key];
|
||||
})
|
||||
.flat(),
|
||||
);
|
||||
|
||||
return returnData;
|
||||
} else {
|
||||
return [items];
|
||||
}
|
||||
}
|
||||
case 'clearDeduplicationHistory': {
|
||||
const mode = this.getNodeParameter('mode', 0) as string;
|
||||
if (mode === 'updateKeyValuesInDatabase') {
|
||||
} else if (mode === 'deleteKeyValuesFromDatabase') {
|
||||
} else if (mode === 'cleanDatabase') {
|
||||
const scope = this.getNodeParameter('options.scope', 0, 'node') as DeduplicationScope;
|
||||
await this.helpers.clearAllProcessedItems(scope, {
|
||||
mode: 'entries',
|
||||
});
|
||||
}
|
||||
|
||||
return [items];
|
||||
}
|
||||
default: {
|
||||
return [items];
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (this.continueOnFail()) {
|
||||
returnData.push([{ json: this.getInputData(0)[0].json, error }]);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return returnData;
|
||||
}
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type { IExecuteFunctions, INodeExecutionData, INodeTypeBaseDescription } from 'n8n-workflow';
|
||||
|
||||
import { RemoveDuplicatesV2 } from '../RemoveDuplicatesV2.node';
|
||||
|
||||
describe('RemoveDuplicatesV2', () => {
|
||||
let node: RemoveDuplicatesV2;
|
||||
let executeFunctions: IExecuteFunctions;
|
||||
|
||||
beforeEach(() => {
|
||||
const baseDescription: INodeTypeBaseDescription = {
|
||||
displayName: 'Remove Duplicates',
|
||||
name: 'removeDuplicates',
|
||||
icon: 'file:removeDuplicates.svg',
|
||||
group: ['transform'],
|
||||
description: 'Delete items with matching field values',
|
||||
};
|
||||
node = new RemoveDuplicatesV2(baseDescription);
|
||||
executeFunctions = mock<IExecuteFunctions>();
|
||||
executeFunctions.helpers = {
|
||||
checkProcessedAndRecord: jest.fn(),
|
||||
clearAllProcessedItems: jest.fn(),
|
||||
} as any;
|
||||
executeFunctions.getInputData = jest.fn();
|
||||
executeFunctions.getNodeParameter = jest.fn();
|
||||
});
|
||||
|
||||
it('should Remove items repeated within current input based on all fields', async () => {
|
||||
const items: INodeExecutionData[] = [
|
||||
{ json: { id: 1, name: 'John' } },
|
||||
{ json: { id: 2, name: 'Jane' } },
|
||||
{ json: { id: 1, name: 'John' } },
|
||||
];
|
||||
|
||||
(executeFunctions.getInputData as jest.Mock<any>).mockReturnValue(items);
|
||||
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
|
||||
(paramName: string) => {
|
||||
if (paramName === 'operation') return 'removeDuplicateInputItems';
|
||||
if (paramName === 'compare') return 'allFields';
|
||||
return undefined;
|
||||
},
|
||||
);
|
||||
|
||||
const result = await node.execute.call(executeFunctions);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toHaveLength(2);
|
||||
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
|
||||
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
|
||||
});
|
||||
|
||||
it('should Remove items repeated within current input based on selected fields', async () => {
|
||||
const items: INodeExecutionData[] = [
|
||||
{ json: { id: 1, name: 'John' } },
|
||||
{ json: { id: 2, name: 'Jane' } },
|
||||
{ json: { id: 1, name: 'Doe' } },
|
||||
];
|
||||
|
||||
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
|
||||
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
|
||||
(paramName: string) => {
|
||||
if (paramName === 'operation') return 'removeDuplicateInputItems';
|
||||
if (paramName === 'compare') return 'selectedFields';
|
||||
if (paramName === 'fieldsToCompare') return 'id';
|
||||
return undefined;
|
||||
},
|
||||
);
|
||||
|
||||
const result = await node.execute.call(executeFunctions);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toHaveLength(2);
|
||||
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
|
||||
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
|
||||
});
|
||||
|
||||
it('should remove items seen in previous executions', async () => {
|
||||
const items: INodeExecutionData[] = [
|
||||
{ json: { id: 1, name: 'John' } },
|
||||
{ json: { id: 2, name: 'Jane' } },
|
||||
{ json: { id: 3, name: 'Doe' } },
|
||||
];
|
||||
|
||||
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
|
||||
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
|
||||
(paramName: string, itemIndex: number) => {
|
||||
if (paramName === 'operation') return 'removeItemsSeenInPreviousExecutions';
|
||||
if (paramName === 'logic') return 'removeItemsWithAlreadySeenKeyValues';
|
||||
if (paramName === 'dedupeValue' && itemIndex === 0) return 1;
|
||||
if (paramName === 'dedupeValue' && itemIndex === 1) return 2;
|
||||
if (paramName === 'dedupeValue' && itemIndex === 2) return 3;
|
||||
if (paramName === 'options.scope') return 'node';
|
||||
if (paramName === 'options.historySize') return 10;
|
||||
},
|
||||
);
|
||||
executeFunctions.helpers.getProcessedDataCount = jest.fn().mockReturnValue(3);
|
||||
(executeFunctions.helpers.checkProcessedAndRecord as jest.Mock).mockReturnValue({
|
||||
new: [1, 3],
|
||||
processed: [2],
|
||||
});
|
||||
|
||||
const result = await node.execute.call(executeFunctions);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toHaveLength(2);
|
||||
expect(result[1]).toHaveLength(1);
|
||||
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
|
||||
expect(result[0][1].json).toEqual({ id: 3, name: 'Doe' });
|
||||
});
|
||||
|
||||
it('should clean database when managing key values', async () => {
|
||||
const items: INodeExecutionData[] = [
|
||||
{ json: { id: 1, name: 'John' } },
|
||||
{ json: { id: 2, name: 'Jane' } },
|
||||
];
|
||||
|
||||
(executeFunctions.getInputData as jest.Mock<any, any>).mockReturnValue(items);
|
||||
(executeFunctions.getNodeParameter as jest.Mock<any, any>).mockImplementation(
|
||||
(paramName: string) => {
|
||||
if (paramName === 'operation') return 'clearDeduplicationHistory';
|
||||
if (paramName === 'mode') return 'cleanDatabase';
|
||||
if (paramName === 'options.scope') return 'node';
|
||||
return undefined;
|
||||
},
|
||||
);
|
||||
|
||||
const result = await node.execute.call(executeFunctions);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toHaveLength(2);
|
||||
expect(result[0][0].json).toEqual({ id: 1, name: 'John' });
|
||||
expect(result[0][1].json).toEqual({ id: 2, name: 'Jane' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.sort",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.sort/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": ["Sort", "Order", "Transform", "Array", "List", "Item", "Random"],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
import get from 'lodash/get';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import lt from 'lodash/lt';
|
||||
import {
|
||||
NodeOperationError,
|
||||
type IDataObject,
|
||||
type IExecuteFunctions,
|
||||
type INodeExecutionData,
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
NodeConnectionTypes,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { shuffleArray } from '@utils/utilities';
|
||||
|
||||
import { sortByCode } from './utils';
|
||||
|
||||
export class Sort implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Sort',
|
||||
name: 'sort',
|
||||
icon: 'file:sort.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: 1,
|
||||
description: 'Change items order',
|
||||
defaults: {
|
||||
name: 'Sort',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Type',
|
||||
name: 'type',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Simple',
|
||||
value: 'simple',
|
||||
},
|
||||
{
|
||||
name: 'Random',
|
||||
value: 'random',
|
||||
},
|
||||
{
|
||||
name: 'Code',
|
||||
value: 'code',
|
||||
},
|
||||
],
|
||||
default: 'simple',
|
||||
description: 'The type of sorting to perform',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Sort By',
|
||||
name: 'sortFieldsUi',
|
||||
type: 'fixedCollection',
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
placeholder: 'Add Field To Sort By',
|
||||
options: [
|
||||
{
|
||||
displayName: '',
|
||||
name: 'sortField',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Field Name',
|
||||
name: 'fieldName',
|
||||
type: 'string',
|
||||
required: true,
|
||||
default: '',
|
||||
description: 'The field to sort by',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
|
||||
placeholder: 'e.g. id',
|
||||
hint: ' Enter the field name as text',
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
{
|
||||
displayName: 'Order',
|
||||
name: 'order',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Ascending',
|
||||
value: 'ascending',
|
||||
},
|
||||
{
|
||||
name: 'Descending',
|
||||
value: 'descending',
|
||||
},
|
||||
],
|
||||
default: 'ascending',
|
||||
description: 'The order to sort by',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
default: {},
|
||||
description: 'The fields of the input items to sort by',
|
||||
displayOptions: {
|
||||
show: {
|
||||
type: ['simple'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Code',
|
||||
name: 'code',
|
||||
type: 'string',
|
||||
typeOptions: {
|
||||
alwaysOpenEditWindow: true,
|
||||
editor: 'jsEditor',
|
||||
rows: 10,
|
||||
},
|
||||
default: `// The two items to compare are in the variables a and b
|
||||
// Access the fields in a.json and b.json
|
||||
// Return -1 if a should go before b
|
||||
// Return 1 if b should go before a
|
||||
// Return 0 if there's no difference
|
||||
|
||||
fieldName = 'myField';
|
||||
|
||||
if (a.json[fieldName] < b.json[fieldName]) {
|
||||
return -1;
|
||||
}
|
||||
if (a.json[fieldName] > b.json[fieldName]) {
|
||||
return 1;
|
||||
}
|
||||
return 0;`,
|
||||
description: 'Javascript code to determine the order of any two items',
|
||||
displayOptions: {
|
||||
show: {
|
||||
type: ['code'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
displayOptions: {
|
||||
show: {
|
||||
type: ['simple'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
let returnData = [...items];
|
||||
const type = this.getNodeParameter('type', 0) as string;
|
||||
const disableDotNotation = this.getNodeParameter(
|
||||
'options.disableDotNotation',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
|
||||
if (type === 'random') {
|
||||
shuffleArray(returnData);
|
||||
return [returnData];
|
||||
}
|
||||
|
||||
if (type === 'simple') {
|
||||
const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject;
|
||||
const sortFields = sortFieldsUi.sortField as Array<{
|
||||
fieldName: string;
|
||||
order: 'ascending' | 'descending';
|
||||
}>;
|
||||
|
||||
if (!sortFields?.length) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'No sorting specified. Please add a field to sort by',
|
||||
);
|
||||
}
|
||||
|
||||
for (const { fieldName } of sortFields) {
|
||||
let found = false;
|
||||
for (const item of items) {
|
||||
if (!disableDotNotation) {
|
||||
if (get(item.json, fieldName) !== undefined) {
|
||||
found = true;
|
||||
}
|
||||
} else if (item.json.hasOwnProperty(fieldName)) {
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
if (!found && disableDotNotation && fieldName.includes('.')) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`Couldn't find the field '${fieldName}' in the input data`,
|
||||
{
|
||||
description:
|
||||
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
|
||||
},
|
||||
);
|
||||
} else if (!found) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`Couldn't find the field '${fieldName}' in the input data`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const sortFieldsWithDirection = sortFields.map((field) => ({
|
||||
name: field.fieldName,
|
||||
dir: field.order === 'ascending' ? 1 : -1,
|
||||
}));
|
||||
|
||||
returnData.sort((a, b) => {
|
||||
let result = 0;
|
||||
for (const field of sortFieldsWithDirection) {
|
||||
let equal;
|
||||
if (!disableDotNotation) {
|
||||
const _a =
|
||||
typeof get(a.json, field.name) === 'string'
|
||||
? (get(a.json, field.name) as string).toLowerCase()
|
||||
: get(a.json, field.name);
|
||||
const _b =
|
||||
typeof get(b.json, field.name) === 'string'
|
||||
? (get(b.json, field.name) as string).toLowerCase()
|
||||
: get(b.json, field.name);
|
||||
equal = isEqual(_a, _b);
|
||||
} else {
|
||||
const _a =
|
||||
typeof a.json[field.name] === 'string'
|
||||
? (a.json[field.name] as string).toLowerCase()
|
||||
: a.json[field.name];
|
||||
const _b =
|
||||
typeof b.json[field.name] === 'string'
|
||||
? (b.json[field.name] as string).toLowerCase()
|
||||
: b.json[field.name];
|
||||
equal = isEqual(_a, _b);
|
||||
}
|
||||
|
||||
if (!equal) {
|
||||
let lessThan;
|
||||
if (!disableDotNotation) {
|
||||
const _a =
|
||||
typeof get(a.json, field.name) === 'string'
|
||||
? (get(a.json, field.name) as string).toLowerCase()
|
||||
: get(a.json, field.name);
|
||||
const _b =
|
||||
typeof get(b.json, field.name) === 'string'
|
||||
? (get(b.json, field.name) as string).toLowerCase()
|
||||
: get(b.json, field.name);
|
||||
lessThan = lt(_a, _b);
|
||||
} else {
|
||||
const _a =
|
||||
typeof a.json[field.name] === 'string'
|
||||
? (a.json[field.name] as string).toLowerCase()
|
||||
: a.json[field.name];
|
||||
const _b =
|
||||
typeof b.json[field.name] === 'string'
|
||||
? (b.json[field.name] as string).toLowerCase()
|
||||
: b.json[field.name];
|
||||
lessThan = lt(_a, _b);
|
||||
}
|
||||
if (lessThan) {
|
||||
result = -1 * field.dir;
|
||||
} else {
|
||||
result = 1 * field.dir;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
} else {
|
||||
returnData = await sortByCode.call(this);
|
||||
}
|
||||
return [returnData];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><path fill="#8287EB" fill-rule="evenodd" d="M130.5 422.5c-6.627 0-12-5.373-12-12v-351c0-6.627 5.373-12 12-12h24c6.627 0 12 5.373 12 12v351c0 6.627-5.373 12-12 12z" clip-rule="evenodd"/><path fill="#8287EB" fill-rule="evenodd" d="M36.077 333.482c9.398-9.346 24.594-9.304 33.94.095l72.483 72.887 72.482-72.887c9.347-9.399 24.543-9.441 33.941-.095s9.441 24.543.095 33.941l-89.5 90a24 24 0 0 1-34.036 0l-89.5-90c-9.346-9.398-9.304-24.594.095-33.941M381.5 89.5c6.627 0 12 5.373 12 12v351c0 6.627-5.373 12-12 12h-24c-6.627 0-12-5.373-12-12v-351c0-6.627 5.373-12 12-12z" clip-rule="evenodd"/><path fill="#8287EB" fill-rule="evenodd" d="M475.923 178.518c-9.398 9.346-24.594 9.304-33.941-.095L369.5 105.536l-72.482 72.887c-9.347 9.399-24.543 9.441-33.941.095s-9.441-24.543-.095-33.941l89.5-90a24 24 0 0 1 34.036 0l89.5 90c9.346 9.398 9.304 24.594-.095 33.941" clip-rule="evenodd"/></svg>
|
||||
|
After Width: | Height: | Size: 955 B |
@@ -0,0 +1,25 @@
|
||||
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
|
||||
import { JsTaskRunnerSandbox } from '../../Code/JsTaskRunnerSandbox';
|
||||
|
||||
const returnRegExp = /\breturn\b/;
|
||||
|
||||
export async function sortByCode(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
||||
const userCode = this.getNodeParameter('code', 0) as string;
|
||||
if (!returnRegExp.test(userCode)) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"Sort code doesn't return. Please add a 'return' statement to your code",
|
||||
);
|
||||
}
|
||||
|
||||
const mode = this.getMode();
|
||||
const items = this.getInputData();
|
||||
const code = `return items.sort((a, b) => { ${userCode} })`;
|
||||
const chunkSize = undefined;
|
||||
const sandbox = new JsTaskRunnerSandbox(mode, this, chunkSize, { items });
|
||||
const executionResult = await sandbox.runCode<INodeExecutionData[]>(code);
|
||||
|
||||
return executionResult;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.splitOut",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.splitout/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": ["Split", "Nested", "Transform", "Array", "List", "Item"],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
import get from 'lodash/get';
|
||||
import unset from 'lodash/unset';
|
||||
import { NodeOperationError, deepCopy, NodeConnectionTypes } from 'n8n-workflow';
|
||||
import type {
|
||||
IBinaryData,
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeType,
|
||||
INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { prepareFieldsArray } from '../utils/utils';
|
||||
import { FieldsTracker } from './utils';
|
||||
|
||||
export class SplitOut implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Split Out',
|
||||
name: 'splitOut',
|
||||
icon: 'file:splitOut.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: 1,
|
||||
description: 'Turn a list inside item(s) into separate items',
|
||||
defaults: {
|
||||
name: 'Split Out',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
builderHint: {
|
||||
relatedNodes: [
|
||||
{
|
||||
nodeType: 'n8n-nodes-base.aggregate',
|
||||
relationHint: 'Reverse operation - combine items back',
|
||||
},
|
||||
],
|
||||
},
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Fields To Split Out',
|
||||
name: 'fieldToSplitOut',
|
||||
type: 'string',
|
||||
default: '',
|
||||
required: true,
|
||||
placeholder: 'Drag fields from the left or type their names',
|
||||
description:
|
||||
'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.',
|
||||
requiresDataPath: 'multiple',
|
||||
hint: 'Use $binary to split out the input item by binary data',
|
||||
},
|
||||
{
|
||||
displayName: 'Include',
|
||||
name: 'include',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'No Other Fields',
|
||||
value: 'noOtherFields',
|
||||
},
|
||||
{
|
||||
name: 'All Other Fields',
|
||||
value: 'allOtherFields',
|
||||
},
|
||||
{
|
||||
name: 'Selected Other Fields',
|
||||
value: 'selectedOtherFields',
|
||||
},
|
||||
],
|
||||
default: 'noOtherFields',
|
||||
description: 'Whether to copy any other fields into the new items',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Include',
|
||||
name: 'fieldsToInclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input items to aggregate together',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
include: ['selectedOtherFields'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Destination Field Name',
|
||||
name: 'destinationFieldName',
|
||||
type: 'string',
|
||||
requiresDataPath: 'multiple',
|
||||
default: '',
|
||||
description: 'The field in the output under which to put the split field contents',
|
||||
},
|
||||
{
|
||||
displayName: 'Include Binary',
|
||||
name: 'includeBinary',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to include the binary data in the new items',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
const items = this.getInputData();
|
||||
const fieldsTracker = new FieldsTracker();
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string)
|
||||
.split(',')
|
||||
.map((field) => field.trim().replace(/^\$json\./, ''));
|
||||
|
||||
const options = this.getNodeParameter('options', i, {});
|
||||
|
||||
const disableDotNotation = options.disableDotNotation as boolean;
|
||||
|
||||
const destinationFields = ((options.destinationFieldName as string) || '')
|
||||
.split(',')
|
||||
.filter((field) => field.trim() !== '')
|
||||
.map((field) => field.trim());
|
||||
|
||||
if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'If multiple fields to split out are given, the same number of destination fields must be given',
|
||||
);
|
||||
}
|
||||
|
||||
const include = this.getNodeParameter('include', i) as
|
||||
| 'selectedOtherFields'
|
||||
| 'allOtherFields'
|
||||
| 'noOtherFields';
|
||||
|
||||
const multiSplit = fieldsToSplitOut.length > 1;
|
||||
|
||||
const item = { ...items[i].json };
|
||||
const splited: INodeExecutionData[] = [];
|
||||
for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) {
|
||||
const destinationFieldName = destinationFields[entryIndex] || '';
|
||||
|
||||
let entityToSplit: IDataObject[] = [];
|
||||
|
||||
if (fieldToSplitOut === '$binary') {
|
||||
entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({
|
||||
[key]: value,
|
||||
}));
|
||||
} else {
|
||||
if (!disableDotNotation) {
|
||||
entityToSplit = get(item, fieldToSplitOut) as IDataObject[];
|
||||
} else {
|
||||
entityToSplit = item[fieldToSplitOut] as IDataObject[];
|
||||
}
|
||||
|
||||
fieldsTracker.add(fieldToSplitOut);
|
||||
|
||||
const entryExists = entityToSplit !== undefined;
|
||||
|
||||
if (!entryExists) {
|
||||
entityToSplit = [];
|
||||
}
|
||||
|
||||
fieldsTracker.update(fieldToSplitOut, entryExists);
|
||||
|
||||
if (typeof entityToSplit !== 'object' || entityToSplit === null) {
|
||||
entityToSplit = [entityToSplit] as unknown as IDataObject[];
|
||||
}
|
||||
|
||||
if (!Array.isArray(entityToSplit)) {
|
||||
entityToSplit = Object.values(entityToSplit);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [elementIndex, element] of entityToSplit.entries()) {
|
||||
if (splited[elementIndex] === undefined) {
|
||||
splited[elementIndex] = { json: {}, pairedItem: { item: i } };
|
||||
}
|
||||
|
||||
const fieldName = destinationFieldName || fieldToSplitOut;
|
||||
|
||||
if (fieldToSplitOut === '$binary') {
|
||||
if (splited[elementIndex].binary === undefined) {
|
||||
splited[elementIndex].binary = {};
|
||||
}
|
||||
splited[elementIndex].binary[Object.keys(element)[0]] = Object.values(
|
||||
element,
|
||||
)[0] as IBinaryData;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeof element === 'object' && element !== null && include === 'noOtherFields') {
|
||||
if (destinationFieldName === '' && !multiSplit) {
|
||||
splited[elementIndex] = {
|
||||
json: { ...splited[elementIndex].json, ...element },
|
||||
pairedItem: { item: i },
|
||||
};
|
||||
} else {
|
||||
splited[elementIndex].json[fieldName] = element;
|
||||
}
|
||||
} else {
|
||||
splited[elementIndex].json[fieldName] = element;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const splitEntry of splited) {
|
||||
let newItem: INodeExecutionData = splitEntry;
|
||||
|
||||
if (include === 'allOtherFields') {
|
||||
const itemCopy = deepCopy(item);
|
||||
for (const fieldToSplitOut of fieldsToSplitOut) {
|
||||
if (!disableDotNotation) {
|
||||
unset(itemCopy, fieldToSplitOut);
|
||||
} else {
|
||||
delete itemCopy[fieldToSplitOut];
|
||||
}
|
||||
}
|
||||
newItem.json = { ...itemCopy, ...splitEntry.json };
|
||||
}
|
||||
|
||||
if (include === 'selectedOtherFields') {
|
||||
const fieldsToInclude = prepareFieldsArray(
|
||||
this.getNodeParameter('fieldsToInclude', i, '') as string,
|
||||
'Fields To Include',
|
||||
);
|
||||
|
||||
if (!fieldsToInclude.length) {
|
||||
throw new NodeOperationError(this.getNode(), 'No fields specified', {
|
||||
description: 'Please add a field to include',
|
||||
});
|
||||
}
|
||||
|
||||
for (const field of fieldsToInclude) {
|
||||
if (!disableDotNotation) {
|
||||
splitEntry.json[field] = get(item, field);
|
||||
} else {
|
||||
splitEntry.json[field] = item[field];
|
||||
}
|
||||
}
|
||||
|
||||
newItem = splitEntry;
|
||||
}
|
||||
|
||||
const includeBinary = options.includeBinary as boolean;
|
||||
|
||||
if (includeBinary) {
|
||||
if (items[i].binary && !newItem.binary) {
|
||||
newItem.binary = items[i].binary;
|
||||
}
|
||||
}
|
||||
|
||||
returnData.push(newItem);
|
||||
}
|
||||
}
|
||||
|
||||
const hints = fieldsTracker.getHints();
|
||||
|
||||
if (hints.length) {
|
||||
this.addExecutionHints(...hints);
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><g fill="#9B6DD5" clip-path="url(#a)"><path fill-rule="evenodd" d="M480 148c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12zm0 96c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12zm0 96c0-6.627-5.373-12-12-12H322c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h146c6.627 0 12-5.373 12-12z" clip-rule="evenodd"/><path d="M438 76c0 6.627-5.373 12-12 12H309.783c-17.673 0-32 14.327-32 32v56c0 26.978-10.272 51.557-27.119 70.039-5.055 5.545-5.055 14.377 0 19.922 16.847 18.482 27.119 43.061 27.119 70.039v56c0 17.673 14.327 32 32 32H426c6.627 0 12 5.373 12 12v24c0 6.627-5.373 12-12 12H309.783c-44.183 0-80-35.817-80-80v-56c0-30.928-25.072-56-56-56a5.783 5.783 0 0 1-5.783-5.783v-36.434a5.783 5.783 0 0 1 5.783-5.783c30.928 0 56-25.072 56-56v-56c0-44.183 35.817-80 80-80H426c6.627 0 12 5.373 12 12z"/><path fill-rule="evenodd" d="M136 244c0-6.627-5.373-12-12-12H12c-6.627 0-12 5.373-12 12v24c0 6.627 5.373 12 12 12h112c6.627 0 12-5.373 12-12z" clip-rule="evenodd"/></g><defs><clipPath id="a"><path fill="#fff" d="M512 0H0v512h512z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
@@ -0,0 +1,5 @@
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
|
||||
describe('Test Split Out Node', () => {
|
||||
new NodeTestHarness().setupTests();
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { FieldsTracker } from '../utils';
|
||||
|
||||
describe('FieldsTracker', () => {
|
||||
let fieldsTracker: FieldsTracker;
|
||||
|
||||
beforeEach(() => {
|
||||
fieldsTracker = new FieldsTracker();
|
||||
});
|
||||
|
||||
describe('add', () => {
|
||||
it('should add field with false value', () => {
|
||||
fieldsTracker.add('testField');
|
||||
|
||||
expect(fieldsTracker.fields.testField).toBe(false);
|
||||
});
|
||||
|
||||
it('should not overwrite existing field', () => {
|
||||
fieldsTracker.add('testField');
|
||||
fieldsTracker.fields.testField = true;
|
||||
fieldsTracker.add('testField');
|
||||
|
||||
expect(fieldsTracker.fields.testField).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('update', () => {
|
||||
it('should update field from false to true', () => {
|
||||
fieldsTracker.add('testField');
|
||||
fieldsTracker.update('testField', true);
|
||||
|
||||
expect(fieldsTracker.fields.testField).toBe(true);
|
||||
});
|
||||
|
||||
it('should not update field from true to false', () => {
|
||||
fieldsTracker.add('testField');
|
||||
fieldsTracker.fields.testField = true;
|
||||
fieldsTracker.update('testField', false);
|
||||
|
||||
expect(fieldsTracker.fields.testField).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHints', () => {
|
||||
it('should return empty array when no fields tracked', () => {
|
||||
expect(fieldsTracker.getHints()).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return hint for missing field', () => {
|
||||
fieldsTracker.add('missingField');
|
||||
|
||||
const hints = fieldsTracker.getHints();
|
||||
|
||||
expect(hints).toEqual([
|
||||
{
|
||||
message: "The field 'missingField' wasn't found in any input item",
|
||||
location: 'outputPane',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should not return hint for found field', () => {
|
||||
fieldsTracker.add('foundField');
|
||||
fieldsTracker.update('foundField', true);
|
||||
|
||||
expect(fieldsTracker.getHints()).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,388 @@
|
||||
{
|
||||
"name": "splitOut test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [-400, 400]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": {
|
||||
"data": [
|
||||
{
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
}
|
||||
],
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [-180, 400]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data",
|
||||
"options": {}
|
||||
},
|
||||
"id": "e7eac465-8fe6-498c-9942-ebd47df537c1",
|
||||
"name": "Item Lists",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 160]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data",
|
||||
"include": "allOtherFields",
|
||||
"options": {}
|
||||
},
|
||||
"id": "09b7fe15-dbad-4ca6-bf1e-3093139d14e5",
|
||||
"name": "Item Lists1",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 320]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data",
|
||||
"include": "selectedOtherFields",
|
||||
"fieldsToInclude": ["data3"],
|
||||
|
||||
"options": {}
|
||||
},
|
||||
"id": "7ea63dc7-8141-4233-af47-9894919c7fe4",
|
||||
"name": "Item Lists2",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 480]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data",
|
||||
"options": {
|
||||
"destinationFieldName": "output"
|
||||
}
|
||||
},
|
||||
"id": "89c3c1b4-9577-480a-931f-3b34450b23cb",
|
||||
"name": "Item Lists3",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [80, 660]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Item Lists": [
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
}
|
||||
}
|
||||
],
|
||||
"Item Lists1": [
|
||||
{
|
||||
"json": {
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": [
|
||||
{
|
||||
"text": "foo"
|
||||
}
|
||||
],
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Item Lists2": [
|
||||
{
|
||||
"json": {
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": [
|
||||
{
|
||||
"text": "bar"
|
||||
}
|
||||
],
|
||||
"data": {
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Item Lists3": [
|
||||
{
|
||||
"json": {
|
||||
"output": {
|
||||
"id": 3,
|
||||
"char": "c"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"output": {
|
||||
"id": 4,
|
||||
"char": "d"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"output": {
|
||||
"id": 5,
|
||||
"char": "e"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"output": {
|
||||
"id": 1,
|
||||
"char": "a"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"output": {
|
||||
"id": 2,
|
||||
"char": "b"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Item Lists",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists3",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "9230f580-6f41-47c9-9949-bf258fc3fa47",
|
||||
"id": "105",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
{
|
||||
"name": "itemList split Object",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "ade46a75-ab57-48c6-886b-0c118f5ef1c6",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [520, 800]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data",
|
||||
"include": "selectedOtherFields",
|
||||
"fieldsToInclude": ["tag"],
|
||||
"options": {}
|
||||
},
|
||||
"id": "45e1d7a3-d6e8-4b69-a68a-1038db13be4c",
|
||||
"name": "Item Lists1",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [1120, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": {
|
||||
"data": {
|
||||
"entry1": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
},
|
||||
"entry2": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
},
|
||||
"entry3": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
},
|
||||
"data2": ["a", "b", "c"],
|
||||
"data3": {
|
||||
"a": 1,
|
||||
"b": 2,
|
||||
"c": 3
|
||||
},
|
||||
"data4": null,
|
||||
"tag": "bar"
|
||||
}
|
||||
},
|
||||
"id": "faa78fac-468d-42b8-96e9-0fb62c312da3",
|
||||
"name": "Code1",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [760, 800]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "5baaf321-7e89-473d-a313-7cb90b3f13b3",
|
||||
"name": "No Operation, do nothing",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data3",
|
||||
"include": "allOtherFields",
|
||||
"options": {
|
||||
"destinationFieldName": "extracted"
|
||||
}
|
||||
},
|
||||
"id": "a786bea9-eb29-4c6d-aea6-a22aee622bc6",
|
||||
"name": "Item Lists",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [1120, 720]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "0521a24b-c74a-48fa-ae50-48a242b97806",
|
||||
"name": "No Operation, do nothing1",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 720]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data3",
|
||||
"options": {}
|
||||
},
|
||||
"id": "0c1c8827-72ab-4738-918c-d529e66505c6",
|
||||
"name": "Item Lists2",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [1120, 540]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "4c0dca36-c2ae-4d40-8952-0e728ac93fa3",
|
||||
"name": "No Operation, do nothing2",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 540]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data2",
|
||||
"options": {}
|
||||
},
|
||||
"id": "b2031380-b2a8-426d-8f7a-ab072d23b979",
|
||||
"name": "Item Lists3",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [1120, 920]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "617f7259-beee-42f1-bba2-4e75a83fe369",
|
||||
"name": "No Operation, do nothing3",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 920]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldToSplitOut": "data4",
|
||||
"include": "allOtherFields",
|
||||
"options": {}
|
||||
},
|
||||
"id": "8909b8eb-e5a9-4436-8e62-09d8c9670ac1",
|
||||
"name": "Item Lists4",
|
||||
"type": "n8n-nodes-base.splitOut",
|
||||
"typeVersion": 1,
|
||||
"position": [1120, 1140],
|
||||
"continueOnFail": true
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "a9278f90-8ad9-42dc-85b6-28bf1b6764b7",
|
||||
"name": "No Operation, do nothing4",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [1380, 1140]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"No Operation, do nothing": [
|
||||
{
|
||||
"json": {
|
||||
"tag": "bar",
|
||||
"data": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"tag": "bar",
|
||||
"data": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"tag": "bar",
|
||||
"data": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"No Operation, do nothing2": [
|
||||
{
|
||||
"json": {
|
||||
"data3": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data3": 3
|
||||
}
|
||||
}
|
||||
],
|
||||
"No Operation, do nothing1": [
|
||||
{
|
||||
"json": {
|
||||
"data": {
|
||||
"entry1": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
},
|
||||
"entry2": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
},
|
||||
"entry3": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
},
|
||||
"data2": ["a", "b", "c"],
|
||||
"data4": null,
|
||||
"tag": "bar",
|
||||
"extracted": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data": {
|
||||
"entry1": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
},
|
||||
"entry2": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
},
|
||||
"entry3": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
},
|
||||
"data2": ["a", "b", "c"],
|
||||
"data4": null,
|
||||
"tag": "bar",
|
||||
"extracted": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data": {
|
||||
"entry1": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
},
|
||||
"entry2": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
},
|
||||
"entry3": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
},
|
||||
"data2": ["a", "b", "c"],
|
||||
"data4": null,
|
||||
"tag": "bar",
|
||||
"extracted": 3
|
||||
}
|
||||
}
|
||||
],
|
||||
"No Operation, do nothing3": [
|
||||
{
|
||||
"json": {
|
||||
"data2": "a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": "b"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"data2": "c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"No Operation, do nothing4": [
|
||||
{
|
||||
"json": {
|
||||
"data": {
|
||||
"entry1": {
|
||||
"id": 1,
|
||||
"info": "some info 1"
|
||||
},
|
||||
"entry2": {
|
||||
"id": 2,
|
||||
"info": "some info 2"
|
||||
},
|
||||
"entry3": {
|
||||
"id": 3,
|
||||
"info": "some info 3"
|
||||
}
|
||||
},
|
||||
"data2": ["a", "b", "c"],
|
||||
"data3": {
|
||||
"a": 1,
|
||||
"b": 2,
|
||||
"c": 3
|
||||
},
|
||||
"data4": null,
|
||||
"tag": "bar"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code1": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Item Lists1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists3",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Item Lists4",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Item Lists1": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "No Operation, do nothing",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Item Lists": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "No Operation, do nothing1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Item Lists2": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "No Operation, do nothing2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Item Lists3": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "No Operation, do nothing3",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Item Lists4": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "No Operation, do nothing4",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { NodeExecutionHint } from 'n8n-workflow';
|
||||
|
||||
export class FieldsTracker {
|
||||
fields: { [key: string]: boolean } = {};
|
||||
|
||||
add(key: string) {
|
||||
if (this.fields[key] === undefined) {
|
||||
this.fields[key] = false;
|
||||
}
|
||||
}
|
||||
|
||||
update(key: string, value: boolean) {
|
||||
if (!this.fields[key] && value) {
|
||||
this.fields[key] = true;
|
||||
}
|
||||
}
|
||||
|
||||
getHints() {
|
||||
const hints: NodeExecutionHint[] = [];
|
||||
|
||||
for (const [field, value] of Object.entries(this.fields)) {
|
||||
if (!value) {
|
||||
hints.push({
|
||||
message: `The field '${field}' wasn't found in any input item`,
|
||||
location: 'outputPane',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return hints;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.summarize",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.summarize/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": [
|
||||
"Append",
|
||||
"Array",
|
||||
"Average",
|
||||
"Concatenate",
|
||||
"Count",
|
||||
"Group",
|
||||
"Item",
|
||||
"List",
|
||||
"Max",
|
||||
"Min",
|
||||
"Pivot",
|
||||
"Sum",
|
||||
"Summarise",
|
||||
"Summarize",
|
||||
"Transform",
|
||||
"Unique"
|
||||
],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
import {
|
||||
type IExecuteFunctions,
|
||||
type INodeExecutionData,
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
NodeConnectionTypes,
|
||||
type NodeExecutionHint,
|
||||
NodeOperationError,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import {
|
||||
type Aggregations,
|
||||
NUMERICAL_AGGREGATIONS,
|
||||
type SummarizeOptions,
|
||||
aggregateAndSplitData,
|
||||
checkIfFieldExists,
|
||||
fieldValueGetter,
|
||||
flattenAggregationResultToArray,
|
||||
flattenAggregationResultToObject,
|
||||
} from './utils';
|
||||
|
||||
export class Summarize implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Summarize',
|
||||
name: 'summarize',
|
||||
icon: 'file:summarize.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: [1, 1.1],
|
||||
description: 'Sum, count, max, etc. across items',
|
||||
defaults: {
|
||||
name: 'Summarize',
|
||||
},
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Fields to Summarize',
|
||||
name: 'fieldsToSummarize',
|
||||
type: 'fixedCollection',
|
||||
placeholder: 'Add Field',
|
||||
default: { values: [{ aggregation: 'count', field: '' }] },
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: '',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Aggregation',
|
||||
name: 'aggregation',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Append',
|
||||
value: 'append',
|
||||
},
|
||||
{
|
||||
name: 'Average',
|
||||
value: 'average',
|
||||
},
|
||||
{
|
||||
name: 'Concatenate',
|
||||
value: 'concatenate',
|
||||
},
|
||||
{
|
||||
name: 'Count',
|
||||
value: 'count',
|
||||
},
|
||||
{
|
||||
name: 'Count Unique',
|
||||
value: 'countUnique',
|
||||
},
|
||||
{
|
||||
name: 'Max',
|
||||
value: 'max',
|
||||
},
|
||||
{
|
||||
name: 'Min',
|
||||
value: 'min',
|
||||
},
|
||||
{
|
||||
name: 'Sum',
|
||||
value: 'sum',
|
||||
},
|
||||
],
|
||||
default: 'count',
|
||||
description: 'How to combine the values of the field you want to summarize',
|
||||
},
|
||||
//field repeated to have different descriptions for different aggregations --------------------------------
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'The name of an input field that you want to summarize',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'],
|
||||
},
|
||||
},
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: NUMERICAL_AGGREGATIONS,
|
||||
},
|
||||
},
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['countUnique', 'count', 'max', 'min'],
|
||||
},
|
||||
},
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
// ----------------------------------------------------------------------------------------------------------
|
||||
{
|
||||
displayName: 'Include Empty Values',
|
||||
name: 'includeEmpty',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['append', 'concatenate', 'count', 'countUnique'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Separator',
|
||||
name: 'separateBy',
|
||||
type: 'options',
|
||||
default: ',',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
|
||||
options: [
|
||||
{
|
||||
name: 'Comma',
|
||||
value: ',',
|
||||
},
|
||||
{
|
||||
name: 'Comma and Space',
|
||||
value: ', ',
|
||||
},
|
||||
{
|
||||
name: 'New Line',
|
||||
value: '\n',
|
||||
},
|
||||
{
|
||||
name: 'None',
|
||||
value: '',
|
||||
},
|
||||
{
|
||||
name: 'Space',
|
||||
value: ' ',
|
||||
},
|
||||
{
|
||||
name: 'Other',
|
||||
value: 'other',
|
||||
},
|
||||
],
|
||||
hint: 'What to insert between values',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['concatenate'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Custom Separator',
|
||||
name: 'customSeparator',
|
||||
type: 'string',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['concatenate'],
|
||||
separateBy: ['other'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
// fieldsToSplitBy repeated to have different displayName for singleItem and separateItems -----------------------------
|
||||
{
|
||||
displayName: 'Fields to Split By',
|
||||
name: 'fieldsToSplitBy',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. country, city',
|
||||
default: '',
|
||||
description: 'The name of the input fields that you want to split the summary by',
|
||||
hint: 'Enter the name of the fields as text (separated by commas)',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/options.outputFormat': ['singleItem'],
|
||||
},
|
||||
},
|
||||
requiresDataPath: 'multiple',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Group By',
|
||||
name: 'fieldsToSplitBy',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. country, city',
|
||||
default: '',
|
||||
description: 'The name of the input fields that you want to split the summary by',
|
||||
hint: 'Enter the name of the fields as text (separated by commas)',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/options.outputFormat': ['singleItem'],
|
||||
},
|
||||
},
|
||||
requiresDataPath: 'multiple',
|
||||
},
|
||||
// ----------------------------------------------------------------------------------------------------------
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Continue if Field Not Found',
|
||||
name: 'continueIfFieldNotFound',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether to continue if field to summarize can't be found in any items and return single empty item, otherwise an error would be thrown",
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'@version': [{ _cnd: { gte: 1.1 } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Output Format',
|
||||
name: 'outputFormat',
|
||||
type: 'options',
|
||||
default: 'separateItems',
|
||||
options: [
|
||||
{
|
||||
name: 'Each Split in a Separate Item',
|
||||
value: 'separateItems',
|
||||
},
|
||||
{
|
||||
name: 'All Splits in a Single Item',
|
||||
value: 'singleItem',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
|
||||
displayName: 'Ignore items without valid fields to group by',
|
||||
name: 'skipEmptySplitFields',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i }));
|
||||
|
||||
const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions;
|
||||
|
||||
const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string)
|
||||
.split(',')
|
||||
.map((field) => field.trim())
|
||||
.filter((field) => field);
|
||||
|
||||
const fieldsToSummarize = this.getNodeParameter(
|
||||
'fieldsToSummarize.values',
|
||||
0,
|
||||
[],
|
||||
) as Aggregations;
|
||||
|
||||
if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'",
|
||||
);
|
||||
}
|
||||
|
||||
const getValue = fieldValueGetter(options.disableDotNotation);
|
||||
|
||||
const nodeVersion = this.getNode().typeVersion;
|
||||
|
||||
const aggregationResult = aggregateAndSplitData({
|
||||
splitKeys: fieldsToSplitBy,
|
||||
inputItems: newItems,
|
||||
fieldsToSummarize,
|
||||
options,
|
||||
getValue,
|
||||
convertKeysToString: nodeVersion === 1,
|
||||
});
|
||||
|
||||
const fieldsNotFound: NodeExecutionHint[] = [];
|
||||
try {
|
||||
checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue);
|
||||
} catch (error) {
|
||||
if (nodeVersion > 1 || options.continueIfFieldNotFound) {
|
||||
const fieldNotFoundHint: NodeExecutionHint = {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
location: 'outputPane',
|
||||
};
|
||||
fieldsNotFound.push(fieldNotFoundHint);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (fieldsNotFound.length) {
|
||||
this.addExecutionHints(...fieldsNotFound);
|
||||
}
|
||||
|
||||
if (options.outputFormat === 'singleItem') {
|
||||
const executionData: INodeExecutionData = {
|
||||
json: flattenAggregationResultToObject(aggregationResult),
|
||||
pairedItem: newItems.map((_v, index) => ({
|
||||
item: index,
|
||||
})),
|
||||
};
|
||||
return [[executionData]];
|
||||
} else {
|
||||
if (!fieldsToSplitBy.length && 'pairedItems' in aggregationResult) {
|
||||
const { pairedItems, returnData } = aggregationResult;
|
||||
const executionData: INodeExecutionData = {
|
||||
json: returnData,
|
||||
pairedItem: (pairedItems ?? []).map((index) => ({ item: index })),
|
||||
};
|
||||
return [[executionData]];
|
||||
}
|
||||
const flatAggregationResults = flattenAggregationResultToArray(aggregationResult);
|
||||
const executionData = flatAggregationResults.map((item) => {
|
||||
const { pairedItems, returnData } = item;
|
||||
return {
|
||||
json: returnData,
|
||||
pairedItem: (pairedItems ?? []).map((index) => ({ item: index })),
|
||||
};
|
||||
});
|
||||
return [executionData];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" fill="none"><path fill="#F92" fill-rule="evenodd" d="M132 91c-15.464 0-28 12.536-28 28v317c0 15.464 12.536 28 28 28h248c15.464 0 28-12.536 28-28V119c0-15.464-12.536-28-28-28h-6a6 6 0 0 1-6-6V49a6 6 0 0 1 6-6h6c41.974 0 76 34.026 76 76v317c0 41.974-34.026 76-76 76H132c-41.974 0-76-34.026-76-76V119c0-41.974 34.026-76 76-76h6a6 6 0 0 1 6 6v36a6 6 0 0 1-6 6z" clip-rule="evenodd"/><path fill="#F92" fill-rule="evenodd" d="M256 0c-27.232 0-50.227 18.142-57.558 43H182a6 6 0 0 0-6 6v70a6 6 0 0 0 6 6h148a6 6 0 0 0 6-6V49a6 6 0 0 0-6-6h-16.442C306.227 18.142 283.232 0 256 0m0 40a19.9 19.9 0 0 0-10.541 3C239.781 46.528 236 52.823 236 60c0 11.046 8.954 20 20 20s20-8.954 20-20c0-7.177-3.781-13.472-9.459-17A19.9 19.9 0 0 0 256 40m101 179c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 207c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/><path fill="#F92" fill-rule="evenodd" d="M357 395c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 383c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/><path fill="#F92" fill-rule="evenodd" d="M357 307c0 6.627-5.373 12-12 12H233c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h112c6.627 0 12 5.373 12 12z" clip-rule="evenodd"/><path fill="#F92" d="M197 295c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24"/></svg>
|
||||
|
After Width: | Height: | Size: 1.6 KiB |
@@ -0,0 +1,5 @@
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
|
||||
describe('Test Summarize Node', () => {
|
||||
new NodeTestHarness().setupTests();
|
||||
});
|
||||
+524
@@ -0,0 +1,524 @@
|
||||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is false: split-field-with-spaces-array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"Warehouse": "WH1",
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"Warehouse": null,
|
||||
"appended_Warehouse": [
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": null,
|
||||
"Warehouse": "WH2",
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is false: split-field-with-spaces-result 1`] = `
|
||||
{
|
||||
"fieldName": "Product",
|
||||
"splits": Map {
|
||||
"Widget A" => {
|
||||
"fieldName": "Warehouse",
|
||||
"splits": Map {
|
||||
"WH1" => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
],
|
||||
},
|
||||
},
|
||||
null => {
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null => {
|
||||
"fieldName": "Warehouse",
|
||||
"splits": Map {
|
||||
"WH2" => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is true: split-field-with-spaces-array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"Warehouse": "WH1",
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"Warehouse": "null",
|
||||
"appended_Warehouse": [
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "null",
|
||||
"Warehouse": "WH2",
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle multiple split field values containing null when convertKeysToString is true: split-field-with-spaces-result 1`] = `
|
||||
{
|
||||
"fieldName": "Product",
|
||||
"splits": Map {
|
||||
"Widget A" => {
|
||||
"fieldName": "Warehouse",
|
||||
"splits": Map {
|
||||
"WH1" => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
],
|
||||
},
|
||||
},
|
||||
"null" => {
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"null" => {
|
||||
"fieldName": "Warehouse",
|
||||
"splits": Map {
|
||||
"WH2" => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is not set: split-field-with-spaces-array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
"WH3",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget B",
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is not set: split-field-with-spaces-result 1`] = `
|
||||
{
|
||||
"fieldName": "Product",
|
||||
"splits": Map {
|
||||
"Widget A" => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
"WH3",
|
||||
],
|
||||
},
|
||||
},
|
||||
"Widget B" => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is true: split-field-with-spaces-array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget A",
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
"WH3",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Product": "Widget B",
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should handle split field values containing spaces when convertKeysToString is true: split-field-with-spaces-result 1`] = `
|
||||
{
|
||||
"fieldName": "Product",
|
||||
"splits": Map {
|
||||
"Widget A" => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH1",
|
||||
"WH3",
|
||||
],
|
||||
},
|
||||
},
|
||||
"Widget B" => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"WH2",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should not convert numbers to strings: array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": 1,
|
||||
"Sku": 12345,
|
||||
"appended_Warehouse": [
|
||||
"BER_0G",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": 2,
|
||||
"Sku": 12345,
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": 1,
|
||||
"Sku": 6534563534,
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should not convert numbers to strings: result 1`] = `
|
||||
{
|
||||
"fieldName": "Sku",
|
||||
"splits": Map {
|
||||
12345 => {
|
||||
"fieldName": "Qty",
|
||||
"splits": Map {
|
||||
1 => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0G",
|
||||
],
|
||||
},
|
||||
},
|
||||
2 => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
6534563534 => {
|
||||
"fieldName": "Qty",
|
||||
"splits": Map {
|
||||
1 => {
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should not convert strings to numbers: array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": "1",
|
||||
"Sku": "012345",
|
||||
"appended_Warehouse": [
|
||||
"BER_0G",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": "2",
|
||||
"Sku": "012345",
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Qty": "1",
|
||||
"Sku": "06534563534",
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData should not convert strings to numbers: result 1`] = `
|
||||
{
|
||||
"fieldName": "Sku",
|
||||
"splits": Map {
|
||||
"012345" => {
|
||||
"fieldName": "Qty",
|
||||
"splits": Map {
|
||||
"1" => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0G",
|
||||
],
|
||||
},
|
||||
},
|
||||
"2" => {
|
||||
"pairedItems": [
|
||||
1,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"06534563534" => {
|
||||
"fieldName": "Qty",
|
||||
"splits": Map {
|
||||
"1" => {
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"appended_Warehouse": [
|
||||
"BER_0L",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData with skipEmptySplitFields=true should skip empty split fields: array 1`] = `
|
||||
[
|
||||
{
|
||||
"pairedItems": [
|
||||
0,
|
||||
3,
|
||||
],
|
||||
"returnData": {
|
||||
"Sku": 12345,
|
||||
"concatenated_Warehouse": "BER_0G//{"name":"BER_0G3"}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"Sku": "{}",
|
||||
"concatenated_Warehouse": "BER_0L",
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`Test Summarize Node, aggregateAndSplitData with skipEmptySplitFields=true should skip empty split fields: result 1`] = `
|
||||
{
|
||||
"fieldName": "Sku",
|
||||
"splits": Map {
|
||||
12345 => {
|
||||
"pairedItems": [
|
||||
0,
|
||||
3,
|
||||
],
|
||||
"returnData": {
|
||||
"concatenated_Warehouse": "BER_0G//{"name":"BER_0G3"}",
|
||||
},
|
||||
},
|
||||
"{}" => {
|
||||
"pairedItems": [
|
||||
2,
|
||||
],
|
||||
"returnData": {
|
||||
"concatenated_Warehouse": "BER_0L",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
import { NodeOperationError, type IExecuteFunctions, type IDataObject } from 'n8n-workflow';
|
||||
|
||||
import { checkIfFieldExists, type Aggregations } from '../../utils';
|
||||
|
||||
describe('Test Summarize Node, checkIfFieldExists', () => {
|
||||
let mockExecuteFunctions: IExecuteFunctions;
|
||||
|
||||
beforeEach(() => {
|
||||
mockExecuteFunctions = {
|
||||
getNode: jest.fn().mockReturnValue({ name: 'test-node' }),
|
||||
} as unknown as IExecuteFunctions;
|
||||
});
|
||||
|
||||
const items = [{ a: 1 }, { b: 2 }, { c: 3 }];
|
||||
|
||||
it('should not throw error if all fields exist', () => {
|
||||
const aggregations: Aggregations = [
|
||||
{ aggregation: 'sum', field: 'a' },
|
||||
{ aggregation: 'count', field: 'c' },
|
||||
];
|
||||
const getValue = (item: IDataObject, field: string) => item[field];
|
||||
expect(() => {
|
||||
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw NodeOperationError if any field does not exist', () => {
|
||||
const aggregations: Aggregations = [
|
||||
{ aggregation: 'sum', field: 'b' },
|
||||
{ aggregation: 'count', field: 'd' },
|
||||
];
|
||||
const getValue = (item: IDataObject, field: string) => item[field];
|
||||
expect(() => {
|
||||
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
|
||||
}).toThrow(NodeOperationError);
|
||||
});
|
||||
|
||||
it("should throw NodeOperationError with error message containing the field name that doesn't exist", () => {
|
||||
const aggregations: Aggregations = [{ aggregation: 'count', field: 'D' }];
|
||||
const getValue = (item: IDataObject, field: string) => item[field];
|
||||
expect(() => {
|
||||
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
|
||||
}).toThrow("The field 'D' does not exist in any items");
|
||||
});
|
||||
|
||||
it('should not throw error if field is empty string', () => {
|
||||
const aggregations: Aggregations = [{ aggregation: 'count', field: '' }];
|
||||
const getValue = (item: IDataObject, field: string) => item[field];
|
||||
expect(() => {
|
||||
checkIfFieldExists.call(mockExecuteFunctions, items, aggregations, getValue);
|
||||
}).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { MockProxy } from 'jest-mock-extended';
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type { IExecuteFunctions } from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
|
||||
import { Summarize } from '../../Summarize.node';
|
||||
import type { Aggregations } from '../../utils';
|
||||
|
||||
let summarizeNode: Summarize;
|
||||
let mockExecuteFunctions: MockProxy<IExecuteFunctions>;
|
||||
|
||||
describe('Test Summarize Node, execute', () => {
|
||||
beforeEach(() => {
|
||||
summarizeNode = new Summarize();
|
||||
mockExecuteFunctions = mock<IExecuteFunctions>({
|
||||
getNode: jest.fn().mockReturnValue({ name: 'test-node' }),
|
||||
getNodeParameter: jest.fn(),
|
||||
getInputData: jest.fn(),
|
||||
helpers: {
|
||||
constructExecutionMetaData: jest.fn().mockReturnValue([]),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should handle field not found with hints if version > 1', async () => {
|
||||
mockExecuteFunctions.getInputData.mockReturnValue([{ json: { someField: 1 } }]);
|
||||
mockExecuteFunctions.getNode.mockReturnValue({
|
||||
id: '1',
|
||||
name: 'test-node',
|
||||
type: 'test-type',
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
typeVersion: 1.1,
|
||||
});
|
||||
mockExecuteFunctions.getNodeParameter
|
||||
.mockReturnValueOnce({}) // options
|
||||
.mockReturnValueOnce('') // fieldsToSplitBy
|
||||
.mockReturnValueOnce([{ field: 'nonexistentField', aggregation: 'sum' }]); // fieldsToSummarize
|
||||
|
||||
const result = await summarizeNode.execute.call(mockExecuteFunctions);
|
||||
|
||||
expect(result).toEqual([[{ json: { sum_nonexistentField: 0 }, pairedItem: [{ item: 0 }] }]]);
|
||||
expect(mockExecuteFunctions.addExecutionHints).toHaveBeenCalledWith({
|
||||
location: 'outputPane',
|
||||
message: "The field 'nonexistentField' does not exist in any items",
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw error if node version is < 1.1 and fields not found', async () => {
|
||||
const items = [{ json: { a: 1, b: 2, c: 3 } }];
|
||||
const aggregations: Aggregations = [
|
||||
{ aggregation: 'sum', field: 'b' },
|
||||
{ aggregation: 'count', field: 'd' },
|
||||
];
|
||||
|
||||
mockExecuteFunctions.getNode.mockReturnValue({
|
||||
id: '1',
|
||||
name: 'test-node',
|
||||
type: 'test-type',
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
typeVersion: 1,
|
||||
});
|
||||
mockExecuteFunctions.getInputData.mockReturnValue(items);
|
||||
mockExecuteFunctions.getNodeParameter
|
||||
.mockReturnValueOnce({}) // options
|
||||
.mockReturnValueOnce('') // fieldsToSplitBy
|
||||
.mockReturnValueOnce(aggregations); // fieldsToSummarize
|
||||
await expect(async () => {
|
||||
await summarizeNode.execute.bind(mockExecuteFunctions)();
|
||||
}).rejects.toThrow(NodeOperationError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,314 @@
|
||||
import {
|
||||
fieldValueGetter,
|
||||
aggregateAndSplitData,
|
||||
flattenAggregationResultToArray,
|
||||
type Aggregations,
|
||||
} from '../../utils';
|
||||
|
||||
describe('Test Summarize Node, aggregateAndSplitData', () => {
|
||||
test('should not convert strings to numbers', () => {
|
||||
const data = [
|
||||
{
|
||||
Sku: '012345',
|
||||
Warehouse: 'BER_0G',
|
||||
Qty: '1',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Sku: '012345',
|
||||
Warehouse: 'BER_0L',
|
||||
Qty: '2',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Sku: '06534563534',
|
||||
Warehouse: 'BER_0L',
|
||||
Qty: '1',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Sku', 'Qty'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
});
|
||||
expect(result).toMatchSnapshot('result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
|
||||
});
|
||||
|
||||
test('should not convert numbers to strings', () => {
|
||||
const data = [
|
||||
{
|
||||
Sku: 12345,
|
||||
Warehouse: 'BER_0G',
|
||||
Qty: 1,
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Sku: 12345,
|
||||
Warehouse: 'BER_0L',
|
||||
Qty: 2,
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Sku: 6534563534,
|
||||
Warehouse: 'BER_0L',
|
||||
Qty: 1,
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Sku', 'Qty'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
});
|
||||
expect(result).toMatchSnapshot('result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
|
||||
});
|
||||
|
||||
test('should handle split field values containing spaces when convertKeysToString is not set', () => {
|
||||
const data = [
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH1',
|
||||
Qty: '5',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Product: 'Widget B',
|
||||
Warehouse: 'WH2',
|
||||
Qty: '3',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH3',
|
||||
Qty: '2',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Product'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
});
|
||||
|
||||
expect(result).toMatchSnapshot('split-field-with-spaces-result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
|
||||
'split-field-with-spaces-array',
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle split field values containing spaces when convertKeysToString is true', () => {
|
||||
const data = [
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH1',
|
||||
Qty: '5',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Product: 'Widget B',
|
||||
Warehouse: 'WH2',
|
||||
Qty: '3',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH3',
|
||||
Qty: '2',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Product'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
convertKeysToString: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchSnapshot('split-field-with-spaces-result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
|
||||
'split-field-with-spaces-array',
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle multiple split field values containing null when convertKeysToString is true', () => {
|
||||
const data = [
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH1',
|
||||
Qty: '5',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Product: null,
|
||||
Warehouse: 'WH2',
|
||||
Qty: '3',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: null,
|
||||
Qty: '2',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Product', 'Warehouse'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
convertKeysToString: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchSnapshot('split-field-with-spaces-result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
|
||||
'split-field-with-spaces-array',
|
||||
);
|
||||
});
|
||||
|
||||
test('should handle multiple split field values containing null when convertKeysToString is false', () => {
|
||||
const data = [
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: 'WH1',
|
||||
Qty: '5',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Product: null,
|
||||
Warehouse: 'WH2',
|
||||
Qty: '3',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Product: 'Widget A',
|
||||
Warehouse: null,
|
||||
Qty: '2',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'append',
|
||||
field: 'Warehouse',
|
||||
includeEmpty: true,
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Product', 'Warehouse'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true },
|
||||
getValue: fieldValueGetter(),
|
||||
convertKeysToString: false,
|
||||
});
|
||||
|
||||
expect(result).toMatchSnapshot('split-field-with-spaces-result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot(
|
||||
'split-field-with-spaces-array',
|
||||
);
|
||||
});
|
||||
|
||||
describe('with skipEmptySplitFields=true', () => {
|
||||
test('should skip empty split fields', () => {
|
||||
const data = [
|
||||
{
|
||||
Sku: 12345,
|
||||
Warehouse: 'BER_0G',
|
||||
_itemIndex: 0,
|
||||
},
|
||||
{
|
||||
Warehouse: 'BER_0L',
|
||||
_itemIndex: 1,
|
||||
},
|
||||
{
|
||||
Sku: {},
|
||||
Warehouse: 'BER_0L',
|
||||
_itemIndex: 2,
|
||||
},
|
||||
{
|
||||
Sku: 12345,
|
||||
Warehouse: { name: 'BER_0G3' },
|
||||
_itemIndex: 3,
|
||||
},
|
||||
];
|
||||
|
||||
const aggregations: Aggregations = [
|
||||
{
|
||||
aggregation: 'concatenate',
|
||||
field: 'Warehouse',
|
||||
separateBy: 'other',
|
||||
customSeparator: '//',
|
||||
},
|
||||
];
|
||||
|
||||
const result = aggregateAndSplitData({
|
||||
splitKeys: ['Sku'],
|
||||
inputItems: data,
|
||||
fieldsToSummarize: aggregations,
|
||||
options: { continueIfFieldNotFound: true, skipEmptySplitFields: true },
|
||||
getValue: fieldValueGetter(),
|
||||
});
|
||||
expect(result).toMatchSnapshot('result');
|
||||
expect(flattenAggregationResultToArray(result)).toMatchSnapshot('array');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"name": "summarize test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [-160, 340],
|
||||
"id": "0cc16e22-72e3-4f2c-a920-b1613fd4bcaf",
|
||||
"name": "When clicking ‘Execute workflow’"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"category": "red",
|
||||
"text": "foo",
|
||||
"char": "a",
|
||||
"value": 1,
|
||||
"params.interval": "1 day",
|
||||
"params": {
|
||||
"interval": "1 day"
|
||||
}
|
||||
},
|
||||
{
|
||||
"category": "blue",
|
||||
"text": "spam",
|
||||
"char": "b",
|
||||
"value": 2,
|
||||
"params": {
|
||||
"interval": "1 hour"
|
||||
}
|
||||
},
|
||||
{
|
||||
"category": "green",
|
||||
"text": "bar",
|
||||
"char": "c",
|
||||
"value": 3
|
||||
},
|
||||
{
|
||||
"category": "red",
|
||||
"text": "foo",
|
||||
"char": "a",
|
||||
"value": 4
|
||||
},
|
||||
{
|
||||
"category": "red",
|
||||
"text": "bar",
|
||||
"char": "a",
|
||||
"value": 5,
|
||||
"params": {
|
||||
"interval": "1 minute"
|
||||
}
|
||||
},
|
||||
{
|
||||
"category": "blue",
|
||||
"text": "foo",
|
||||
"char": "a",
|
||||
"value": 6
|
||||
},
|
||||
{
|
||||
"category": "blue",
|
||||
"text": "foo",
|
||||
"char": "a",
|
||||
"value": 7,
|
||||
"params": {
|
||||
"interval": "1 second"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "5a71d6f2-96fb-4616-94b7-5644c71e3bfb",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [60, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"aggregation": "concatenate",
|
||||
"field": "char",
|
||||
"separateBy": ", "
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldsToSplitBy": "category, text",
|
||||
"options": {}
|
||||
},
|
||||
"id": "281b21e0-5737-4150-bce8-8331a3f83366",
|
||||
"name": "Summarize1",
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1,
|
||||
"position": [280, -60]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "char"
|
||||
},
|
||||
{
|
||||
"aggregation": "concatenate",
|
||||
"field": "char",
|
||||
"separateBy": ", "
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldsToSplitBy": "category, text",
|
||||
"options": {
|
||||
"outputFormat": "singleItem"
|
||||
}
|
||||
},
|
||||
"id": "04bf538c-24d4-46d8-9c43-76ebc4fe7368",
|
||||
"name": "Summarize2",
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1,
|
||||
"position": [280, 140]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "average",
|
||||
"field": "value"
|
||||
},
|
||||
{
|
||||
"aggregation": "max",
|
||||
"field": "value"
|
||||
},
|
||||
{
|
||||
"aggregation": "min",
|
||||
"field": "value"
|
||||
},
|
||||
{
|
||||
"aggregation": "max",
|
||||
"field": "value"
|
||||
},
|
||||
{
|
||||
"aggregation": "sum",
|
||||
"field": "value"
|
||||
},
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "value"
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldsToSplitBy": "category",
|
||||
"options": {}
|
||||
},
|
||||
"id": "e6f750f2-15c1-4de9-80a1-a255649d6e99",
|
||||
"name": "Summarize3",
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1,
|
||||
"position": [280, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "params.interval"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [280, 540],
|
||||
"id": "bfe0521a-c060-454f-99bb-b682aab44f63",
|
||||
"name": "Dot notation"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "params.interval"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"disableDotNotation": true
|
||||
}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [280, 740],
|
||||
"id": "b2c05f23-449f-49aa-80a3-4e1cd9156aa1",
|
||||
"name": "Dot notation (disabled)"
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Summarize1": [
|
||||
{
|
||||
"json": {
|
||||
"category": "red",
|
||||
"text": "foo",
|
||||
"appended_char": ["a", "a"],
|
||||
"count_char": 2,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a, a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "red",
|
||||
"text": "bar",
|
||||
"appended_char": ["a"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "blue",
|
||||
"text": "spam",
|
||||
"appended_char": ["b"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "b"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "blue",
|
||||
"text": "foo",
|
||||
"appended_char": ["a", "a"],
|
||||
"count_char": 2,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a, a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "green",
|
||||
"text": "bar",
|
||||
"appended_char": ["c"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "c"
|
||||
}
|
||||
}
|
||||
],
|
||||
"Summarize2": [
|
||||
{
|
||||
"json": {
|
||||
"red": {
|
||||
"foo": {
|
||||
"appended_char": ["a", "a"],
|
||||
"count_char": 2,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a, a"
|
||||
},
|
||||
"bar": {
|
||||
"appended_char": ["a"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a"
|
||||
}
|
||||
},
|
||||
"blue": {
|
||||
"spam": {
|
||||
"appended_char": ["b"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "b"
|
||||
},
|
||||
"foo": {
|
||||
"appended_char": ["a", "a"],
|
||||
"count_char": 2,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "a, a"
|
||||
}
|
||||
},
|
||||
"green": {
|
||||
"bar": {
|
||||
"appended_char": ["c"],
|
||||
"count_char": 1,
|
||||
"unique_count_char": 1,
|
||||
"concatenated_char": "c"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Summarize3": [
|
||||
{
|
||||
"json": {
|
||||
"category": "red",
|
||||
"average_value": 3.3333333333333335,
|
||||
"max_value": 5,
|
||||
"min_value": 1,
|
||||
"sum_value": 10,
|
||||
"appended_value": [1, 4, 5]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "blue",
|
||||
"average_value": 5,
|
||||
"max_value": 7,
|
||||
"min_value": 2,
|
||||
"sum_value": 15,
|
||||
"appended_value": [2, 6, 7]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"category": "green",
|
||||
"average_value": 3,
|
||||
"max_value": 3,
|
||||
"min_value": 3,
|
||||
"sum_value": 3,
|
||||
"appended_value": [3]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Dot notation": [
|
||||
{
|
||||
"json": {
|
||||
"appended_params_interval": ["1 day", "1 hour", "1 minute", "1 second"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Dot notation (disabled)": [
|
||||
{
|
||||
"json": {
|
||||
"appended_params_interval": ["1 day"]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking ‘Execute workflow’": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Summarize1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Summarize2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Summarize3",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Dot notation",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Dot notation (disabled)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {
|
||||
"executionOrder": "v1"
|
||||
},
|
||||
"versionId": "60e793d6-8c99-4bd2-aeb8-8ff350fc3101",
|
||||
"meta": {
|
||||
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
|
||||
},
|
||||
"id": "0HNQZNduHDmRCBBW",
|
||||
"tags": []
|
||||
}
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
{
|
||||
"name": "Summarize Test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "e46eed6e-4a6b-4e1f-bd7a-96cf579dd2ee",
|
||||
"name": "When clicking \"Execute workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [-900, 260]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{ "review": "Great product!" },
|
||||
{ "review": "" },
|
||||
{ "review": null },
|
||||
{ "review": null },
|
||||
{ "review": "Bad product" },
|
||||
{},
|
||||
{}
|
||||
]
|
||||
},
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [-520, -80],
|
||||
"id": "80f3462d-7ea5-4af1-bd13-f3f46b0ee43f",
|
||||
"name": "sample data1"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"field": "review"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-280, -320],
|
||||
"id": "98ff6686-bf9f-436a-8a64-3b3b1e648184",
|
||||
"name": "count non-empty"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "review"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-280, -160],
|
||||
"id": "aa094d4c-5e00-49a4-9799-e7692b49f204",
|
||||
"name": "countUnique non-empty"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"field": "review",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-280, -40],
|
||||
"id": "c4a439c9-4d25-4ebb-b36c-c72d0d738049",
|
||||
"name": "count with Empty"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "review",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-280, 100],
|
||||
"id": "e8abc054-5285-4dcc-980b-4bba4ef45ae0",
|
||||
"name": "countUnique non-empty1"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"region": "North",
|
||||
"sales": 100
|
||||
},
|
||||
{
|
||||
"region": "North",
|
||||
"sales": 200
|
||||
},
|
||||
{
|
||||
"region": "South",
|
||||
"sales": null
|
||||
},
|
||||
{
|
||||
"region": "",
|
||||
"sales": 300
|
||||
},
|
||||
{
|
||||
"region": "East"
|
||||
},
|
||||
{
|
||||
"region": "West",
|
||||
"sales": 400
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [-520, 500],
|
||||
"id": "0de1d232-09c1-412f-9481-0120db0792bf",
|
||||
"name": "sample data2"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"field": "region"
|
||||
},
|
||||
{
|
||||
"field": "sales"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-260, 300],
|
||||
"id": "b81e9c48-828c-4ac8-b7ca-818022384ca6",
|
||||
"name": "count non-empty1"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "region"
|
||||
},
|
||||
{
|
||||
"field": "sales"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-260, 440],
|
||||
"id": "22bd90da-9b57-4b38-9cc3-356901a6f666",
|
||||
"name": "countUnique non-empty2"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"field": "region",
|
||||
"includeEmpty": true
|
||||
},
|
||||
{
|
||||
"field": "sales",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-260, 580],
|
||||
"id": "02bfe12f-eb71-483b-bef5-6007af739ace",
|
||||
"name": "count with Empty1"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "countUnique",
|
||||
"field": "region",
|
||||
"includeEmpty": true
|
||||
},
|
||||
{
|
||||
"field": "sales",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [-260, 720],
|
||||
"id": "e67cd4b5-47ae-4d73-b933-905b04183db8",
|
||||
"name": "countUnique non-empty3"
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"count non-empty": [
|
||||
{
|
||||
"json": {
|
||||
"count_review": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"countUnique non-empty": [
|
||||
{
|
||||
"json": {
|
||||
"unique_count_review": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"count with Empty": [
|
||||
{
|
||||
"json": {
|
||||
"count_review": 7
|
||||
}
|
||||
}
|
||||
],
|
||||
"countUnique non-empty1": [
|
||||
{
|
||||
"json": {
|
||||
"unique_count_review": 5
|
||||
}
|
||||
}
|
||||
],
|
||||
"count non-empty1": [
|
||||
{
|
||||
"json": {
|
||||
"count_region": 5,
|
||||
"count_sales": 4
|
||||
}
|
||||
}
|
||||
],
|
||||
"countUnique non-empty2": [
|
||||
{
|
||||
"json": {
|
||||
"unique_count_region": 4,
|
||||
"count_sales": 4
|
||||
}
|
||||
}
|
||||
],
|
||||
"count with Empty1": [
|
||||
{
|
||||
"json": {
|
||||
"count_region": 6,
|
||||
"count_sales": 6
|
||||
}
|
||||
}
|
||||
],
|
||||
"countUnique non-empty3": [
|
||||
{
|
||||
"json": {
|
||||
"unique_count_region": 5,
|
||||
"count_sales": 6
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "sample data1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "sample data2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"sample data1": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "count non-empty",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "countUnique non-empty",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "count with Empty",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "countUnique non-empty1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"sample data2": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "count non-empty1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "countUnique non-empty2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "count with Empty1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "countUnique non-empty3",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {
|
||||
"executionOrder": "v1"
|
||||
},
|
||||
"versionId": "bb837da6-9160-4c61-b3a4-e7ef192bd5c6",
|
||||
"meta": {
|
||||
"templateCredsSetupCompleted": true,
|
||||
"instanceId": "b0d1a7f453aa2195078ce658269afa7743e3dc59edb733395b6c75bde57da7ff"
|
||||
},
|
||||
"id": "8ja9H2443AhLdBsS",
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
{
|
||||
"name": "Summarize Test",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [-320, -80],
|
||||
"id": "509c50eb-a5c3-4172-a553-82cce7ca0455",
|
||||
"name": "When clicking ‘Execute workflow’"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"data": [
|
||||
{
|
||||
"uid": "a0e797c2-ffcf-495b-96f3-982f7bc4eae5",
|
||||
"email": "Alfredo83@gmail.com",
|
||||
"firstname": "Charlie",
|
||||
"lastname": "Jacobi",
|
||||
"password": "34~XLKdGit",
|
||||
"Group": 0
|
||||
},
|
||||
{
|
||||
"uid": "5713682e-4f32-47dc-ae89-ae4a8ac28a46",
|
||||
"email": "Zachary43@yahoo.com",
|
||||
"firstname": "Leonard",
|
||||
"lastname": "Blick",
|
||||
"password": "4sh5z!R",
|
||||
"Group": 0
|
||||
},
|
||||
{
|
||||
"uid": "1bb2a432-d2fb-48e4-a7b0-39030ef0d784",
|
||||
"email": "Cory_Wilderman@yahoo.com",
|
||||
"firstname": "Debra",
|
||||
"lastname": "Corkery",
|
||||
"password": "-8X1CKMFh",
|
||||
"Group": 1
|
||||
},
|
||||
{
|
||||
"uid": "4edb11bc-9256-4f26-9be6-f8b04e90ecb2",
|
||||
"email": "Mindy_Murazik86@gmail.com",
|
||||
"firstname": "Wendy",
|
||||
"lastname": "Schiller",
|
||||
"password": "WF373t,fe",
|
||||
"Group": 2
|
||||
},
|
||||
{
|
||||
"uid": "ec6efd3d-508c-4f1e-acae-c9467b74f5a6",
|
||||
"email": "Darin_Vandervort@yahoo.com",
|
||||
"firstname": "Dolores",
|
||||
"lastname": "Walter",
|
||||
"password": ".45KCBk",
|
||||
"Group": 2
|
||||
},
|
||||
{
|
||||
"uid": "173ee780-4e4f-4164-aa39-41a0402cd2ad",
|
||||
"email": "Claude.Grady6@gmail.com",
|
||||
"firstname": "Leonard",
|
||||
"lastname": "Krajcik",
|
||||
"password": "6x6-MN8vch",
|
||||
"Group": null
|
||||
},
|
||||
{
|
||||
"uid": "b1f191d8-4f95-49f6-8533-042048b4c162",
|
||||
"email": "Ismael_Volkman@hotmail.com",
|
||||
"firstname": "Terri",
|
||||
"lastname": "Kuphal",
|
||||
"password": "62+BNNCk",
|
||||
"Group": null
|
||||
},
|
||||
{
|
||||
"uid": "ee8c6d18-1e77-435d-8c9a-5215141d7697",
|
||||
"email": "Paulette.Hudson@hotmail.com",
|
||||
"firstname": "Clinton",
|
||||
"lastname": "Murphy",
|
||||
"password": "2+R8IRxHL",
|
||||
"Group": null
|
||||
},
|
||||
{
|
||||
"uid": "feeb4357-0e20-4b15-9199-f161c23da1a2",
|
||||
"email": "Guillermo.Kub33@gmail.com",
|
||||
"firstname": "Edward",
|
||||
"lastname": "Schaden",
|
||||
"password": "Kv23S1_H",
|
||||
"Group": 0
|
||||
},
|
||||
{
|
||||
"uid": "676003ec-8a40-4105-8237-ba1d9eb51fc2",
|
||||
"email": "Cindy.White@hotmail.com",
|
||||
"firstname": "Esther",
|
||||
"lastname": "Monahan",
|
||||
"password": "5q$RyBhp",
|
||||
"Group": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "n8n-nodes-testing.testData",
|
||||
"typeVersion": 1,
|
||||
"position": [-100, -80],
|
||||
"id": "691f2744-8689-4b5c-91e7-2bb378691ecf",
|
||||
"name": "sample data"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "email",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldsToSplitBy": "Group",
|
||||
"options": {
|
||||
"outputFormat": "separateItems"
|
||||
}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [160, 0],
|
||||
"id": "d1267da5-0cab-453c-9450-1bbd78346bd6",
|
||||
"name": "Summarize v1.1"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"aggregation": "append",
|
||||
"field": "email",
|
||||
"includeEmpty": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"fieldsToSplitBy": "Group",
|
||||
"options": {
|
||||
"continueIfFieldNotFound": false,
|
||||
"outputFormat": "separateItems"
|
||||
}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1,
|
||||
"position": [160, -160],
|
||||
"id": "680b7269-c920-4cce-b9f5-b2189955848d",
|
||||
"name": "Summarize v1"
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Summarize v1": [
|
||||
{
|
||||
"json": {
|
||||
"Group": "0",
|
||||
"appended_email": [
|
||||
"Alfredo83@gmail.com",
|
||||
"Zachary43@yahoo.com",
|
||||
"Guillermo.Kub33@gmail.com"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": "1",
|
||||
"appended_email": ["Cory_Wilderman@yahoo.com"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": "2",
|
||||
"appended_email": [
|
||||
"Mindy_Murazik86@gmail.com",
|
||||
"Darin_Vandervort@yahoo.com",
|
||||
"Cindy.White@hotmail.com"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": "null",
|
||||
"appended_email": [
|
||||
"Claude.Grady6@gmail.com",
|
||||
"Ismael_Volkman@hotmail.com",
|
||||
"Paulette.Hudson@hotmail.com"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Summarize v1.1": [
|
||||
{
|
||||
"json": {
|
||||
"Group": 0,
|
||||
"appended_email": [
|
||||
"Alfredo83@gmail.com",
|
||||
"Zachary43@yahoo.com",
|
||||
"Guillermo.Kub33@gmail.com"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": 1,
|
||||
"appended_email": ["Cory_Wilderman@yahoo.com"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": 2,
|
||||
"appended_email": [
|
||||
"Mindy_Murazik86@gmail.com",
|
||||
"Darin_Vandervort@yahoo.com",
|
||||
"Cindy.White@hotmail.com"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"Group": null,
|
||||
"appended_email": [
|
||||
"Claude.Grady6@gmail.com",
|
||||
"Ismael_Volkman@hotmail.com",
|
||||
"Paulette.Hudson@hotmail.com"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking ‘Execute workflow’": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "sample data",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"sample data": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Summarize v1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Summarize v1.1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {
|
||||
"executionOrder": "v1"
|
||||
},
|
||||
"versionId": "7304eae2-3bae-4934-8d1f-1135a0ff8ab8",
|
||||
"meta": {
|
||||
"templateCredsSetupCompleted": true,
|
||||
"instanceId": "b0d1a7f453aa2195078ce658269afa7743e3dc59edb733395b6c75bde57da7ff"
|
||||
},
|
||||
"id": "8ja9H2443AhLdBsS",
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [520, -80],
|
||||
"id": "9ff1d8e0-ebe8-4c52-931a-e482e14ac911",
|
||||
"name": "When clicking ‘Execute workflow’"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"category": "randomData",
|
||||
"randomDataSeed": "ria",
|
||||
"randomDataCount": 5
|
||||
},
|
||||
"type": "n8n-nodes-base.debugHelper",
|
||||
"typeVersion": 1,
|
||||
"position": [700, -80],
|
||||
"id": "50cea846-f8ed-4b5a-a14a-c11bf4ba0f5d",
|
||||
"name": "DebugHelper2"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fieldsToSummarize": {
|
||||
"values": [
|
||||
{
|
||||
"field": "passwor"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
"type": "n8n-nodes-base.summarize",
|
||||
"typeVersion": 1.1,
|
||||
"position": [900, -80],
|
||||
"id": "d39fedc9-62e1-45bc-8c9a-d20f2b63b543",
|
||||
"name": "Summarize1"
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"When clicking ‘Execute workflow’": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "DebugHelper2",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"DebugHelper2": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Summarize1",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"pinData": {},
|
||||
"meta": {
|
||||
"templateCredsSetupCompleted": true,
|
||||
"instanceId": "ee90fdf8d57662f949e6c691dc07fa0fd2f66e1eee28ed82ef06658223e67255"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
import get from 'lodash/get';
|
||||
import {
|
||||
type GenericValue,
|
||||
type IDataObject,
|
||||
type IExecuteFunctions,
|
||||
NodeOperationError,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
type AggregationType =
|
||||
| 'append'
|
||||
| 'average'
|
||||
| 'concatenate'
|
||||
| 'count'
|
||||
| 'countUnique'
|
||||
| 'max'
|
||||
| 'min'
|
||||
| 'sum';
|
||||
|
||||
export type Aggregation = {
|
||||
aggregation: AggregationType;
|
||||
field: string;
|
||||
includeEmpty?: boolean;
|
||||
separateBy?: string;
|
||||
customSeparator?: string;
|
||||
};
|
||||
|
||||
export type Aggregations = Aggregation[];
|
||||
|
||||
const AggregationDisplayNames = {
|
||||
append: 'appended_',
|
||||
average: 'average_',
|
||||
concatenate: 'concatenated_',
|
||||
count: 'count_',
|
||||
countUnique: 'unique_count_',
|
||||
max: 'max_',
|
||||
min: 'min_',
|
||||
sum: 'sum_',
|
||||
};
|
||||
|
||||
export const NUMERICAL_AGGREGATIONS = ['average', 'sum'];
|
||||
|
||||
export type SummarizeOptions = {
|
||||
continueIfFieldNotFound: boolean;
|
||||
disableDotNotation?: boolean;
|
||||
outputFormat?: 'separateItems' | 'singleItem';
|
||||
skipEmptySplitFields?: boolean;
|
||||
};
|
||||
|
||||
export type ValueGetterFn = (
|
||||
item: IDataObject,
|
||||
field: string,
|
||||
) => IDataObject | IDataObject[] | GenericValue | GenericValue[];
|
||||
|
||||
function isEmpty<T>(value: T) {
|
||||
return value === undefined || value === null || value === '';
|
||||
}
|
||||
|
||||
function normalizeFieldName(fieldName: string) {
|
||||
return fieldName.replace(/[\]\["]/g, '').replace(/[ .]/g, '_');
|
||||
}
|
||||
|
||||
export const fieldValueGetter = (disableDotNotation?: boolean) => {
|
||||
return (item: IDataObject, field: string) =>
|
||||
disableDotNotation ? item[field] : get(item, field);
|
||||
};
|
||||
|
||||
export function checkIfFieldExists(
|
||||
this: IExecuteFunctions,
|
||||
items: IDataObject[],
|
||||
aggregations: Aggregations,
|
||||
getValue: ValueGetterFn,
|
||||
) {
|
||||
for (const aggregation of aggregations) {
|
||||
if (aggregation.field === '') {
|
||||
continue;
|
||||
}
|
||||
const exist = items.some((item) => getValue(item, aggregation.field) !== undefined);
|
||||
if (!exist) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The field '${aggregation.field}' does not exist in any items`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) {
|
||||
const { aggregation, field } = entry;
|
||||
let data = [...items];
|
||||
|
||||
if (NUMERICAL_AGGREGATIONS.includes(aggregation)) {
|
||||
data = data.filter(
|
||||
(item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)),
|
||||
);
|
||||
}
|
||||
|
||||
switch (aggregation) {
|
||||
//combine operations
|
||||
case 'append':
|
||||
if (!entry.includeEmpty) {
|
||||
data = data.filter((item) => !isEmpty(getValue(item, field)));
|
||||
}
|
||||
return data.map((item) => getValue(item, field));
|
||||
case 'concatenate':
|
||||
const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy;
|
||||
if (!entry.includeEmpty) {
|
||||
data = data.filter((item) => !isEmpty(getValue(item, field)));
|
||||
}
|
||||
return data
|
||||
.map((item) => {
|
||||
let value = getValue(item, field);
|
||||
if (typeof value === 'object') {
|
||||
value = JSON.stringify(value);
|
||||
}
|
||||
if (typeof value === 'undefined') {
|
||||
value = 'undefined';
|
||||
}
|
||||
|
||||
return value;
|
||||
})
|
||||
.join(separateBy);
|
||||
|
||||
//numerical operations
|
||||
case 'average':
|
||||
return (
|
||||
data.reduce((acc, item) => {
|
||||
return acc + (getValue(item, field) as number);
|
||||
}, 0) / data.length
|
||||
);
|
||||
case 'sum':
|
||||
return data.reduce((acc, item) => {
|
||||
return acc + (getValue(item, field) as number);
|
||||
}, 0);
|
||||
//comparison operations
|
||||
case 'min':
|
||||
let min;
|
||||
for (const item of data) {
|
||||
const value = getValue(item, field);
|
||||
if (value !== undefined && value !== null && value !== '') {
|
||||
if (min === undefined || value < min) {
|
||||
min = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return min ?? null;
|
||||
case 'max':
|
||||
let max;
|
||||
for (const item of data) {
|
||||
const value = getValue(item, field);
|
||||
if (value !== undefined && value !== null && value !== '') {
|
||||
if (max === undefined || value > max) {
|
||||
max = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return max ?? null;
|
||||
|
||||
//count operations
|
||||
case 'countUnique':
|
||||
if (!entry.includeEmpty) {
|
||||
return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item)))
|
||||
.size;
|
||||
}
|
||||
return new Set(data.map((item) => getValue(item, field))).size;
|
||||
|
||||
default:
|
||||
//count by default
|
||||
if (!entry.includeEmpty) {
|
||||
return data.filter((item) => !isEmpty(getValue(item, field))).length;
|
||||
}
|
||||
return data.length;
|
||||
}
|
||||
}
|
||||
|
||||
function aggregateData(
|
||||
data: IDataObject[],
|
||||
fieldsToSummarize: Aggregations,
|
||||
options: SummarizeOptions,
|
||||
getValue: ValueGetterFn,
|
||||
): { returnData: IDataObject; pairedItems?: number[] } {
|
||||
const returnData = Object.fromEntries(
|
||||
fieldsToSummarize.map((aggregation) => {
|
||||
const key = normalizeFieldName(
|
||||
`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`,
|
||||
);
|
||||
const result = aggregate(data, aggregation, getValue);
|
||||
return [key, result];
|
||||
}),
|
||||
);
|
||||
|
||||
if (options.outputFormat === 'singleItem') {
|
||||
return { returnData };
|
||||
}
|
||||
|
||||
return { returnData, pairedItems: data.map((item) => item._itemIndex as number) };
|
||||
}
|
||||
|
||||
type AggregationResult = { returnData: IDataObject; pairedItems?: number[] };
|
||||
type NestedAggregationResult =
|
||||
| AggregationResult
|
||||
| { fieldName: string; splits: Map<unknown, NestedAggregationResult> };
|
||||
|
||||
// Using Map to preserve types
|
||||
// With a plain JS object, keys are converted to string
|
||||
export function aggregateAndSplitData({
|
||||
splitKeys,
|
||||
inputItems,
|
||||
fieldsToSummarize,
|
||||
options,
|
||||
getValue,
|
||||
convertKeysToString = false,
|
||||
}: {
|
||||
splitKeys: string[] | undefined;
|
||||
inputItems: IDataObject[];
|
||||
fieldsToSummarize: Aggregations;
|
||||
options: SummarizeOptions;
|
||||
getValue: ValueGetterFn;
|
||||
convertKeysToString?: boolean; // Legacy option for v1
|
||||
}): NestedAggregationResult {
|
||||
if (!splitKeys?.length) {
|
||||
return aggregateData(inputItems, fieldsToSummarize, options, getValue);
|
||||
}
|
||||
|
||||
const [firstSplitKey, ...restSplitKeys] = splitKeys;
|
||||
|
||||
const groupedItems = new Map<unknown, IDataObject[]>();
|
||||
for (const item of inputItems) {
|
||||
let splitValue = getValue(item, firstSplitKey);
|
||||
|
||||
if (splitValue && typeof splitValue === 'object') {
|
||||
splitValue = JSON.stringify(splitValue);
|
||||
}
|
||||
|
||||
if (convertKeysToString) {
|
||||
splitValue = String(splitValue);
|
||||
}
|
||||
|
||||
if (options.skipEmptySplitFields && typeof splitValue !== 'number' && !splitValue) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const group = groupedItems.get(splitValue) ?? [];
|
||||
groupedItems.set(splitValue, group.concat([item]));
|
||||
}
|
||||
|
||||
const splits = new Map(
|
||||
Array.from(groupedItems.entries()).map(([groupKey, items]) => [
|
||||
groupKey,
|
||||
aggregateAndSplitData({
|
||||
splitKeys: restSplitKeys,
|
||||
inputItems: items,
|
||||
fieldsToSummarize,
|
||||
options,
|
||||
getValue,
|
||||
convertKeysToString,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
return { fieldName: firstSplitKey, splits };
|
||||
}
|
||||
|
||||
export function flattenAggregationResultToObject(result: NestedAggregationResult): IDataObject {
|
||||
if ('splits' in result) {
|
||||
return Object.fromEntries(
|
||||
Array.from(result.splits.entries()).map(([key, value]) => [
|
||||
key,
|
||||
flattenAggregationResultToObject(value),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return result.returnData;
|
||||
}
|
||||
|
||||
export function flattenAggregationResultToArray(
|
||||
result: NestedAggregationResult,
|
||||
): AggregationResult[] {
|
||||
if ('splits' in result) {
|
||||
return Array.from(result.splits.entries()).flatMap(([value, innerResult]) =>
|
||||
flattenAggregationResultToArray(innerResult).map((v) => {
|
||||
v.returnData[normalizeFieldName(result.fieldName)] = value as IDataObject;
|
||||
return v;
|
||||
}),
|
||||
);
|
||||
}
|
||||
return [result];
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { ApplicationError } from '@n8n/errors';
|
||||
|
||||
export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => {
|
||||
if (typeof fields === 'string') {
|
||||
return fields
|
||||
.split(',')
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry !== '');
|
||||
}
|
||||
if (Array.isArray(fields)) {
|
||||
return fields;
|
||||
}
|
||||
throw new ApplicationError(
|
||||
`The \'${fieldName}\' parameter must be a string of fields separated by commas or an array of strings.`,
|
||||
{ level: 'warning' },
|
||||
);
|
||||
};
|
||||
Reference in New Issue
Block a user