first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
import type {
|
||||
IExecuteFunctions,
|
||||
INodeType,
|
||||
INodeTypeBaseDescription,
|
||||
INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { router } from './actions/router';
|
||||
import { versionDescription } from './actions/versionDescription';
|
||||
import { loadOptions } from './methods';
|
||||
|
||||
export class MergeV3 implements INodeType {
|
||||
description: INodeTypeDescription;
|
||||
|
||||
constructor(baseDescription: INodeTypeBaseDescription) {
|
||||
this.description = {
|
||||
...baseDescription,
|
||||
...versionDescription,
|
||||
};
|
||||
}
|
||||
|
||||
methods = {
|
||||
loadOptions,
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions) {
|
||||
return await router.call(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import {
|
||||
type IExecuteFunctions,
|
||||
type INodeExecutionData,
|
||||
type INodeProperties,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { numberInputsProperty } from '../../helpers/descriptions';
|
||||
|
||||
export const properties: INodeProperties[] = [numberInputsProperty];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['append'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
for (let i = 0; i < inputsData.length; i++) {
|
||||
returnData.push.apply(returnData, inputsData[i]);
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
|
||||
|
||||
import { preparePairedItemDataArray, updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { numberInputsProperty } from '../../helpers/descriptions';
|
||||
|
||||
export const properties: INodeProperties[] = [
|
||||
numberInputsProperty,
|
||||
{
|
||||
displayName: 'Output Type',
|
||||
name: 'chooseBranchMode',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Wait for All Inputs to Arrive',
|
||||
value: 'waitForAll',
|
||||
},
|
||||
],
|
||||
default: 'waitForAll',
|
||||
},
|
||||
{
|
||||
displayName: 'Output',
|
||||
name: 'output',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Data of Specified Input',
|
||||
value: 'specifiedInput',
|
||||
},
|
||||
{
|
||||
name: 'A Single, Empty Item',
|
||||
value: 'empty',
|
||||
},
|
||||
],
|
||||
default: 'specifiedInput',
|
||||
displayOptions: {
|
||||
show: {
|
||||
chooseBranchMode: ['waitForAll'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options
|
||||
displayName: 'Use Data of Input',
|
||||
name: 'useDataOfInput',
|
||||
type: 'options',
|
||||
default: 1,
|
||||
displayOptions: {
|
||||
show: {
|
||||
output: ['specifiedInput'],
|
||||
},
|
||||
},
|
||||
typeOptions: {
|
||||
minValue: 1,
|
||||
loadOptionsMethod: 'getInputs',
|
||||
loadOptionsDependsOn: ['numberInputs'],
|
||||
},
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-description-wrong-for-dynamic-options
|
||||
description: 'The number of the input to use data of',
|
||||
validateType: 'number',
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['chooseBranch'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
const chooseBranchMode = this.getNodeParameter('chooseBranchMode', 0) as string;
|
||||
|
||||
if (chooseBranchMode === 'waitForAll') {
|
||||
const output = this.getNodeParameter('output', 0) as string;
|
||||
|
||||
if (output === 'specifiedInput') {
|
||||
const useDataOfInput = this.getNodeParameter('useDataOfInput', 0) as number;
|
||||
if (useDataOfInput > inputsData.length) {
|
||||
throw new NodeOperationError(this.getNode(), `Input ${useDataOfInput} doesn't exist`, {
|
||||
description: `The node has only ${inputsData.length} inputs, so selecting input ${useDataOfInput} is not possible.`,
|
||||
});
|
||||
}
|
||||
|
||||
const inputData = inputsData[useDataOfInput - 1];
|
||||
|
||||
returnData.push.apply(returnData, inputData);
|
||||
}
|
||||
if (output === 'empty') {
|
||||
const pairedItem = [
|
||||
...this.getInputData(0).map((inputData) => inputData.pairedItem),
|
||||
...this.getInputData(1).map((inputData) => inputData.pairedItem),
|
||||
].flatMap(preparePairedItemDataArray);
|
||||
|
||||
returnData.push({
|
||||
json: {},
|
||||
pairedItem,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import merge from 'lodash/merge';
|
||||
import type {
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
IPairedItemData,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { clashHandlingProperties, fuzzyCompareProperty } from '../../helpers/descriptions';
|
||||
import type { ClashResolveOptions } from '../../helpers/interfaces';
|
||||
import { addSuffixToEntriesKeys, selectMergeMethod } from '../../helpers/utils';
|
||||
|
||||
export const properties: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [clashHandlingProperties, fuzzyCompareProperty],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['combine'],
|
||||
combineBy: ['combineAll'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
const clashHandling = this.getNodeParameter(
|
||||
'options.clashHandling.values',
|
||||
0,
|
||||
{},
|
||||
) as ClashResolveOptions;
|
||||
|
||||
let input1 = inputsData[0];
|
||||
let input2 = inputsData[1];
|
||||
|
||||
if (clashHandling.resolveClash === 'preferInput1') {
|
||||
[input1, input2] = [input2, input1];
|
||||
}
|
||||
|
||||
if (clashHandling.resolveClash === 'addSuffix') {
|
||||
input1 = addSuffixToEntriesKeys(input1, '1');
|
||||
input2 = addSuffixToEntriesKeys(input2, '2');
|
||||
}
|
||||
|
||||
const mergeIntoSingleObject = selectMergeMethod(clashHandling);
|
||||
|
||||
if (!input1 || !input2) {
|
||||
return [returnData];
|
||||
}
|
||||
|
||||
let entry1: INodeExecutionData;
|
||||
let entry2: INodeExecutionData;
|
||||
|
||||
for (entry1 of input1) {
|
||||
for (entry2 of input2) {
|
||||
returnData.push({
|
||||
json: {
|
||||
...mergeIntoSingleObject(entry1.json, entry2.json),
|
||||
},
|
||||
binary: {
|
||||
...merge({}, entry1.binary, entry2.binary),
|
||||
},
|
||||
pairedItem: [entry1.pairedItem as IPairedItemData, entry2.pairedItem as IPairedItemData],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
import type {
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { clashHandlingProperties, fuzzyCompareProperty } from '../../helpers/descriptions';
|
||||
import type {
|
||||
ClashResolveOptions,
|
||||
MatchFieldsJoinMode,
|
||||
MatchFieldsOptions,
|
||||
MatchFieldsOutput,
|
||||
} from '../../helpers/interfaces';
|
||||
import {
|
||||
addSourceField,
|
||||
addSuffixToEntriesKeys,
|
||||
checkInput,
|
||||
checkMatchFieldsInput,
|
||||
findMatches,
|
||||
mergeMatched,
|
||||
} from '../../helpers/utils';
|
||||
|
||||
const multipleMatchesProperty: INodeProperties = {
|
||||
displayName: 'Multiple Matches',
|
||||
name: 'multipleMatches',
|
||||
type: 'options',
|
||||
default: 'all',
|
||||
options: [
|
||||
{
|
||||
name: 'Include All Matches',
|
||||
value: 'all',
|
||||
description: 'Output multiple items if there are multiple matches',
|
||||
},
|
||||
{
|
||||
name: 'Include First Match Only',
|
||||
value: 'first',
|
||||
description: 'Only ever output a single item per match',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const properties: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Fields To Match Have Different Names',
|
||||
name: 'advanced',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether name(s) of field to match are different in input 1 and input 2',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Match',
|
||||
name: 'fieldsToMatchString',
|
||||
type: 'string',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
|
||||
placeholder: 'e.g. id, name',
|
||||
default: '',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Specify the fields to use for matching input items',
|
||||
hint: 'Drag or type the input field name',
|
||||
displayOptions: {
|
||||
show: {
|
||||
advanced: [false],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Match',
|
||||
name: 'mergeByFields',
|
||||
type: 'fixedCollection',
|
||||
placeholder: 'Add Fields to Match',
|
||||
default: { values: [{ field1: '', field2: '' }] },
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
description: 'Specify the fields to use for matching input items',
|
||||
displayOptions: {
|
||||
show: {
|
||||
advanced: [true],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Values',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Input 1 Field',
|
||||
name: 'field1',
|
||||
type: 'string',
|
||||
default: '',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
|
||||
placeholder: 'e.g. id',
|
||||
hint: 'Drag or type the input field name',
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
{
|
||||
displayName: 'Input 2 Field',
|
||||
name: 'field2',
|
||||
type: 'string',
|
||||
default: '',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
|
||||
placeholder: 'e.g. id',
|
||||
hint: 'Drag or type the input field name',
|
||||
requiresDataPath: 'single',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Output Type',
|
||||
name: 'joinMode',
|
||||
type: 'options',
|
||||
description: 'How to select the items to send to output',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
|
||||
options: [
|
||||
{
|
||||
name: 'Keep Matches',
|
||||
value: 'keepMatches',
|
||||
description: 'Items that match, merged together (inner join)',
|
||||
},
|
||||
{
|
||||
name: 'Keep Non-Matches',
|
||||
value: 'keepNonMatches',
|
||||
description: "Items that don't match",
|
||||
},
|
||||
{
|
||||
name: 'Keep Everything',
|
||||
value: 'keepEverything',
|
||||
description: "Items that match merged together, plus items that don't match (outer join)",
|
||||
},
|
||||
{
|
||||
name: 'Enrich Input 1',
|
||||
value: 'enrichInput1',
|
||||
description: 'All of input 1, with data from input 2 added in (left join)',
|
||||
},
|
||||
{
|
||||
name: 'Enrich Input 2',
|
||||
value: 'enrichInput2',
|
||||
description: 'All of input 2, with data from input 1 added in (right join)',
|
||||
},
|
||||
],
|
||||
default: 'keepMatches',
|
||||
},
|
||||
{
|
||||
displayName: 'Output Data From',
|
||||
name: 'outputDataFrom',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Both Inputs Merged Together',
|
||||
value: 'both',
|
||||
},
|
||||
{
|
||||
name: 'Input 1',
|
||||
value: 'input1',
|
||||
},
|
||||
{
|
||||
name: 'Input 2',
|
||||
value: 'input2',
|
||||
},
|
||||
],
|
||||
default: 'both',
|
||||
displayOptions: {
|
||||
show: {
|
||||
joinMode: ['keepMatches'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Output Data From',
|
||||
name: 'outputDataFrom',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Both Inputs Appended Together',
|
||||
value: 'both',
|
||||
},
|
||||
{
|
||||
name: 'Input 1',
|
||||
value: 'input1',
|
||||
},
|
||||
{
|
||||
name: 'Input 2',
|
||||
value: 'input2',
|
||||
},
|
||||
],
|
||||
default: 'both',
|
||||
displayOptions: {
|
||||
show: {
|
||||
joinMode: ['keepNonMatches'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
...clashHandlingProperties,
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/joinMode': ['keepMatches', 'keepNonMatches'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
...clashHandlingProperties,
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/joinMode': ['keepMatches'],
|
||||
'/outputDataFrom': ['both'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
fuzzyCompareProperty,
|
||||
{
|
||||
...multipleMatchesProperty,
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/joinMode': ['keepMatches'],
|
||||
'/outputDataFrom': ['both'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
...multipleMatchesProperty,
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/joinMode': ['enrichInput1', 'enrichInput2', 'keepEverything'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['combine'],
|
||||
combineBy: ['combineByFields'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
let returnData: INodeExecutionData[] = [];
|
||||
const advanced = this.getNodeParameter('advanced', 0) as boolean;
|
||||
let matchFields;
|
||||
|
||||
if (advanced) {
|
||||
matchFields = this.getNodeParameter('mergeByFields.values', 0, []) as IDataObject[];
|
||||
} else {
|
||||
matchFields = (this.getNodeParameter('fieldsToMatchString', 0, '') as string)
|
||||
.split(',')
|
||||
.map((f) => {
|
||||
const field = f.trim();
|
||||
return { field1: field, field2: field };
|
||||
});
|
||||
}
|
||||
|
||||
matchFields = checkMatchFieldsInput(matchFields);
|
||||
|
||||
const joinMode = this.getNodeParameter('joinMode', 0) as MatchFieldsJoinMode;
|
||||
const outputDataFrom = this.getNodeParameter('outputDataFrom', 0, 'both') as MatchFieldsOutput;
|
||||
const options = this.getNodeParameter('options', 0, {}) as MatchFieldsOptions;
|
||||
|
||||
options.joinMode = joinMode;
|
||||
options.outputDataFrom = outputDataFrom;
|
||||
|
||||
const nodeVersion = this.getNode().typeVersion;
|
||||
|
||||
let input1 = inputsData[0];
|
||||
let input2 = inputsData[1];
|
||||
|
||||
if (nodeVersion < 2.1) {
|
||||
input1 = checkInput(
|
||||
this.getInputData(0),
|
||||
matchFields.map((pair) => pair.field1),
|
||||
options.disableDotNotation || false,
|
||||
'Input 1',
|
||||
);
|
||||
if (!input1) return [returnData];
|
||||
|
||||
input2 = checkInput(
|
||||
this.getInputData(1),
|
||||
matchFields.map((pair) => pair.field2),
|
||||
options.disableDotNotation || false,
|
||||
'Input 2',
|
||||
);
|
||||
} else {
|
||||
if (!input1) return [returnData];
|
||||
}
|
||||
|
||||
if (input1.length === 0 || input2.length === 0) {
|
||||
if (!input1.length && joinMode === 'keepNonMatches' && outputDataFrom === 'input1')
|
||||
return [returnData];
|
||||
if (!input2.length && joinMode === 'keepNonMatches' && outputDataFrom === 'input2')
|
||||
return [returnData];
|
||||
|
||||
if (joinMode === 'keepMatches') {
|
||||
// Stop the execution
|
||||
return [];
|
||||
} else if (joinMode === 'enrichInput1' && input1.length === 0) {
|
||||
// No data to enrich so stop
|
||||
return [];
|
||||
} else if (joinMode === 'enrichInput2' && input2.length === 0) {
|
||||
// No data to enrich so stop
|
||||
return [];
|
||||
} else {
|
||||
// Return the data of any of the inputs that contains data
|
||||
return [[...input1, ...input2]];
|
||||
}
|
||||
}
|
||||
|
||||
if (!input1) return [returnData];
|
||||
|
||||
if (!input2 || !matchFields.length) {
|
||||
if (
|
||||
joinMode === 'keepMatches' ||
|
||||
joinMode === 'keepEverything' ||
|
||||
joinMode === 'enrichInput2'
|
||||
) {
|
||||
return [returnData];
|
||||
}
|
||||
return [input1];
|
||||
}
|
||||
|
||||
const matches = findMatches(input1, input2, matchFields, options);
|
||||
|
||||
if (joinMode === 'keepMatches' || joinMode === 'keepEverything') {
|
||||
let output: INodeExecutionData[] = [];
|
||||
const clashResolveOptions = this.getNodeParameter(
|
||||
'options.clashHandling.values',
|
||||
0,
|
||||
{},
|
||||
) as ClashResolveOptions;
|
||||
|
||||
if (outputDataFrom === 'input1') {
|
||||
output = matches.matched.map((match) => match.entry);
|
||||
}
|
||||
if (outputDataFrom === 'input2') {
|
||||
output = matches.matched2;
|
||||
}
|
||||
if (outputDataFrom === 'both') {
|
||||
output = mergeMatched(matches.matched, clashResolveOptions);
|
||||
}
|
||||
|
||||
if (joinMode === 'keepEverything') {
|
||||
let unmatched1 = matches.unmatched1;
|
||||
let unmatched2 = matches.unmatched2;
|
||||
if (clashResolveOptions.resolveClash === 'addSuffix') {
|
||||
unmatched1 = addSuffixToEntriesKeys(unmatched1, '1');
|
||||
unmatched2 = addSuffixToEntriesKeys(unmatched2, '2');
|
||||
}
|
||||
output = [...output, ...unmatched1, ...unmatched2];
|
||||
}
|
||||
|
||||
returnData = returnData.concat(output);
|
||||
}
|
||||
|
||||
if (joinMode === 'keepNonMatches') {
|
||||
if (outputDataFrom === 'input1') {
|
||||
return [matches.unmatched1];
|
||||
}
|
||||
if (outputDataFrom === 'input2') {
|
||||
return [matches.unmatched2];
|
||||
}
|
||||
if (outputDataFrom === 'both') {
|
||||
let output: INodeExecutionData[] = [];
|
||||
output = output.concat(addSourceField(matches.unmatched1, 'input1'));
|
||||
output = output.concat(addSourceField(matches.unmatched2, 'input2'));
|
||||
return [output];
|
||||
}
|
||||
}
|
||||
|
||||
if (joinMode === 'enrichInput1' || joinMode === 'enrichInput2') {
|
||||
const clashResolveOptions = this.getNodeParameter(
|
||||
'options.clashHandling.values',
|
||||
0,
|
||||
{},
|
||||
) as ClashResolveOptions;
|
||||
|
||||
const mergedEntries = mergeMatched(matches.matched, clashResolveOptions, joinMode);
|
||||
|
||||
if (joinMode === 'enrichInput1') {
|
||||
if (clashResolveOptions.resolveClash === 'addSuffix') {
|
||||
returnData = returnData.concat(
|
||||
mergedEntries,
|
||||
addSuffixToEntriesKeys(matches.unmatched1, '1'),
|
||||
);
|
||||
} else {
|
||||
returnData = returnData.concat(mergedEntries, matches.unmatched1);
|
||||
}
|
||||
} else {
|
||||
if (clashResolveOptions.resolveClash === 'addSuffix') {
|
||||
returnData = returnData.concat(
|
||||
mergedEntries,
|
||||
addSuffixToEntriesKeys(matches.unmatched2, '2'),
|
||||
);
|
||||
} else {
|
||||
returnData = returnData.concat(mergedEntries, matches.unmatched2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import merge from 'lodash/merge';
|
||||
import type {
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
IPairedItemData,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { clashHandlingProperties, numberInputsProperty } from '../../helpers/descriptions';
|
||||
import type { ClashResolveOptions } from '../../helpers/interfaces';
|
||||
import { addSuffixToEntriesKeys, selectMergeMethod } from '../../helpers/utils';
|
||||
|
||||
export const properties: INodeProperties[] = [
|
||||
numberInputsProperty,
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
...clashHandlingProperties,
|
||||
default: { values: { resolveClash: 'addSuffix' } },
|
||||
},
|
||||
{
|
||||
displayName: 'Include Any Unpaired Items',
|
||||
name: 'includeUnpaired',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether unpaired items should be included in the result when there are differing numbers of items among the inputs',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['combine'],
|
||||
combineBy: ['combineByPosition'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
const clashHandling = this.getNodeParameter(
|
||||
'options.clashHandling.values',
|
||||
0,
|
||||
{},
|
||||
) as ClashResolveOptions;
|
||||
const includeUnpaired = this.getNodeParameter('options.includeUnpaired', 0, false) as boolean;
|
||||
|
||||
let preferredInputIndex: number;
|
||||
|
||||
if (clashHandling?.resolveClash?.includes('preferInput')) {
|
||||
preferredInputIndex = Number(clashHandling.resolveClash.replace('preferInput', '')) - 1;
|
||||
} else {
|
||||
preferredInputIndex = inputsData.length - 1;
|
||||
}
|
||||
|
||||
const preferred = inputsData[preferredInputIndex];
|
||||
|
||||
if (clashHandling.resolveClash === 'addSuffix') {
|
||||
for (const [inputIndex, input] of inputsData.entries()) {
|
||||
inputsData[inputIndex] = addSuffixToEntriesKeys(input, String(inputIndex + 1));
|
||||
}
|
||||
}
|
||||
|
||||
let numEntries: number;
|
||||
if (includeUnpaired) {
|
||||
numEntries = Math.max(...inputsData.map((input) => input.length), preferred.length);
|
||||
} else {
|
||||
numEntries = Math.min(...inputsData.map((input) => input.length), preferred.length);
|
||||
if (numEntries === 0) {
|
||||
this.addExecutionHints({
|
||||
message: 'Consider enabling "Include Any Unpaired Items" in options or check your inputs',
|
||||
});
|
||||
return [returnData];
|
||||
}
|
||||
}
|
||||
|
||||
const mergeIntoSingleObject = selectMergeMethod(clashHandling);
|
||||
|
||||
for (let i = 0; i < numEntries; i++) {
|
||||
const preferredEntry = preferred[i] ?? ({} as INodeExecutionData);
|
||||
const restEntries = inputsData.map((input) => input[i] ?? ({} as INodeExecutionData));
|
||||
|
||||
const json = {
|
||||
...mergeIntoSingleObject(
|
||||
{},
|
||||
...restEntries.map((entry) => entry.json ?? {}),
|
||||
preferredEntry.json ?? {},
|
||||
),
|
||||
};
|
||||
|
||||
const binary = {
|
||||
...merge({}, ...restEntries.map((entry) => entry.binary ?? {}), preferredEntry.binary ?? {}),
|
||||
};
|
||||
|
||||
const pairedItem = [
|
||||
...restEntries.map((entry) => entry.pairedItem as IPairedItemData).flat(),
|
||||
preferredEntry.pairedItem as IPairedItemData,
|
||||
].filter((item) => item !== undefined);
|
||||
|
||||
returnData.push({ json, binary, pairedItem });
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
import { Container } from '@n8n/di';
|
||||
import { ErrorReporter } from 'n8n-core';
|
||||
|
||||
import type {
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INode,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
IPairedItemData,
|
||||
} from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import { numberInputsProperty } from '../../helpers/descriptions';
|
||||
import { modifySelectQuery, rowToExecutionData } from '../../helpers/utils';
|
||||
import { loadAlaSqlSandbox, runAlaSqlInSandbox } from '../../helpers/sandbox-utils';
|
||||
|
||||
type OperationOptions = {
|
||||
emptyQueryResult: 'success' | 'empty';
|
||||
};
|
||||
|
||||
export const properties: INodeProperties[] = [
|
||||
numberInputsProperty,
|
||||
{
|
||||
displayName: 'Query',
|
||||
name: 'query',
|
||||
type: 'string',
|
||||
default: 'SELECT * FROM input1 LEFT JOIN input2 ON input1.name = input2.id',
|
||||
noDataExpression: true,
|
||||
description: 'Input data available as tables with corresponding number, e.g. input1, input2',
|
||||
hint: 'Supports <a href="https://github.com/alasql/alasql/wiki/Supported-SQL-statements" target="_blank">most</a> of the SQL-99 language',
|
||||
required: true,
|
||||
typeOptions: {
|
||||
rows: 5,
|
||||
editor: 'sqlEditor',
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Empty Query Result',
|
||||
name: 'emptyQueryResult',
|
||||
type: 'options',
|
||||
description: 'What to return if the query executed successfully but returned no results',
|
||||
options: [
|
||||
{
|
||||
name: 'Success',
|
||||
value: 'success',
|
||||
},
|
||||
{
|
||||
name: 'Empty Result',
|
||||
value: 'empty',
|
||||
},
|
||||
],
|
||||
default: 'empty',
|
||||
},
|
||||
],
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [3.2],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
mode: ['combineBySql'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
const prepareError = (node: INode, error: Error) => {
|
||||
let message = '';
|
||||
if (typeof error === 'string') {
|
||||
message = error;
|
||||
} else {
|
||||
message = error.message;
|
||||
}
|
||||
throw new NodeOperationError(node, error, {
|
||||
message: 'Issue while executing query',
|
||||
description: message,
|
||||
itemIndex: 0,
|
||||
});
|
||||
};
|
||||
|
||||
async function executeSelectWithMappedPairedItems(
|
||||
node: INode,
|
||||
inputsData: INodeExecutionData[][],
|
||||
query: string,
|
||||
returnSuccessItemIfEmpty: boolean,
|
||||
context: Awaited<ReturnType<typeof loadAlaSqlSandbox>>,
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
const tableData = inputsData.map((inputData) =>
|
||||
inputData.map((entry) => ({ ...entry.json, pairedItem: entry.pairedItem })),
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await runAlaSqlInSandbox(
|
||||
context,
|
||||
tableData,
|
||||
modifySelectQuery(query, inputsData.length),
|
||||
);
|
||||
|
||||
for (const item of result) {
|
||||
if (Array.isArray(item)) {
|
||||
returnData.push.apply(returnData, item.map(rowToExecutionData));
|
||||
} else if (typeof item === 'object') {
|
||||
returnData.push(rowToExecutionData(item));
|
||||
}
|
||||
}
|
||||
|
||||
if (!returnData.length && returnSuccessItemIfEmpty) {
|
||||
returnData.push({ json: { success: true } });
|
||||
}
|
||||
} catch (error) {
|
||||
prepareError(node, error as Error);
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
inputsData: INodeExecutionData[][],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const node = this.getNode();
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
const pairedItem: IPairedItemData[] = [];
|
||||
const options = this.getNodeParameter('options', 0, {}) as OperationOptions;
|
||||
const workflowId = this.getWorkflow().id;
|
||||
|
||||
let query = this.getNodeParameter('query', 0) as string;
|
||||
|
||||
for (const resolvable of getResolvables(query)) {
|
||||
query = query.replace(resolvable, this.evaluateExpression(resolvable, 0) as string);
|
||||
}
|
||||
|
||||
const context = await loadAlaSqlSandbox();
|
||||
|
||||
const isSelectQuery = node.typeVersion >= 3.1 ? query.toLowerCase().startsWith('select') : false;
|
||||
const returnSuccessItemIfEmpty =
|
||||
node.typeVersion <= 3.1 ? true : options.emptyQueryResult === 'success';
|
||||
|
||||
if (isSelectQuery) {
|
||||
try {
|
||||
return await executeSelectWithMappedPairedItems(
|
||||
node,
|
||||
inputsData,
|
||||
query,
|
||||
returnSuccessItemIfEmpty,
|
||||
context,
|
||||
);
|
||||
} catch (error) {
|
||||
Container.get(ErrorReporter).error(error, {
|
||||
extra: {
|
||||
nodeName: node.name,
|
||||
nodeType: node.type,
|
||||
nodeVersion: node.typeVersion,
|
||||
workflowId,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < inputsData.length; i++) {
|
||||
const inputData = inputsData[i];
|
||||
|
||||
inputData.forEach((item, index) => {
|
||||
if (item.pairedItem === undefined) {
|
||||
item.pairedItem = index;
|
||||
}
|
||||
|
||||
if (typeof item.pairedItem === 'number') {
|
||||
pairedItem.push({
|
||||
item: item.pairedItem,
|
||||
input: i,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(item.pairedItem)) {
|
||||
const pairedItems = item.pairedItem
|
||||
.filter((p) => p !== undefined)
|
||||
.map((p) => (typeof p === 'number' ? { item: p } : p))
|
||||
.map((p) => {
|
||||
return {
|
||||
item: p.item,
|
||||
input: i,
|
||||
};
|
||||
});
|
||||
pairedItem.push.apply(pairedItem, pairedItems);
|
||||
return;
|
||||
}
|
||||
|
||||
pairedItem.push({
|
||||
item: item.pairedItem.item,
|
||||
input: i,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const tableData: IDataObject[][] = inputsData.map((inputData) =>
|
||||
inputData.map((entry) => entry.json),
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await runAlaSqlInSandbox(context, tableData, query);
|
||||
|
||||
for (const item of result) {
|
||||
if (Array.isArray(item)) {
|
||||
returnData.push.apply(
|
||||
returnData,
|
||||
item.map((json) => ({ json, pairedItem })),
|
||||
);
|
||||
} else if (typeof item === 'object') {
|
||||
returnData.push({ json: item, pairedItem });
|
||||
}
|
||||
}
|
||||
|
||||
if (!returnData.length && returnSuccessItemIfEmpty) {
|
||||
returnData.push({ json: { success: true }, pairedItem });
|
||||
}
|
||||
} catch (error) {
|
||||
prepareError(node, error as Error);
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import type { INodeProperties } from 'n8n-workflow';
|
||||
|
||||
import * as append from './append';
|
||||
import * as chooseBranch from './chooseBranch';
|
||||
import * as combineAll from './combineAll';
|
||||
import * as combineByFields from './combineByFields';
|
||||
import * as combineByPosition from './combineByPosition';
|
||||
import * as combineBySql from './combineBySql';
|
||||
|
||||
export { append, chooseBranch, combineAll, combineByFields, combineBySql, combineByPosition };
|
||||
|
||||
export const description: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Mode',
|
||||
name: 'mode',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
options: [
|
||||
{
|
||||
name: 'Append',
|
||||
value: 'append',
|
||||
description: 'Output items of each input, one after the other',
|
||||
builderHint: {
|
||||
message:
|
||||
'Append items from multiple branches into a single list sequentially. Waits for all running branches. Supports any number of inputs. @example input1: [{ x }] [{ y }] input2: [{ z }]. Output: [{ x }, { y }, { z }]. Next node will execute 3 times with each item. Set executeOnce on next node to execute once.',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Combine',
|
||||
value: 'combine',
|
||||
description: 'Merge matching items together',
|
||||
builderHint: {
|
||||
message:
|
||||
'Combines items from 2 branches. Waits for both to have input data. @example **combine by position** input1: [{ x }, { y }] input2: [{ z }] output: [{ x, y }, { x: undefined, y: undefined, z }] @example **combine by key** input1: [{ id: 1, x }, { id: 2, y }] input2: [{ id: 1, z }] output: [{ id: 1, x, z }, { id: 2, y }]',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'SQL Query',
|
||||
value: 'combineBySql',
|
||||
description: 'Write a query to do the merge',
|
||||
builderHint: {
|
||||
message:
|
||||
'Need to combine more than 2 branches? Use SQL Query for advanced operations. Waits for all inputs. @example Results depend on query - can filter, join, aggregate',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Choose Branch',
|
||||
value: 'chooseBranch',
|
||||
description: 'Output data from a specific branch, without modifying it',
|
||||
builderHint: {
|
||||
message:
|
||||
'Do you need to select data from only ONE specific input and discard the others? Use Choose Branch after conditional nodes to pick which path to continue. Waits for all inputs. @example 3 items from Input A + 2 items from Input B, choose Input A → 3 items',
|
||||
},
|
||||
},
|
||||
],
|
||||
default: 'append',
|
||||
description: 'How input data should be merged',
|
||||
},
|
||||
{
|
||||
displayName: 'Combine By',
|
||||
name: 'combineBy',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
options: [
|
||||
{
|
||||
name: 'Matching Fields',
|
||||
value: 'combineByFields',
|
||||
description: 'Combine items with the same field values',
|
||||
},
|
||||
{
|
||||
name: 'Position',
|
||||
value: 'combineByPosition',
|
||||
description: 'Combine items based on their order',
|
||||
},
|
||||
{
|
||||
name: 'All Possible Combinations',
|
||||
value: 'combineAll',
|
||||
description: 'Every pairing of every two items (cross join)',
|
||||
},
|
||||
],
|
||||
default: 'combineByFields',
|
||||
description: 'How input data should be merged',
|
||||
displayOptions: {
|
||||
show: { mode: ['combine'] },
|
||||
},
|
||||
},
|
||||
...append.description,
|
||||
...combineAll.description,
|
||||
...combineByFields.description,
|
||||
...combineBySql.description,
|
||||
...combineByPosition.description,
|
||||
...chooseBranch.description,
|
||||
];
|
||||
@@ -0,0 +1,7 @@
|
||||
export type MergeType =
|
||||
| 'append'
|
||||
| 'combineByFields'
|
||||
| 'combineBySql'
|
||||
| 'combineByPosition'
|
||||
| 'combineAll'
|
||||
| 'chooseBranch';
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { IExecuteFunctions } from 'n8n-workflow';
|
||||
|
||||
import * as mode from './mode';
|
||||
import type { MergeType } from './node.type';
|
||||
import { getNodeInputsData } from '../helpers/utils';
|
||||
|
||||
export async function router(this: IExecuteFunctions) {
|
||||
const inputsData = getNodeInputsData.call(this);
|
||||
let operationMode = this.getNodeParameter('mode', 0) as string;
|
||||
|
||||
if (operationMode === 'combine') {
|
||||
const combineBy = this.getNodeParameter('combineBy', 0) as string;
|
||||
operationMode = combineBy;
|
||||
}
|
||||
|
||||
return await mode[operationMode as MergeType].execute.call(this, inputsData);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
|
||||
import { NodeConnectionTypes, type INodeTypeDescription } from 'n8n-workflow';
|
||||
|
||||
import * as mode from './mode';
|
||||
import { configuredInputs } from '../helpers/utils';
|
||||
|
||||
export const versionDescription: INodeTypeDescription = {
|
||||
displayName: 'Merge',
|
||||
name: 'merge',
|
||||
group: ['transform'],
|
||||
description: 'Merges data of multiple streams once data from both is available',
|
||||
version: [3, 3.1, 3.2],
|
||||
defaults: {
|
||||
name: 'Merge',
|
||||
},
|
||||
inputs: `={{(${configuredInputs})($parameter)}}`,
|
||||
outputs: [NodeConnectionTypes.Main],
|
||||
// If mode is chooseBranch data from both branches is required
|
||||
// to continue, else data from any input suffices
|
||||
requiredInputs: '={{ $parameter["mode"] === "chooseBranch" ? [0, 1] : 1 }}',
|
||||
properties: [...mode.description],
|
||||
};
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { INodeProperties } from 'n8n-workflow';
|
||||
|
||||
export const fuzzyCompareProperty: INodeProperties = {
|
||||
displayName: 'Fuzzy Compare',
|
||||
name: 'fuzzyCompare',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.",
|
||||
};
|
||||
export const numberInputsProperty: INodeProperties = {
|
||||
displayName: 'Number of Inputs',
|
||||
name: 'numberInputs',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
default: 2,
|
||||
options: [
|
||||
{
|
||||
name: '2',
|
||||
value: 2,
|
||||
},
|
||||
{
|
||||
name: '3',
|
||||
value: 3,
|
||||
},
|
||||
{
|
||||
name: '4',
|
||||
value: 4,
|
||||
},
|
||||
{
|
||||
name: '5',
|
||||
value: 5,
|
||||
},
|
||||
{
|
||||
name: '6',
|
||||
value: 6,
|
||||
},
|
||||
{
|
||||
name: '7',
|
||||
value: 7,
|
||||
},
|
||||
{
|
||||
name: '8',
|
||||
value: 8,
|
||||
},
|
||||
{
|
||||
name: '9',
|
||||
value: 9,
|
||||
},
|
||||
{
|
||||
name: '10',
|
||||
value: 10,
|
||||
},
|
||||
],
|
||||
validateType: 'number',
|
||||
description:
|
||||
'The number of data inputs you want to merge. The node waits for all connected inputs to be executed.',
|
||||
};
|
||||
|
||||
export const clashHandlingProperties: INodeProperties = {
|
||||
displayName: 'Clash Handling',
|
||||
name: 'clashHandling',
|
||||
type: 'fixedCollection',
|
||||
default: {
|
||||
values: { resolveClash: 'preferLast', mergeMode: 'deepMerge', overrideEmpty: false },
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Values',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options
|
||||
displayName: 'When Field Values Clash',
|
||||
name: 'resolveClash',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-description-missing-from-dynamic-options
|
||||
type: 'options',
|
||||
default: '',
|
||||
typeOptions: {
|
||||
loadOptionsMethod: 'getResolveClashOptions',
|
||||
loadOptionsDependsOn: ['numberInputs'],
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Merging Nested Fields',
|
||||
name: 'mergeMode',
|
||||
type: 'options',
|
||||
default: 'deepMerge',
|
||||
options: [
|
||||
{
|
||||
name: 'Deep Merge',
|
||||
value: 'deepMerge',
|
||||
description: 'Merge at every level of nesting',
|
||||
},
|
||||
{
|
||||
name: 'Shallow Merge',
|
||||
value: 'shallowMerge',
|
||||
description:
|
||||
'Merge at the top level only (all nested fields will come from the same input)',
|
||||
},
|
||||
],
|
||||
hint: 'How to merge when there are sub-fields below the top-level ones',
|
||||
displayOptions: {
|
||||
show: {
|
||||
resolveClash: [{ _cnd: { not: 'addSuffix' } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Minimize Empty Fields',
|
||||
name: 'overrideEmpty',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether to override the preferred input version for a field if it is empty and the other version isn't. Here 'empty' means undefined, null or an empty string.",
|
||||
displayOptions: {
|
||||
show: {
|
||||
resolveClash: [{ _cnd: { not: 'addSuffix' } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
type MultipleMatches = 'all' | 'first';
|
||||
|
||||
export type MatchFieldsOptions = {
|
||||
joinMode: MatchFieldsJoinMode;
|
||||
outputDataFrom: MatchFieldsOutput;
|
||||
multipleMatches: MultipleMatches;
|
||||
disableDotNotation: boolean;
|
||||
fuzzyCompare?: boolean;
|
||||
};
|
||||
|
||||
type ClashMergeMode = 'deepMerge' | 'shallowMerge';
|
||||
type ClashResolveMode = 'addSuffix' | 'preferInput1' | 'preferLast';
|
||||
|
||||
export type ClashResolveOptions = {
|
||||
resolveClash: ClashResolveMode;
|
||||
mergeMode: ClashMergeMode;
|
||||
overrideEmpty: boolean;
|
||||
};
|
||||
|
||||
export type MatchFieldsOutput = 'both' | 'input1' | 'input2';
|
||||
|
||||
export type MatchFieldsJoinMode =
|
||||
| 'keepEverything'
|
||||
| 'keepMatches'
|
||||
| 'keepNonMatches'
|
||||
| 'enrichInput2'
|
||||
| 'enrichInput1';
|
||||
@@ -0,0 +1,75 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import ivm from 'isolated-vm';
|
||||
|
||||
import type { IDataObject } from 'n8n-workflow';
|
||||
|
||||
// Singleton – recreated only after resetSandboxCache() (tests) or isolate disposal.
|
||||
let sandboxIsolate: ivm.Isolate | null = null;
|
||||
let sandboxContext: ivm.Context | null = null;
|
||||
|
||||
/** Disposes the cached isolate. Exposed for tests only. */
|
||||
export function resetSandboxCache(): void {
|
||||
sandboxContext = null;
|
||||
if (sandboxIsolate && !sandboxIsolate.isDisposed) {
|
||||
sandboxIsolate.dispose();
|
||||
}
|
||||
sandboxIsolate = null;
|
||||
}
|
||||
|
||||
/** Returns a cached isolated-vm context with alasql pre-loaded. Creates it on first call. */
|
||||
export async function loadAlaSqlSandbox(): Promise<ivm.Context> {
|
||||
if (sandboxContext && sandboxIsolate && !sandboxIsolate.isDisposed) {
|
||||
return sandboxContext;
|
||||
}
|
||||
|
||||
sandboxIsolate = new ivm.Isolate({ memoryLimit: 64 }); // 64 MB hard limit
|
||||
sandboxContext = await sandboxIsolate.createContext();
|
||||
|
||||
// Browser bundle only – no Node.js fs/require handlers inside the isolate.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const alasqlBundlePath = require.resolve('alasql/dist/alasql.min.js');
|
||||
await sandboxContext.eval(await readFile(alasqlBundlePath, 'utf-8'));
|
||||
await sandboxContext.eval('Object.freeze(alasql.fn)');
|
||||
|
||||
return sandboxContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a SQL query against plain-object table data inside the isolated-vm sandbox.
|
||||
* Only JSON-serialisable values cross the isolate boundary.
|
||||
*/
|
||||
export async function runAlaSqlInSandbox(
|
||||
context: ivm.Context,
|
||||
tableData: unknown[][],
|
||||
query: string,
|
||||
): Promise<IDataObject[]> {
|
||||
// UUID per invocation so concurrent calls sharing the singleton context
|
||||
// don't collide on alasql.databases.
|
||||
const dbId = randomUUID();
|
||||
|
||||
// Double-serialization: outer JSON.stringify produces a JSON string, inner produces a JSON literal
|
||||
// embedded in the script source. Inside the isolate, JSON.parse reconstructs the plain array.
|
||||
// This ensures data enters the isolate as a parsed JSON literal, never as live objects.
|
||||
const script = `(function() {
|
||||
const __rows = JSON.parse(${JSON.stringify(JSON.stringify(tableData))});
|
||||
const __db = new alasql.Database(${JSON.stringify(dbId)});
|
||||
try {
|
||||
for (let i = 0; i < __rows.length; i++) {
|
||||
__db.exec('CREATE TABLE input' + (i + 1));
|
||||
__db.tables['input' + (i + 1)].data = __rows[i];
|
||||
}
|
||||
return JSON.stringify(__db.exec(${JSON.stringify(query)}));
|
||||
} finally {
|
||||
delete alasql.databases[${JSON.stringify(dbId)}];
|
||||
}
|
||||
})()`;
|
||||
|
||||
const resultJson = (await context.eval(script, { timeout: 5000, copy: true })) as string;
|
||||
try {
|
||||
return JSON.parse(resultJson) as IDataObject[];
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to parse SQL result: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
import assign from 'lodash/assign';
|
||||
import assignWith from 'lodash/assignWith';
|
||||
import get from 'lodash/get';
|
||||
import merge from 'lodash/merge';
|
||||
import mergeWith from 'lodash/mergeWith';
|
||||
import type {
|
||||
GenericValue,
|
||||
IBinaryKeyData,
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeParameters,
|
||||
IPairedItemData,
|
||||
} from 'n8n-workflow';
|
||||
import { ApplicationError, NodeConnectionTypes, NodeHelpers } from 'n8n-workflow';
|
||||
|
||||
import { fuzzyCompare, preparePairedItemDataArray } from '@utils/utilities';
|
||||
|
||||
import type { ClashResolveOptions, MatchFieldsJoinMode, MatchFieldsOptions } from './interfaces';
|
||||
|
||||
type PairToMatch = {
|
||||
field1: string;
|
||||
field2: string;
|
||||
};
|
||||
|
||||
type EntryMatches = {
|
||||
entry: INodeExecutionData;
|
||||
matches: INodeExecutionData[];
|
||||
};
|
||||
|
||||
type CompareFunction = <T, U>(a: T, b: U) => boolean;
|
||||
|
||||
export function addSuffixToEntriesKeys(data: INodeExecutionData[], suffix: string) {
|
||||
return data.map((entry) => {
|
||||
const json: IDataObject = {};
|
||||
Object.keys(entry.json).forEach((key) => {
|
||||
json[`${key}_${suffix}`] = entry.json[key];
|
||||
});
|
||||
return { ...entry, json };
|
||||
});
|
||||
}
|
||||
|
||||
function findAllMatches(
|
||||
data: INodeExecutionData[],
|
||||
lookup: IDataObject,
|
||||
disableDotNotation: boolean,
|
||||
isEntriesEqual: CompareFunction,
|
||||
) {
|
||||
return data.reduce((acc, entry2, i) => {
|
||||
if (entry2 === undefined) return acc;
|
||||
|
||||
for (const key of Object.keys(lookup)) {
|
||||
const expectedValue = lookup[key];
|
||||
let entry2FieldValue;
|
||||
|
||||
if (disableDotNotation) {
|
||||
entry2FieldValue = entry2.json[key];
|
||||
} else {
|
||||
entry2FieldValue = get(entry2.json, key);
|
||||
}
|
||||
|
||||
if (!isEntriesEqual(expectedValue, entry2FieldValue)) {
|
||||
return acc;
|
||||
}
|
||||
}
|
||||
|
||||
return acc.concat({
|
||||
entry: entry2,
|
||||
index: i,
|
||||
});
|
||||
}, [] as IDataObject[]);
|
||||
}
|
||||
|
||||
function findFirstMatch(
|
||||
data: INodeExecutionData[],
|
||||
lookup: IDataObject,
|
||||
disableDotNotation: boolean,
|
||||
isEntriesEqual: CompareFunction,
|
||||
) {
|
||||
const index = data.findIndex((entry2) => {
|
||||
if (entry2 === undefined) return false;
|
||||
|
||||
for (const key of Object.keys(lookup)) {
|
||||
const expectedValue = lookup[key];
|
||||
let entry2FieldValue;
|
||||
|
||||
if (disableDotNotation) {
|
||||
entry2FieldValue = entry2.json[key];
|
||||
} else {
|
||||
entry2FieldValue = get(entry2.json, key);
|
||||
}
|
||||
|
||||
if (!isEntriesEqual(expectedValue, entry2FieldValue)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
if (index === -1) return [];
|
||||
|
||||
return [{ entry: data[index], index }];
|
||||
}
|
||||
|
||||
export function findMatches(
|
||||
input1: INodeExecutionData[],
|
||||
input2: INodeExecutionData[],
|
||||
fieldsToMatch: PairToMatch[],
|
||||
options: MatchFieldsOptions,
|
||||
) {
|
||||
const data1 = [...input1];
|
||||
const data2 = [...input2];
|
||||
|
||||
const isEntriesEqual = fuzzyCompare(options.fuzzyCompare as boolean);
|
||||
const disableDotNotation = options.disableDotNotation || false;
|
||||
const multipleMatches = (options.multipleMatches as string) || 'all';
|
||||
|
||||
const filteredData = {
|
||||
matched: [] as EntryMatches[],
|
||||
matched2: [] as INodeExecutionData[],
|
||||
unmatched1: [] as INodeExecutionData[],
|
||||
unmatched2: [] as INodeExecutionData[],
|
||||
};
|
||||
|
||||
const matchedInInput2 = new Set<number>();
|
||||
|
||||
matchesLoop: for (const entry1 of data1) {
|
||||
const lookup: IDataObject = {};
|
||||
|
||||
fieldsToMatch.forEach((matchCase) => {
|
||||
let valueToCompare;
|
||||
if (disableDotNotation) {
|
||||
valueToCompare = entry1.json[matchCase.field1];
|
||||
} else {
|
||||
valueToCompare = get(entry1.json, matchCase.field1);
|
||||
}
|
||||
lookup[matchCase.field2] = valueToCompare;
|
||||
});
|
||||
|
||||
for (const fieldValue of Object.values(lookup)) {
|
||||
if (fieldValue === undefined) {
|
||||
filteredData.unmatched1.push(entry1);
|
||||
continue matchesLoop;
|
||||
}
|
||||
}
|
||||
|
||||
const foundedMatches =
|
||||
multipleMatches === 'all'
|
||||
? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual)
|
||||
: findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual);
|
||||
|
||||
const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
|
||||
foundedMatches.map((match) => matchedInInput2.add(match.index as number));
|
||||
|
||||
if (matches.length) {
|
||||
if (
|
||||
options.outputDataFrom === 'both' ||
|
||||
options.joinMode === 'enrichInput1' ||
|
||||
options.joinMode === 'enrichInput2'
|
||||
) {
|
||||
matches.forEach((match) => {
|
||||
filteredData.matched.push({
|
||||
entry: entry1,
|
||||
matches: [match],
|
||||
});
|
||||
});
|
||||
} else {
|
||||
filteredData.matched.push({
|
||||
entry: entry1,
|
||||
matches,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
filteredData.unmatched1.push(entry1);
|
||||
}
|
||||
}
|
||||
|
||||
data2.forEach((entry, i) => {
|
||||
if (matchedInInput2.has(i)) {
|
||||
filteredData.matched2.push(entry);
|
||||
} else {
|
||||
filteredData.unmatched2.push(entry);
|
||||
}
|
||||
});
|
||||
|
||||
return filteredData;
|
||||
}
|
||||
|
||||
export function selectMergeMethod(clashResolveOptions: ClashResolveOptions) {
|
||||
const mergeMode = clashResolveOptions.mergeMode as string;
|
||||
|
||||
if (clashResolveOptions.overrideEmpty) {
|
||||
function customizer(targetValue: GenericValue, srcValue: GenericValue) {
|
||||
if (srcValue === undefined || srcValue === null || srcValue === '') {
|
||||
return targetValue;
|
||||
}
|
||||
}
|
||||
if (mergeMode === 'deepMerge') {
|
||||
return (target: IDataObject, ...source: IDataObject[]) => {
|
||||
const targetCopy = Object.assign({}, target);
|
||||
return mergeWith(targetCopy, ...source, customizer);
|
||||
};
|
||||
}
|
||||
if (mergeMode === 'shallowMerge') {
|
||||
return (target: IDataObject, ...source: IDataObject[]) => {
|
||||
const targetCopy = Object.assign({}, target);
|
||||
return assignWith(targetCopy, ...source, customizer);
|
||||
};
|
||||
}
|
||||
} else {
|
||||
if (mergeMode === 'deepMerge') {
|
||||
return (target: IDataObject, ...source: IDataObject[]) => merge({}, target, ...source);
|
||||
}
|
||||
if (mergeMode === 'shallowMerge') {
|
||||
return (target: IDataObject, ...source: IDataObject[]) => assign({}, target, ...source);
|
||||
}
|
||||
}
|
||||
return (target: IDataObject, ...source: IDataObject[]) => merge({}, target, ...source);
|
||||
}
|
||||
|
||||
export function mergeMatched(
|
||||
matched: EntryMatches[],
|
||||
clashResolveOptions: ClashResolveOptions,
|
||||
joinMode?: MatchFieldsJoinMode,
|
||||
) {
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
let resolveClash = clashResolveOptions.resolveClash as string;
|
||||
|
||||
const mergeIntoSingleObject = selectMergeMethod(clashResolveOptions);
|
||||
|
||||
for (const match of matched) {
|
||||
let { entry, matches } = match;
|
||||
|
||||
let json: IDataObject = {};
|
||||
let binary: IBinaryKeyData = {};
|
||||
let pairedItem: IPairedItemData[] = [];
|
||||
|
||||
if (resolveClash === 'addSuffix') {
|
||||
const suffix1 = '1';
|
||||
const suffix2 = '2';
|
||||
|
||||
[entry] = addSuffixToEntriesKeys([entry], suffix1);
|
||||
matches = addSuffixToEntriesKeys(matches, suffix2);
|
||||
|
||||
json = mergeIntoSingleObject({ ...entry.json }, ...matches.map((item) => item.json));
|
||||
binary = mergeIntoSingleObject(
|
||||
{ ...entry.binary },
|
||||
...matches.map((item) => item.binary as IDataObject),
|
||||
);
|
||||
pairedItem = [
|
||||
...preparePairedItemDataArray(entry.pairedItem),
|
||||
...matches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
|
||||
];
|
||||
} else {
|
||||
const preferInput1 = 'preferInput1';
|
||||
const preferLast = 'preferLast';
|
||||
|
||||
if (resolveClash === undefined) {
|
||||
if (joinMode !== 'enrichInput2') {
|
||||
resolveClash = 'preferLast';
|
||||
} else {
|
||||
resolveClash = 'preferInput1';
|
||||
}
|
||||
}
|
||||
|
||||
if (resolveClash === preferInput1) {
|
||||
const [firstMatch, ...restMatches] = matches;
|
||||
json = mergeIntoSingleObject(
|
||||
{ ...firstMatch.json },
|
||||
...restMatches.map((item) => item.json),
|
||||
entry.json,
|
||||
);
|
||||
binary = mergeIntoSingleObject(
|
||||
{ ...firstMatch.binary },
|
||||
...restMatches.map((item) => item.binary as IDataObject),
|
||||
entry.binary as IDataObject,
|
||||
);
|
||||
|
||||
pairedItem = [
|
||||
...preparePairedItemDataArray(firstMatch.pairedItem),
|
||||
...restMatches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
|
||||
...preparePairedItemDataArray(entry.pairedItem),
|
||||
];
|
||||
}
|
||||
|
||||
if (resolveClash === preferLast) {
|
||||
json = mergeIntoSingleObject({ ...entry.json }, ...matches.map((item) => item.json));
|
||||
binary = mergeIntoSingleObject(
|
||||
{ ...entry.binary },
|
||||
...matches.map((item) => item.binary as IDataObject),
|
||||
);
|
||||
pairedItem = [
|
||||
...preparePairedItemDataArray(entry.pairedItem),
|
||||
...matches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
returnData.push({
|
||||
json,
|
||||
binary,
|
||||
pairedItem,
|
||||
});
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
|
||||
export function checkMatchFieldsInput(data: IDataObject[]) {
|
||||
if (data.length === 1 && data[0].field1 === '' && data[0].field2 === '') {
|
||||
throw new ApplicationError(
|
||||
'You need to define at least one pair of fields in "Fields to Match" to match on',
|
||||
{ level: 'warning' },
|
||||
);
|
||||
}
|
||||
for (const [index, pair] of data.entries()) {
|
||||
if (pair.field1 === '' || pair.field2 === '') {
|
||||
throw new ApplicationError(
|
||||
`You need to define both fields in "Fields to Match" for pair ${index + 1},
|
||||
field 1 = '${pair.field1}'
|
||||
field 2 = '${pair.field2}'`,
|
||||
{ level: 'warning' },
|
||||
);
|
||||
}
|
||||
}
|
||||
return data as PairToMatch[];
|
||||
}
|
||||
|
||||
export function checkInput(
|
||||
input: INodeExecutionData[],
|
||||
fields: string[],
|
||||
disableDotNotation: boolean,
|
||||
inputLabel: string,
|
||||
) {
|
||||
for (const field of fields) {
|
||||
const isPresent = (input || []).some((entry) => {
|
||||
if (disableDotNotation) {
|
||||
return entry.json.hasOwnProperty(field);
|
||||
}
|
||||
return get(entry.json, field, undefined) !== undefined;
|
||||
});
|
||||
if (!isPresent) {
|
||||
throw new ApplicationError(
|
||||
`Field '${field}' is not present in any of items in '${inputLabel}'`,
|
||||
{ level: 'warning' },
|
||||
);
|
||||
}
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
export function addSourceField(data: INodeExecutionData[], sourceField: string) {
|
||||
return data.map((entry) => {
|
||||
const json = {
|
||||
...entry.json,
|
||||
_source: sourceField,
|
||||
};
|
||||
return {
|
||||
...entry,
|
||||
json,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export const configuredInputs = (parameters: INodeParameters) => {
|
||||
return Array.from({ length: (parameters.numberInputs as number) || 2 }, (_, i) => ({
|
||||
type: 'main',
|
||||
displayName: `Input ${(i + 1).toString()}`,
|
||||
}));
|
||||
};
|
||||
|
||||
export function getNodeInputsData(this: IExecuteFunctions) {
|
||||
const returnData: INodeExecutionData[][] = [];
|
||||
|
||||
const inputs = NodeHelpers.getConnectionTypes(this.getNodeInputs()).filter(
|
||||
(type) => type === NodeConnectionTypes.Main,
|
||||
);
|
||||
|
||||
for (let i = 0; i < inputs.length; i++) {
|
||||
try {
|
||||
returnData.push(this.getInputData(i) ?? []);
|
||||
} catch (error) {
|
||||
returnData.push([]);
|
||||
}
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
|
||||
export const rowToExecutionData = (data: IDataObject): INodeExecutionData => {
|
||||
const keys = Object.keys(data);
|
||||
const pairedItem: IPairedItemData[] = [];
|
||||
const json: IDataObject = {};
|
||||
|
||||
for (const key of keys) {
|
||||
if (key.startsWith('pairedItem')) {
|
||||
if (data[key] === undefined) continue;
|
||||
pairedItem.push(data[key] as IPairedItemData);
|
||||
} else {
|
||||
json[key] = data[key];
|
||||
}
|
||||
}
|
||||
|
||||
return { json, pairedItem };
|
||||
};
|
||||
|
||||
export function modifySelectQuery(userQuery: string, inputLength: number): string {
|
||||
const selectMatch = userQuery.match(/SELECT\s+(.+?)\s+FROM/i);
|
||||
if (!selectMatch) return userQuery;
|
||||
|
||||
let selectedColumns = selectMatch[1].trim();
|
||||
|
||||
if (selectedColumns === '*') {
|
||||
return userQuery;
|
||||
}
|
||||
|
||||
const pairedItemColumns = [];
|
||||
|
||||
for (let i = 1; i <= inputLength; i++) {
|
||||
if (userQuery.includes(`input${i}`)) {
|
||||
pairedItemColumns.push(`input${i}.pairedItem AS pairedItem${i}`);
|
||||
}
|
||||
}
|
||||
|
||||
selectedColumns += pairedItemColumns.length ? ', ' + pairedItemColumns.join(', ') : '';
|
||||
|
||||
return userQuery.replace(selectMatch[0], `SELECT ${selectedColumns} FROM`);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * as loadOptions from './loadOptions';
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
|
||||
|
||||
export async function getResolveClashOptions(
|
||||
this: ILoadOptionsFunctions,
|
||||
): Promise<INodePropertyOptions[]> {
|
||||
const numberOfInputs = this.getNodeParameter('numberInputs', 2) as number;
|
||||
|
||||
if (numberOfInputs <= 2) {
|
||||
return [
|
||||
{
|
||||
name: 'Always Add Input Number to Field Names',
|
||||
value: 'addSuffix',
|
||||
},
|
||||
{
|
||||
name: 'Prefer Input 1 Version',
|
||||
value: 'preferInput1',
|
||||
},
|
||||
{
|
||||
name: 'Prefer Input 2 Version',
|
||||
value: 'preferLast',
|
||||
},
|
||||
];
|
||||
} else {
|
||||
return [
|
||||
{
|
||||
name: 'Always Add Input Number to Field Names',
|
||||
value: 'addSuffix',
|
||||
},
|
||||
{
|
||||
name: 'Use Earliest Version',
|
||||
value: 'preferInput1',
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
export async function getInputs(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
|
||||
const numberOfInputs = this.getNodeParameter('numberInputs', 2) as number;
|
||||
|
||||
const returnData: INodePropertyOptions[] = [];
|
||||
|
||||
for (let i = 0; i < numberOfInputs; i++) {
|
||||
returnData.push({
|
||||
name: `${i + 1}`,
|
||||
value: i + 1,
|
||||
});
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
Reference in New Issue
Block a user