alighasami/n8n
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Details
Test: Benchmark Nightly / build (push) Has been cancelled
Details
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
Details
CI: Python / Checks (push) Has been cancelled
Details
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Details
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Details
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Details
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Details
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Details
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Details
Test: Benchmark Destroy Env / build (push) Has been cancelled
Details
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Details
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
alighasami
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts
+200
View File
@@ -0,0 +1,200 @@
import type { TextSplitter } from '@langchain/textsplitters';
import {
NodeConnectionTypes,
type INodeType,
type INodeTypeDescription,
type ISupplyDataFunctions,
type SupplyData,
} from 'n8n-workflow';
import {
logWrapper,
N8nBinaryLoader,
getConnectionHintNoticeField,
metadataFilterField,
} from '@n8n/ai-utilities';
// Dependencies needed underneath the hood for the loaders. We add them
// here only to track where what dependency is sued
// import 'd3-dsv'; // for csv
import 'mammoth'; // for docx
import 'epub2'; // for epub
import 'pdf-parse'; // for pdf
export class DocumentBinaryInputLoader implements INodeType {
description: INodeTypeDescription = {
// This node is deprecated and will be removed in the future.
// The functionality was merged with the `DocumentJSONInputLoader` to `DocumentDefaultDataLoader`
hidden: true,
displayName: 'Binary Input Loader',
name: 'documentBinaryInputLoader',
icon: 'file:binary.svg',
group: ['transform'],
version: 1,
description: 'Use binary data from a previous step in the workflow',
defaults: {
name: 'Binary Input Loader',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Document Loaders'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.documentdefaultdataloader/',
},
],
},
},
inputs: [
{
displayName: 'Text Splitter',
maxConnections: 1,
type: NodeConnectionTypes.AiTextSplitter,
required: true,
},
],
outputs: [NodeConnectionTypes.AiDocument],
outputNames: ['Document'],
builderHint: {
inputs: {
ai_textSplitter: { required: true },
},
},
properties: [
getConnectionHintNoticeField([NodeConnectionTypes.AiVectorStore]),
{
displayName: 'Loader Type',
name: 'loader',
type: 'options',
default: 'jsonLoader',
required: true,
options: [
{
name: 'CSV Loader',
value: 'csvLoader',
description: 'Load CSV files',
},
{
name: 'Docx Loader',
value: 'docxLoader',
description: 'Load Docx documents',
},
{
name: 'EPub Loader',
value: 'epubLoader',
description: 'Load EPub files',
},
{
name: 'JSON Loader',
value: 'jsonLoader',
description: 'Load JSON files',
},
{
name: 'PDF Loader',
value: 'pdfLoader',
description: 'Load PDF documents',
},
{
name: 'Text Loader',
value: 'textLoader',
description: 'Load plain text files',
},
],
},
{
displayName: 'Binary Data Key',
name: 'binaryDataKey',
type: 'string',
default: 'data',
required: true,
description: 'Name of the binary property from which to read the file buffer',
},
// PDF Only Fields
{
displayName: 'Split Pages',
name: 'splitPages',
type: 'boolean',
default: true,
displayOptions: {
show: {
loader: ['pdfLoader'],
},
},
},
// CSV Only Fields
{
displayName: 'Column',
name: 'column',
type: 'string',
default: '',
description: 'Column to extract from CSV',
displayOptions: {
show: {
loader: ['csvLoader'],
},
},
},
{
displayName: 'Separator',
name: 'separator',
type: 'string',
description: 'Separator to use for CSV',
default: ',',
displayOptions: {
show: {
loader: ['csvLoader'],
},
},
},
// JSON Only Fields
{
displayName: 'Pointers',
name: 'pointers',
type: 'string',
default: '',
description: 'Pointers to extract from JSON, e.g. "/text" or "/text, /meta/title"',
displayOptions: {
show: {
loader: ['jsonLoader'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
...metadataFilterField,
displayName: 'Metadata',
description:
'Metadata to add to each document. Could be used for filtering during retrieval',
placeholder: 'Add property',
},
],
},
],
};
async supplyData(this: ISupplyDataFunctions): Promise<SupplyData> {
this.logger.debug('Supply Data for Binary Input Loader');
const textSplitter = (await this.getInputConnectionData(
NodeConnectionTypes.AiTextSplitter,
0,
)) as TextSplitter | undefined;
const binaryDataKey = this.getNodeParameter('binaryDataKey', 0) as string;
const processor = new N8nBinaryLoader(this, undefined, binaryDataKey, textSplitter);
return {
response: logWrapper(processor, this),
};
}
}
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/binary.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="768" height="1024"><path fill="#7D7D87" d="M0 960V64h576l192 192v704zm704-640L512 128H64v768h640zM320 512H128V256h192zm-64-192h-64v128h64zm0 448h64v64H128v-64h64V640h-64v-64h128zm256-320h64v64H384v-64h64V320h-64v-64h128zm64 384H384V576h192zm-64-192h-64v128h64z"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 316 B

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts
+364
View File
@@ -0,0 +1,364 @@
import { RecursiveCharacterTextSplitter, type TextSplitter } from '@langchain/textsplitters';
import {
NodeConnectionTypes,
type INodeType,
type INodeTypeDescription,
type ISupplyDataFunctions,
type SupplyData,
type IDataObject,
type INodeInputConfiguration,
} from 'n8n-workflow';
import { logWrapper, N8nBinaryLoader, N8nJsonLoader, metadataFilterField } from '@n8n/ai-utilities';
// Dependencies needed underneath the hood for the loaders. We add them
// here only to track where what dependency is sued
// import 'd3-dsv'; // for csv
import 'mammoth'; // for docx
import 'epub2'; // for epub
import 'pdf-parse'; // for pdf
/* istanbul ignore next */
function getInputs(parameters: IDataObject) {
const inputs: INodeInputConfiguration[] = [];
const textSplittingMode = parameters?.textSplittingMode;
// If text splitting mode is 'custom' or does not exist (v1), we need to add an input for the text splitter
if (!textSplittingMode || textSplittingMode === 'custom') {
inputs.push({
displayName: 'Text Splitter',
maxConnections: 1,
type: 'ai_textSplitter',
required: true,
});
}
return inputs;
}
export class DocumentDefaultDataLoader implements INodeType {
description: INodeTypeDescription = {
displayName: 'Default Data Loader',
name: 'documentDefaultDataLoader',
icon: 'file:binary.svg',
group: ['transform'],
version: [1, 1.1],
defaultVersion: 1.1,
description: 'Load data from previous step in the workflow',
defaults: {
name: 'Default Data Loader',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Document Loaders'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.documentdefaultdataloader/',
},
],
},
},
inputs: `={{ ((parameter) => { ${getInputs.toString()}; return getInputs(parameter) })($parameter) }}`,
outputs: [NodeConnectionTypes.AiDocument],
outputNames: ['Document'],
builderHint: {
inputs: {
ai_textSplitter: {
required: true,
displayOptions: { show: { textSplittingMode: ['custom'] } },
},
},
},
properties: [
{
displayName:
'This will load data from a previous step in the workflow. <a href="/templates/1962" target="_blank">Example</a>',
name: 'notice',
type: 'notice',
default: '',
},
{
displayName: 'Type of Data',
name: 'dataType',
type: 'options',
default: 'json',
required: true,
noDataExpression: true,
options: [
{
name: 'JSON',
value: 'json',
description: 'Process JSON data from previous step in the workflow',
},
{
name: 'Binary',
value: 'binary',
description: 'Process binary data from previous step in the workflow',
},
],
},
{
displayName: 'Mode',
name: 'jsonMode',
type: 'options',
default: 'allInputData',
required: true,
displayOptions: {
show: {
dataType: ['json'],
},
},
options: [
{
name: 'Load All Input Data',
value: 'allInputData',
description: 'Use all JSON data that flows into the parent agent or chain',
},
{
name: 'Load Specific Data',
value: 'expressionData',
description:
'Load a subset of data, and/or data from any previous step in the workflow',
},
],
},
{
displayName: 'Mode',
name: 'binaryMode',
type: 'options',
default: 'allInputData',
required: true,
displayOptions: {
show: {
dataType: ['binary'],
},
},
options: [
{
name: 'Load All Input Data',
value: 'allInputData',
description: 'Use all Binary data that flows into the parent agent or chain',
},
{
name: 'Load Specific Data',
value: 'specificField',
description: 'Load data from a specific field in the parent agent or chain',
},
],
},
{
displayName: 'Data Format',
name: 'loader',
type: 'options',
default: 'auto',
required: true,
displayOptions: {
show: {
dataType: ['binary'],
},
},
options: [
{
name: 'Automatically Detect by Mime Type',
value: 'auto',
description: 'Uses the mime type to detect the format',
},
{
name: 'CSV',
value: 'csvLoader',
description: 'Load CSV files',
},
{
name: 'Docx',
value: 'docxLoader',
description: 'Load Docx documents',
},
{
name: 'EPub',
value: 'epubLoader',
description: 'Load EPub files',
},
{
name: 'JSON',
value: 'jsonLoader',
description: 'Load JSON files',
},
{
name: 'PDF',
value: 'pdfLoader',
description: 'Load PDF documents',
},
{
name: 'Text',
value: 'textLoader',
description: 'Load plain text files',
},
],
},
{
displayName: 'Data',
name: 'jsonData',
type: 'string',
typeOptions: {
rows: 6,
},
default: '',
required: true,
description: 'Drag and drop fields from the input pane, or use an expression',
displayOptions: {
show: {
dataType: ['json'],
jsonMode: ['expressionData'],
},
},
},
{
displayName: 'Input Data Field Name',
name: 'binaryDataKey',
type: 'string',
default: 'data',
required: true,
description:
'The name of the field in the agent or chain’s input that contains the binary file to be processed',
displayOptions: {
show: {
dataType: ['binary'],
},
hide: {
binaryMode: ['allInputData'],
},
},
},
{
displayName: 'Text Splitting',
name: 'textSplittingMode',
type: 'options',
default: 'simple',
required: true,
noDataExpression: true,
displayOptions: {
show: {
'@version': [1.1],
},
},
options: [
{
name: 'Simple',
value: 'simple',
description: 'Splits every 1000 characters with a 200 character overlap',
},
{
name: 'Custom',
value: 'custom',
description: 'Connect a custom text-splitting sub-node',
},
],
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'JSON Pointers',
name: 'pointers',
type: 'string',
default: '',
description: 'Pointers to extract from JSON, e.g. "/text" or "/text, /meta/title"',
displayOptions: {
show: {
'/loader': ['jsonLoader', 'auto'],
},
},
},
{
displayName: 'CSV Separator',
name: 'separator',
type: 'string',
description: 'Separator to use for CSV',
default: ',',
displayOptions: {
show: {
'/loader': ['csvLoader', 'auto'],
},
},
},
{
displayName: 'CSV Column',
name: 'column',
type: 'string',
default: '',
description: 'Column to extract from CSV',
displayOptions: {
show: {
'/loader': ['csvLoader', 'auto'],
},
},
},
{
displayName: 'Split Pages in PDF',
description: 'Whether to split PDF pages into separate documents',
name: 'splitPages',
type: 'boolean',
default: true,
displayOptions: {
show: {
'/loader': ['pdfLoader', 'auto'],
},
},
},
{
...metadataFilterField,
displayName: 'Metadata',
description:
'Metadata to add to each document. Could be used for filtering during retrieval',
placeholder: 'Add property',
},
],
},
],
};
async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {
const node = this.getNode();
const dataType = this.getNodeParameter('dataType', itemIndex, 'json') as 'json' | 'binary';
let textSplitter: TextSplitter | undefined;
if (node.typeVersion === 1.1) {
const textSplittingMode = this.getNodeParameter('textSplittingMode', itemIndex, 'simple') as
| 'simple'
| 'custom';
if (textSplittingMode === 'simple') {
textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
} else if (textSplittingMode === 'custom') {
textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
| TextSplitter
| undefined;
}
} else {
textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
| TextSplitter
| undefined;
}
const binaryDataKey = this.getNodeParameter('binaryDataKey', itemIndex, '') as string;
const processor =
dataType === 'binary'
? new N8nBinaryLoader(this, 'options.', binaryDataKey, textSplitter)
: new N8nJsonLoader(this, 'options.', textSplitter);
return {
response: logWrapper(processor, this),
};
}
}
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/binary.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="768" height="1024"><path fill="#7D7D87" d="M0 960V64h576l192 192v704zm704-640L512 128H64v768h640zM320 512H128V256h192zm-64-192h-64v128h64zm0 448h64v64H128v-64h64V640h-64v-64h128zm256-320h64v64H384v-64h64V320h-64v-64h128zm64 384H384V576h192zm-64-192h-64v128h64z"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 316 B

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/test/DocumentDefaultDataLoader.node.test.ts
+72
View File
@@ -0,0 +1,72 @@
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import type { ISupplyDataFunctions } from 'n8n-workflow';
import { NodeConnectionTypes } from 'n8n-workflow';
import { DocumentDefaultDataLoader } from '../DocumentDefaultDataLoader.node';
jest.mock('@langchain/textsplitters', () => ({
RecursiveCharacterTextSplitter: jest.fn().mockImplementation(() => ({
splitDocuments: jest.fn(
async (docs: Array<Record<string, unknown>>): Promise<Array<Record<string, unknown>>> =>
docs.map((doc) => ({ ...doc, split: true })),
),
})),
}));
describe('DocumentDefaultDataLoader', () => {
let loader: DocumentDefaultDataLoader;
beforeEach(() => {
loader = new DocumentDefaultDataLoader();
jest.clearAllMocks();
});
it('should supply data with recursive char text splitter', async () => {
const context = {
getNode: jest.fn(() => ({ typeVersion: 1.1 })),
getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
switch (paramName) {
case 'dataType':
return 'json';
case 'textSplittingMode':
return 'simple';
case 'binaryDataKey':
return 'data';
default:
return;
}
}),
} as unknown as ISupplyDataFunctions;
await loader.supplyData.call(context, 0);
expect(RecursiveCharacterTextSplitter).toHaveBeenCalledWith({
chunkSize: 1000,
chunkOverlap: 200,
});
});
it('should supply data with custom text splitter', async () => {
const customSplitter = { splitDocuments: jest.fn(async (docs) => docs) };
const context = {
getNode: jest.fn(() => ({ typeVersion: 1.1 })),
getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
switch (paramName) {
case 'dataType':
return 'json';
case 'textSplittingMode':
return 'custom';
case 'binaryDataKey':
return 'data';
default:
return;
}
}),
getInputConnectionData: jest.fn(async () => customSplitter),
} as unknown as ISupplyDataFunctions;
await loader.supplyData.call(context, 0);
expect(context.getInputConnectionData).toHaveBeenCalledWith(
NodeConnectionTypes.AiTextSplitter,
0,
);
});
});
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts
+196
View File
@@ -0,0 +1,196 @@
import { GithubRepoLoader } from '@langchain/community/document_loaders/web/github';
import type { TextSplitter } from '@langchain/textsplitters';
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import { logWrapper, getConnectionHintNoticeField } from '@n8n/ai-utilities';
import {
NodeConnectionTypes,
type INodeType,
type INodeTypeDescription,
type ISupplyDataFunctions,
type SupplyData,
type IDataObject,
type INodeInputConfiguration,
} from 'n8n-workflow';
/* istanbul ignore next */
function getInputs(parameters: IDataObject) {
const inputs: INodeInputConfiguration[] = [];
const textSplittingMode = parameters?.textSplittingMode;
// If text splitting mode is 'custom' or does not exist (v1), we need to add an input for the text splitter
if (!textSplittingMode || textSplittingMode === 'custom') {
inputs.push({
displayName: 'Text Splitter',
maxConnections: 1,
type: 'ai_textSplitter',
required: true,
});
}
return inputs;
}
export class DocumentGithubLoader implements INodeType {
description: INodeTypeDescription = {
displayName: 'GitHub Document Loader',
name: 'documentGithubLoader',
icon: 'file:github.svg',
group: ['transform'],
version: [1, 1.1],
defaultVersion: 1.1,
description: 'Use GitHub data as input to this chain',
hidden: true,
defaults: {
name: 'GitHub Document Loader',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Document Loaders'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.documentgithubloader/',
},
],
},
},
credentials: [
{
name: 'githubApi',
required: true,
},
],
inputs: `={{ ((parameter) => { ${getInputs.toString()}; return getInputs(parameter) })($parameter) }}`,
inputNames: ['Text Splitter'],
outputs: [NodeConnectionTypes.AiDocument],
outputNames: ['Document'],
builderHint: {
inputs: {
ai_textSplitter: {
required: true,
displayOptions: { show: { textSplittingMode: ['custom'] } },
},
},
},
properties: [
getConnectionHintNoticeField([NodeConnectionTypes.AiVectorStore]),
{
displayName: 'Repository Link',
name: 'repository',
type: 'string',
default: '',
},
{
displayName: 'Branch',
name: 'branch',
type: 'string',
default: 'main',
},
{
displayName: 'Text Splitting',
name: 'textSplittingMode',
type: 'options',
default: 'simple',
required: true,
noDataExpression: true,
displayOptions: {
show: {
'@version': [1.1],
},
},
options: [
{
name: 'Simple',
value: 'simple',
description: 'Splits every 1000 characters with a 200 character overlap',
},
{
name: 'Custom',
value: 'custom',
description: 'Connect a custom text-splitting sub-node',
},
],
},
{
displayName: 'Options',
name: 'additionalOptions',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Recursive',
name: 'recursive',
type: 'boolean',
default: false,
},
{
displayName: 'Ignore Paths',
name: 'ignorePaths',
type: 'string',
description: 'Comma-separated list of paths to ignore, e.g. "docs, src/tests',
default: '',
},
],
},
],
};
async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {
this.logger.debug('Supplying data for Github Document Loader');
const node = this.getNode();
const repository = this.getNodeParameter('repository', itemIndex) as string;
const branch = this.getNodeParameter('branch', itemIndex) as string;
const credentials = await this.getCredentials('githubApi');
const { ignorePaths, recursive } = this.getNodeParameter('additionalOptions', 0) as {
recursive: boolean;
ignorePaths: string;
};
let textSplitter: TextSplitter | undefined;
if (node.typeVersion === 1.1) {
const textSplittingMode = this.getNodeParameter('textSplittingMode', itemIndex, 'simple') as
| 'simple'
| 'custom';
if (textSplittingMode === 'simple') {
textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
} else if (textSplittingMode === 'custom') {
textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
| TextSplitter
| undefined;
}
} else {
textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
| TextSplitter
| undefined;
}
const { index } = this.addInputData(NodeConnectionTypes.AiDocument, [
[{ json: { repository, branch, ignorePaths, recursive } }],
]);
const docs = new GithubRepoLoader(repository, {
branch,
ignorePaths: (ignorePaths ?? '').split(',').map((p) => p.trim()),
recursive,
accessToken: (credentials.accessToken as string) || '',
apiUrl: credentials.server as string,
});
const loadedDocs = textSplitter
? await textSplitter.splitDocuments(await docs.load())
: await docs.load();
this.addOutputData(NodeConnectionTypes.AiDocument, index, [[{ json: { loadedDocs } }]]);
return {
response: logWrapper(loadedDocs, this),
};
}
}
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/github.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" fill="#fff" fill-rule="evenodd" stroke="#000" stroke-linecap="round" stroke-linejoin="round" viewBox="0 0 148.744 150.744"><use xlink:href="#a" x=".872" y=".872"/><symbol id="a" overflow="visible"><path fill="#7D7D87" stroke="none" d="M73.256 0C32.801 0 0 34.029 0 76.001c0 33.586 20.988 62.069 50.1 72.115 3.663.698 4.999-1.652 4.999-3.656l-.105-14.149c-20.372 4.593-24.677-8.961-24.677-8.961-3.335-8.777-8.133-11.114-8.133-11.114-6.658-4.713.523-4.622.523-4.622 7.355.529 11.227 7.831 11.227 7.831 6.537 11.616 17.151 8.257 21.319 6.309.666-4.901 2.564-8.257 4.65-10.151-16.261-1.919-33.366-8.442-33.366-37.565 0-8.302 2.857-15.075 7.535-20.396-.747-1.929-3.269-9.663.724-20.123 0 0 6.143-2.041 20.145 7.793 5.84-1.692 12.105-2.529 18.314-2.555 6.223.028 12.492.872 18.34 2.564 13.978-9.844 20.128-7.793 20.128-7.793 4.006 10.47 1.483 18.192.733 20.114 4.695 5.32 7.53 12.093 7.53 20.396 0 29.198-17.133 35.627-33.453 37.509 2.639 2.355 4.971 6.977 4.971 14.065l-.098 20.855c0 2.023 1.333 4.388 5.044 3.663 29.091-10.078 50.062-38.561 50.062-72.129C146.512 34.029 113.71 0 73.256 0"/></symbol></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/test/DocumentGithubLoader.node.test.ts
+99
View File
@@ -0,0 +1,99 @@
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import type { ISupplyDataFunctions } from 'n8n-workflow';
import { NodeConnectionTypes } from 'n8n-workflow';
import { DocumentGithubLoader } from '../DocumentGithubLoader.node';
jest.mock('@langchain/textsplitters', () => ({
RecursiveCharacterTextSplitter: jest.fn().mockImplementation(() => ({
splitDocuments: jest.fn(
async (docs: Array<{ [key: string]: unknown }>): Promise<Array<{ [key: string]: unknown }>> =>
docs.map((doc) => ({ ...doc, split: true })),
),
})),
}));
jest.mock('@langchain/community/document_loaders/web/github', () => ({
GithubRepoLoader: jest.fn().mockImplementation(() => ({
load: jest.fn(async () => [{ pageContent: 'doc1' }, { pageContent: 'doc2' }]),
})),
}));
const mockLogger = { debug: jest.fn() };
describe('DocumentGithubLoader', () => {
let loader: DocumentGithubLoader;
beforeEach(() => {
loader = new DocumentGithubLoader();
jest.clearAllMocks();
});
it('should supply data with recursive char text splitter', async () => {
const context = {
logger: mockLogger,
getNode: jest.fn(() => ({ typeVersion: 1.1 })),
getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
switch (paramName) {
case 'repository':
return 'owner/repo';
case 'branch':
return 'main';
case 'textSplittingMode':
return 'simple';
case 'additionalOptions':
return { recursive: true, ignorePaths: 'docs,tests' };
default:
return;
}
}),
getCredentials: jest.fn().mockResolvedValue({
accessToken: 'token',
server: 'https://api.github.com',
}),
addInputData: jest.fn(() => ({ index: 0 })),
addOutputData: jest.fn(),
} as unknown as ISupplyDataFunctions;
await loader.supplyData.call(context, 0);
expect(RecursiveCharacterTextSplitter).toHaveBeenCalledWith({
chunkSize: 1000,
chunkOverlap: 200,
});
});
it('should use custom text splitter when textSplittingMode is custom', async () => {
const customSplitter = { splitDocuments: jest.fn(async (docs) => docs) };
const context = {
logger: mockLogger,
getNode: jest.fn(() => ({ typeVersion: 1.1 })),
getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
switch (paramName) {
case 'repository':
return 'owner/repo';
case 'branch':
return 'main';
case 'textSplittingMode':
return 'custom';
case 'additionalOptions':
return { recursive: true, ignorePaths: 'docs,tests' };
default:
return;
}
}),
getCredentials: jest.fn().mockResolvedValue({
accessToken: 'token',
server: 'https://api.github.com',
}),
getInputConnectionData: jest.fn(async () => customSplitter),
addInputData: jest.fn(() => ({ index: 0 })),
addOutputData: jest.fn(),
} as unknown as ISupplyDataFunctions;
await loader.supplyData.call(context, 0);
expect(context.getInputConnectionData).toHaveBeenCalledWith(
NodeConnectionTypes.AiTextSplitter,
0,
);
expect(customSplitter.splitDocuments).toHaveBeenCalled();
});
});
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts
+102
View File
@@ -0,0 +1,102 @@
import type { TextSplitter } from '@langchain/textsplitters';
import {
NodeConnectionTypes,
type INodeType,
type INodeTypeDescription,
type ISupplyDataFunctions,
type SupplyData,
} from 'n8n-workflow';
import {
logWrapper,
N8nJsonLoader,
getConnectionHintNoticeField,
metadataFilterField,
} from '@n8n/ai-utilities';
export class DocumentJsonInputLoader implements INodeType {
description: INodeTypeDescription = {
// This node is deprecated and will be removed in the future.
// The functionality was merged with the `DocumentBinaryInputLoader` to `DocumentDefaultDataLoader`
hidden: true,
displayName: 'JSON Input Loader',
name: 'documentJsonInputLoader',
icon: 'file:json.svg',
group: ['transform'],
version: 1,
description: 'Use JSON data from a previous step in the workflow',
defaults: {
name: 'JSON Input Loader',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Document Loaders'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.documentdefaultdataloader/',
},
],
},
},
inputs: [
{
displayName: 'Text Splitter',
maxConnections: 1,
type: NodeConnectionTypes.AiTextSplitter,
},
],
inputNames: ['Text Splitter'],
outputs: [NodeConnectionTypes.AiDocument],
outputNames: ['Document'],
builderHint: {
inputs: {
ai_textSplitter: { required: false },
},
},
properties: [
getConnectionHintNoticeField([NodeConnectionTypes.AiVectorStore]),
{
displayName: 'Pointers',
name: 'pointers',
type: 'string',
default: '',
description: 'Pointers to extract from JSON, e.g. "/text" or "/text, /meta/title"',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
...metadataFilterField,
displayName: 'Metadata',
description:
'Metadata to add to each document. Could be used for filtering during retrieval',
placeholder: 'Add property',
},
],
},
],
};
async supplyData(this: ISupplyDataFunctions): Promise<SupplyData> {
this.logger.debug('Supply Data for JSON Input Loader');
const textSplitter = (await this.getInputConnectionData(
NodeConnectionTypes.AiTextSplitter,
0,
)) as TextSplitter | undefined;
const processor = new N8nJsonLoader(this, undefined, textSplitter);
return {
response: logWrapper(processor, this),
};
}
}
packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/json.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="800" height="800" fill="#7D7D87" viewBox="0 0 58 58"><path d="m50.949 12.187-1.361-1.361-9.504-9.505-.002-.001-.77-.771A1.87 1.87 0 0 0 37.985 0H8.963C7.776 0 6.5.916 6.5 2.926V56c0 .837.841 1.652 1.836 1.909.051.014.1.033.152.043q.235.047.475.048h40.074q.24 0 .475-.048c.052-.01.101-.029.152-.043.995-.257 1.836-1.072 1.836-1.909V13.978c0-.767-.093-1.334-.551-1.791M39.5 3.565 47.935 12H39.5zM8.963 56c-.071 0-.135-.025-.198-.049a.46.46 0 0 1-.265-.414V41h41v14.537a.46.46 0 0 1-.265.414c-.063.024-.127.049-.198.049zM8.5 39V2.926c0-.217.033-.926.463-.926h28.595a1.5 1.5 0 0 0-.058.391V13.78a2.98 2.98 0 0 0-2-.78 1 1 0 1 0 0 2c.552 0 1 .449 1 1v4c0 1.2.542 2.266 1.382 3a3.98 3.98 0 0 0-1.382 3v4c0 .551-.448 1-1 1a1 1 0 1 0 0 2c1.654 0 3-1.346 3-3v-4c0-1.103.897-2 2-2a1 1 0 1 0 0-2c-1.103 0-2-.897-2-2v-4a2.98 2.98 0 0 0-.78-2h11.389c.135 0 .265-.025.391-.058l.001.036V39z"/><path d="M16.354 51.43q-.028.67-.458.95c-.43.28-.672.28-1.155.28q-.287 0-.615-.068c-.328-.068-.429-.098-.629-.157s-.385-.123-.554-.191-.299-.135-.39-.198l-.697 1.107q.274.205.67.369c.396.164.54.207.827.294s.565.15.834.191.504.062.704.062q.602 0 1.169-.116t1.005-.41c.438-.294.524-.456.697-.779s.26-.723.26-1.196V43.72h-1.668zm8.729-2.366q-.471-.342-1.019-.581c-.548-.239-.702-.323-1.012-.492s-.569-.364-.779-.588-.314-.518-.314-.882q0-.22.109-.458c.109-.238.173-.303.301-.431s.273-.234.438-.321.337-.139.52-.157q.492-.04.807-.014c.315.026.378.05.506.096s.226.091.294.137.13.082.185.109q.013-.014.082-.137c.069-.123.101-.185.164-.308l.205-.396a9 9 0 0 0 .191-.39q-.396-.26-1.039-.376c-.643-.116-.853-.116-1.271-.116q-.615 0-1.169.191c-.554.191-.692.313-.971.554s-.499.535-.663.882-.248.744-.248 1.19q0 .738.314 1.23c.314.492.474.613.793.854s.661.451 1.025.629.704.355 1.019.533.576.376.786.595.314.483.314.793q0 .767-.444 1.155c-.444.388-.723.39-1.278.39q-.274 0-.588-.055c-.314-.055-.419-.084-.629-.144s-.412-.123-.608-.191-.357-.139-.485-.212l-.287 1.176q.233.205.554.349c.321.144.439.171.677.226q.356.083.704.116c.348.033.458.034.677.034q.766 0 1.367-.232c.601-.232.738-.362 1.012-.622s.485-.561.636-.902.226-.695.226-1.06q0-.807-.314-1.319c-.314-.512-.474-.627-.788-.855m9.789-3.992q-.567-.643-1.326-.978c-.759-.335-1.06-.335-1.661-.335s-1.155.111-1.661.335-.948.549-1.326.978-.675.964-.889 1.606-.321 1.388-.321 2.235.107 1.595.321 2.242.511 1.185.889 1.613.82.752 1.326.971 1.06.328 1.661.328 1.155-.109 1.661-.328.948-.542 1.326-.971.675-.966.889-1.613.321-1.395.321-2.242-.107-1.593-.321-2.235-.511-1.177-.889-1.606m-.677 5.626q-.205.731-.567 1.183c-.362.452-.515.518-.82.649s-.627.198-.964.198q-.492 0-.937-.212c-.445-.212-.561-.364-.793-.67s-.415-.699-.547-1.183-.203-1.066-.212-1.75q.014-1.053.219-1.777.206-.724.567-1.183c.361-.459.515-.521.82-.649s.627-.191.964-.191q.493 0 .937.205c.444.205.561.36.793.67s.415.704.547 1.183.203 1.06.212 1.743q-.013 1.053-.219 1.784m9.817.171-3.951-6.945h-1.668V54h1.668v-6.945L44.012 54h1.668V43.924h-1.668zM20.5 20v-4c0-.551.448-1 1-1a1 1 0 1 0 0-2c-1.654 0-3 1.346-3 3v4c0 1.103-.897 2-2 2a1 1 0 1 0 0 2c1.103 0 2 .897 2 2v4c0 1.654 1.346 3 3 3a1 1 0 1 0 0-2c-.552 0-1-.449-1-1v-4c0-1.2-.542-2.266-1.382-3a3.98 3.98 0 0 0 1.382-3"/><circle cx="28.5" cy="19.5" r="1.5"/><path d="M28.5 25a1 1 0 0 0-1 1v3a1 1 0 1 0 2 0v-3a1 1 0 0 0-1-1"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 3.3 KiB