first commit

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/DocumentGithubLoader.node.ts

@@ -0,0 +1,196 @@
+import { GithubRepoLoader } from '@langchain/community/document_loaders/web/github';
+import type { TextSplitter } from '@langchain/textsplitters';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { logWrapper, getConnectionHintNoticeField } from '@n8n/ai-utilities';
+
+import {
+	NodeConnectionTypes,
+	type INodeType,
+	type INodeTypeDescription,
+	type ISupplyDataFunctions,
+	type SupplyData,
+	type IDataObject,
+	type INodeInputConfiguration,
+} from 'n8n-workflow';
+
+/* istanbul ignore next */
+function getInputs(parameters: IDataObject) {
+	const inputs: INodeInputConfiguration[] = [];
+
+	const textSplittingMode = parameters?.textSplittingMode;
+	// If text splitting mode is 'custom' or does not exist (v1), we need to add an input for the text splitter
+	if (!textSplittingMode || textSplittingMode === 'custom') {
+		inputs.push({
+			displayName: 'Text Splitter',
+			maxConnections: 1,
+			type: 'ai_textSplitter',
+			required: true,
+		});
+	}
+
+	return inputs;
+}
+
+export class DocumentGithubLoader implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'GitHub Document Loader',
+		name: 'documentGithubLoader',
+		icon: 'file:github.svg',
+		group: ['transform'],
+		version: [1, 1.1],
+		defaultVersion: 1.1,
+		description: 'Use GitHub data as input to this chain',
+		hidden: true,
+		defaults: {
+			name: 'GitHub Document Loader',
+		},
+		codex: {
+			categories: ['AI'],
+			subcategories: {
+				AI: ['Document Loaders'],
+			},
+			resources: {
+				primaryDocumentation: [
+					{
+						url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.documentgithubloader/',
+					},
+				],
+			},
+		},
+		credentials: [
+			{
+				name: 'githubApi',
+				required: true,
+			},
+		],
+
+		inputs: `={{ ((parameter) => { ${getInputs.toString()}; return getInputs(parameter) })($parameter) }}`,
+		inputNames: ['Text Splitter'],
+
+		outputs: [NodeConnectionTypes.AiDocument],
+		outputNames: ['Document'],
+		builderHint: {
+			inputs: {
+				ai_textSplitter: {
+					required: true,
+					displayOptions: { show: { textSplittingMode: ['custom'] } },
+				},
+			},
+		},
+		properties: [
+			getConnectionHintNoticeField([NodeConnectionTypes.AiVectorStore]),
+			{
+				displayName: 'Repository Link',
+				name: 'repository',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Branch',
+				name: 'branch',
+				type: 'string',
+				default: 'main',
+			},
+			{
+				displayName: 'Text Splitting',
+				name: 'textSplittingMode',
+				type: 'options',
+				default: 'simple',
+				required: true,
+				noDataExpression: true,
+				displayOptions: {
+					show: {
+						'@version': [1.1],
+					},
+				},
+				options: [
+					{
+						name: 'Simple',
+						value: 'simple',
+						description: 'Splits every 1000 characters with a 200 character overlap',
+					},
+					{
+						name: 'Custom',
+						value: 'custom',
+						description: 'Connect a custom text-splitting sub-node',
+					},
+				],
+			},
+			{
+				displayName: 'Options',
+				name: 'additionalOptions',
+				type: 'collection',
+				placeholder: 'Add Option',
+				default: {},
+
+				options: [
+					{
+						displayName: 'Recursive',
+						name: 'recursive',
+						type: 'boolean',
+						default: false,
+					},
+					{
+						displayName: 'Ignore Paths',
+						name: 'ignorePaths',
+						type: 'string',
+						description: 'Comma-separated list of paths to ignore, e.g. "docs, src/tests',
+						default: '',
+					},
+				],
+			},
+		],
+	};
+
+	async supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {
+		this.logger.debug('Supplying data for Github Document Loader');
+		const node = this.getNode();
+
+		const repository = this.getNodeParameter('repository', itemIndex) as string;
+		const branch = this.getNodeParameter('branch', itemIndex) as string;
+		const credentials = await this.getCredentials('githubApi');
+		const { ignorePaths, recursive } = this.getNodeParameter('additionalOptions', 0) as {
+			recursive: boolean;
+			ignorePaths: string;
+		};
+		let textSplitter: TextSplitter | undefined;
+
+		if (node.typeVersion === 1.1) {
+			const textSplittingMode = this.getNodeParameter('textSplittingMode', itemIndex, 'simple') as
+				| 'simple'
+				| 'custom';
+
+			if (textSplittingMode === 'simple') {
+				textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
+			} else if (textSplittingMode === 'custom') {
+				textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
+					| TextSplitter
+					| undefined;
+			}
+		} else {
+			textSplitter = (await this.getInputConnectionData(NodeConnectionTypes.AiTextSplitter, 0)) as
+				| TextSplitter
+				| undefined;
+		}
+
+		const { index } = this.addInputData(NodeConnectionTypes.AiDocument, [
+			[{ json: { repository, branch, ignorePaths, recursive } }],
+		]);
+		const docs = new GithubRepoLoader(repository, {
+			branch,
+			ignorePaths: (ignorePaths ?? '').split(',').map((p) => p.trim()),
+			recursive,
+			accessToken: (credentials.accessToken as string) || '',
+			apiUrl: credentials.server as string,
+		});
+
+		const loadedDocs = textSplitter
+			? await textSplitter.splitDocuments(await docs.load())
+			: await docs.load();
+
+		this.addOutputData(NodeConnectionTypes.AiDocument, index, [[{ json: { loadedDocs } }]]);
+		return {
+			response: logWrapper(loadedDocs, this),
+		};
+	}
+}

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/github.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" fill="#fff" fill-rule="evenodd" stroke="#000" stroke-linecap="round" stroke-linejoin="round" viewBox="0 0 148.744 150.744"><use xlink:href="#a" x=".872" y=".872"/><symbol id="a" overflow="visible"><path fill="#7D7D87" stroke="none" d="M73.256 0C32.801 0 0 34.029 0 76.001c0 33.586 20.988 62.069 50.1 72.115 3.663.698 4.999-1.652 4.999-3.656l-.105-14.149c-20.372 4.593-24.677-8.961-24.677-8.961-3.335-8.777-8.133-11.114-8.133-11.114-6.658-4.713.523-4.622.523-4.622 7.355.529 11.227 7.831 11.227 7.831 6.537 11.616 17.151 8.257 21.319 6.309.666-4.901 2.564-8.257 4.65-10.151-16.261-1.919-33.366-8.442-33.366-37.565 0-8.302 2.857-15.075 7.535-20.396-.747-1.929-3.269-9.663.724-20.123 0 0 6.143-2.041 20.145 7.793 5.84-1.692 12.105-2.529 18.314-2.555 6.223.028 12.492.872 18.34 2.564 13.978-9.844 20.128-7.793 20.128-7.793 4.006 10.47 1.483 18.192.733 20.114 4.695 5.32 7.53 12.093 7.53 20.396 0 29.198-17.133 35.627-33.453 37.509 2.639 2.355 4.971 6.977 4.971 14.065l-.098 20.855c0 2.023 1.333 4.388 5.044 3.663 29.091-10.078 50.062-38.561 50.062-72.129C146.512 34.029 113.71 0 73.256 0"/></symbol></svg>

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentGithubLoader/test/DocumentGithubLoader.node.test.ts

@@ -0,0 +1,99 @@
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import type { ISupplyDataFunctions } from 'n8n-workflow';
+import { NodeConnectionTypes } from 'n8n-workflow';
+
+import { DocumentGithubLoader } from '../DocumentGithubLoader.node';
+
+jest.mock('@langchain/textsplitters', () => ({
+	RecursiveCharacterTextSplitter: jest.fn().mockImplementation(() => ({
+		splitDocuments: jest.fn(
+			async (docs: Array<{ [key: string]: unknown }>): Promise<Array<{ [key: string]: unknown }>> =>
+				docs.map((doc) => ({ ...doc, split: true })),
+		),
+	})),
+}));
+jest.mock('@langchain/community/document_loaders/web/github', () => ({
+	GithubRepoLoader: jest.fn().mockImplementation(() => ({
+		load: jest.fn(async () => [{ pageContent: 'doc1' }, { pageContent: 'doc2' }]),
+	})),
+}));
+
+const mockLogger = { debug: jest.fn() };
+
+describe('DocumentGithubLoader', () => {
+	let loader: DocumentGithubLoader;
+
+	beforeEach(() => {
+		loader = new DocumentGithubLoader();
+		jest.clearAllMocks();
+	});
+
+	it('should supply data with recursive char text splitter', async () => {
+		const context = {
+			logger: mockLogger,
+			getNode: jest.fn(() => ({ typeVersion: 1.1 })),
+			getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
+				switch (paramName) {
+					case 'repository':
+						return 'owner/repo';
+					case 'branch':
+						return 'main';
+					case 'textSplittingMode':
+						return 'simple';
+					case 'additionalOptions':
+						return { recursive: true, ignorePaths: 'docs,tests' };
+					default:
+						return;
+				}
+			}),
+			getCredentials: jest.fn().mockResolvedValue({
+				accessToken: 'token',
+				server: 'https://api.github.com',
+			}),
+			addInputData: jest.fn(() => ({ index: 0 })),
+			addOutputData: jest.fn(),
+		} as unknown as ISupplyDataFunctions;
+		await loader.supplyData.call(context, 0);
+
+		expect(RecursiveCharacterTextSplitter).toHaveBeenCalledWith({
+			chunkSize: 1000,
+			chunkOverlap: 200,
+		});
+	});
+
+	it('should use custom text splitter when textSplittingMode is custom', async () => {
+		const customSplitter = { splitDocuments: jest.fn(async (docs) => docs) };
+		const context = {
+			logger: mockLogger,
+			getNode: jest.fn(() => ({ typeVersion: 1.1 })),
+			getNodeParameter: jest.fn().mockImplementation((paramName, _itemIndex) => {
+				switch (paramName) {
+					case 'repository':
+						return 'owner/repo';
+					case 'branch':
+						return 'main';
+					case 'textSplittingMode':
+						return 'custom';
+					case 'additionalOptions':
+						return { recursive: true, ignorePaths: 'docs,tests' };
+					default:
+						return;
+				}
+			}),
+			getCredentials: jest.fn().mockResolvedValue({
+				accessToken: 'token',
+				server: 'https://api.github.com',
+			}),
+			getInputConnectionData: jest.fn(async () => customSplitter),
+			addInputData: jest.fn(() => ({ index: 0 })),
+			addOutputData: jest.fn(),
+		} as unknown as ISupplyDataFunctions;
+		await loader.supplyData.call(context, 0);
+
+		expect(context.getInputConnectionData).toHaveBeenCalledWith(
+			NodeConnectionTypes.AiTextSplitter,
+			0,
+		);
+		expect(customSplitter.splitDocuments).toHaveBeenCalled();
+	});
+});