first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
import { BaseOutputParser, OutputParserException } from '@langchain/core/output_parsers';
|
||||
|
||||
export class N8nItemListOutputParser extends BaseOutputParser<string[]> {
|
||||
lc_namespace = ['n8n-nodes-langchain', 'output_parsers', 'list_items'];
|
||||
|
||||
private numberOfItems: number | undefined;
|
||||
|
||||
private separator: string;
|
||||
|
||||
constructor(options: { numberOfItems?: number; separator?: string }) {
|
||||
super();
|
||||
|
||||
const { numberOfItems = 3, separator = '\n' } = options;
|
||||
|
||||
if (numberOfItems && numberOfItems > 0) {
|
||||
this.numberOfItems = numberOfItems;
|
||||
}
|
||||
|
||||
this.separator = separator;
|
||||
|
||||
if (this.separator === '\\n') {
|
||||
this.separator = '\n';
|
||||
}
|
||||
}
|
||||
|
||||
async parse(text: string): Promise<string[]> {
|
||||
const response = text
|
||||
.split(this.separator)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item);
|
||||
|
||||
if (this.numberOfItems && response.length < this.numberOfItems) {
|
||||
// Only error if to few items got returned, if there are to many we can autofix it
|
||||
throw new OutputParserException(
|
||||
`Wrong number of items returned. Expected ${this.numberOfItems} items but got ${response.length} items instead.`,
|
||||
);
|
||||
}
|
||||
|
||||
return response.slice(0, this.numberOfItems);
|
||||
}
|
||||
|
||||
getFormatInstructions(): string {
|
||||
const instructions = `Your response should be a list of ${
|
||||
this.numberOfItems ? this.numberOfItems + ' ' : ''
|
||||
}items separated by`;
|
||||
|
||||
const numberOfExamples = this.numberOfItems ?? 3; // Default number of examples in case numberOfItems is not set
|
||||
|
||||
const examples: string[] = [];
|
||||
for (let i = 1; i <= numberOfExamples; i++) {
|
||||
examples.push(`item${i}`);
|
||||
}
|
||||
|
||||
return `${instructions} "${this.separator}" (for example: "${examples.join(this.separator)}")`;
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { Callbacks } from '@langchain/core/callbacks/manager';
|
||||
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
|
||||
import type { AIMessage } from '@langchain/core/messages';
|
||||
import { BaseOutputParser, OutputParserException } from '@langchain/core/output_parsers';
|
||||
import type { PromptTemplate } from '@langchain/core/prompts';
|
||||
import type { ISupplyDataFunctions } from 'n8n-workflow';
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
|
||||
import type { N8nStructuredOutputParser } from './N8nStructuredOutputParser';
|
||||
import { logAiEvent } from '@n8n/ai-utilities';
|
||||
|
||||
export class N8nOutputFixingParser extends BaseOutputParser {
|
||||
lc_namespace = ['langchain', 'output_parsers', 'fix'];
|
||||
|
||||
constructor(
|
||||
private context: ISupplyDataFunctions,
|
||||
private model: BaseLanguageModel,
|
||||
private outputParser: N8nStructuredOutputParser,
|
||||
private fixPromptTemplate: PromptTemplate,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
getRetryChain() {
|
||||
return this.fixPromptTemplate.pipe(this.model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to parse the completion string using the output parser.
|
||||
* If the initial parse fails, it tries to fix the output using a retry chain.
|
||||
* @param completion The string to be parsed
|
||||
* @returns The parsed response
|
||||
* @throws Error if both parsing attempts fail
|
||||
*/
|
||||
async parse(completion: string, callbacks?: Callbacks) {
|
||||
const { index } = this.context.addInputData(NodeConnectionTypes.AiOutputParser, [
|
||||
[{ json: { action: 'parse', text: completion } }],
|
||||
]);
|
||||
|
||||
try {
|
||||
// First attempt to parse the completion
|
||||
const response = await this.outputParser.parse(completion, callbacks, (e) => {
|
||||
if (e instanceof OutputParserException) {
|
||||
return e;
|
||||
}
|
||||
return new OutputParserException(e.message, completion);
|
||||
});
|
||||
logAiEvent(this.context, 'ai-output-parsed', { text: completion, response });
|
||||
|
||||
this.context.addOutputData(NodeConnectionTypes.AiOutputParser, index, [
|
||||
[{ json: { action: 'parse', response } }],
|
||||
]);
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
if (!(error instanceof OutputParserException)) {
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
// Second attempt: use retry chain to fix the output
|
||||
const result = (await this.getRetryChain().invoke({
|
||||
completion,
|
||||
error: error.message,
|
||||
instructions: this.getFormatInstructions(),
|
||||
})) as AIMessage;
|
||||
|
||||
const resultText = result.content.toString();
|
||||
const parsed = await this.outputParser.parse(resultText, callbacks);
|
||||
|
||||
// Add the successfully parsed output to the context
|
||||
this.context.addOutputData(NodeConnectionTypes.AiOutputParser, index, [
|
||||
[{ json: { action: 'parse', response: parsed } }],
|
||||
]);
|
||||
|
||||
return parsed;
|
||||
} catch (autoParseError) {
|
||||
// If both attempts fail, add the error to the output and throw
|
||||
this.context.addOutputData(NodeConnectionTypes.AiOutputParser, index, autoParseError);
|
||||
throw autoParseError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to get the format instructions for the parser.
|
||||
* @returns The format instructions for the parser.
|
||||
*/
|
||||
getFormatInstructions() {
|
||||
return this.outputParser.getFormatInstructions();
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return this.outputParser.schema;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type { IExecuteFunctions } from 'n8n-workflow';
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
|
||||
import { getOptionalOutputParser } from './N8nOutputParser';
|
||||
import type { N8nStructuredOutputParser } from './N8nStructuredOutputParser';
|
||||
|
||||
describe('getOptionalOutputParser', () => {
|
||||
let mockContext: jest.Mocked<IExecuteFunctions>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockContext = mock<IExecuteFunctions>();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return undefined when hasOutputParser is false', async () => {
|
||||
mockContext.getNodeParameter.mockReturnValue(false);
|
||||
|
||||
const result = await getOptionalOutputParser(mockContext);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(mockContext.getNodeParameter).toHaveBeenCalledWith('hasOutputParser', 0, true);
|
||||
expect(mockContext.getInputConnectionData).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return output parser when hasOutputParser is true with default index', async () => {
|
||||
const mockParser = mock<N8nStructuredOutputParser>();
|
||||
mockContext.getNodeParameter.mockReturnValue(true);
|
||||
mockContext.getInputConnectionData.mockResolvedValue(mockParser);
|
||||
|
||||
const result = await getOptionalOutputParser(mockContext);
|
||||
|
||||
expect(result).toBe(mockParser);
|
||||
expect(mockContext.getNodeParameter).toHaveBeenCalledWith('hasOutputParser', 0, true);
|
||||
expect(mockContext.getInputConnectionData).toHaveBeenCalledWith(
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('should use provided index when fetching output parser', async () => {
|
||||
const mockParser = mock<N8nStructuredOutputParser>();
|
||||
mockContext.getNodeParameter.mockReturnValue(true);
|
||||
mockContext.getInputConnectionData.mockResolvedValue(mockParser);
|
||||
|
||||
const result = await getOptionalOutputParser(mockContext, 2);
|
||||
|
||||
expect(result).toBe(mockParser);
|
||||
expect(mockContext.getNodeParameter).toHaveBeenCalledWith('hasOutputParser', 0, true);
|
||||
expect(mockContext.getInputConnectionData).toHaveBeenCalledWith(
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle different index values correctly', async () => {
|
||||
const mockParser1 = mock<N8nStructuredOutputParser>();
|
||||
const mockParser2 = mock<N8nStructuredOutputParser>();
|
||||
const mockParser3 = mock<N8nStructuredOutputParser>();
|
||||
|
||||
mockContext.getNodeParameter.mockReturnValue(true);
|
||||
mockContext.getInputConnectionData
|
||||
.mockResolvedValueOnce(mockParser1)
|
||||
.mockResolvedValueOnce(mockParser2)
|
||||
.mockResolvedValueOnce(mockParser3);
|
||||
|
||||
const result1 = await getOptionalOutputParser(mockContext, 0);
|
||||
const result2 = await getOptionalOutputParser(mockContext, 1);
|
||||
const result3 = await getOptionalOutputParser(mockContext, 5);
|
||||
|
||||
expect(result1).toBe(mockParser1);
|
||||
expect(result2).toBe(mockParser2);
|
||||
expect(result3).toBe(mockParser3);
|
||||
|
||||
expect(mockContext.getInputConnectionData).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
0,
|
||||
);
|
||||
expect(mockContext.getInputConnectionData).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
1,
|
||||
);
|
||||
expect(mockContext.getInputConnectionData).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
5,
|
||||
);
|
||||
});
|
||||
|
||||
it('should always check hasOutputParser at index 0', async () => {
|
||||
mockContext.getNodeParameter.mockReturnValue(false);
|
||||
|
||||
await getOptionalOutputParser(mockContext, 3);
|
||||
|
||||
// Even when called with index 3, hasOutputParser is checked at index 0
|
||||
expect(mockContext.getNodeParameter).toHaveBeenCalledWith('hasOutputParser', 0, true);
|
||||
expect(mockContext.getInputConnectionData).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { IExecuteFunctions, ISupplyDataFunctions, IWebhookFunctions } from 'n8n-workflow';
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
|
||||
import { N8nItemListOutputParser } from './N8nItemListOutputParser';
|
||||
import { N8nOutputFixingParser } from './N8nOutputFixingParser';
|
||||
import { N8nStructuredOutputParser } from './N8nStructuredOutputParser';
|
||||
|
||||
export type N8nOutputParser =
|
||||
| N8nOutputFixingParser
|
||||
| N8nStructuredOutputParser
|
||||
| N8nItemListOutputParser;
|
||||
|
||||
export { N8nOutputFixingParser, N8nItemListOutputParser, N8nStructuredOutputParser };
|
||||
|
||||
export async function getOptionalOutputParser(
|
||||
ctx: IExecuteFunctions | ISupplyDataFunctions | IWebhookFunctions,
|
||||
index: number = 0,
|
||||
): Promise<N8nOutputParser | undefined> {
|
||||
let outputParser: N8nOutputParser | undefined;
|
||||
|
||||
if (ctx.getNodeParameter('hasOutputParser', 0, true) === true) {
|
||||
outputParser = (await ctx.getInputConnectionData(
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
index,
|
||||
)) as N8nOutputParser;
|
||||
}
|
||||
|
||||
return outputParser;
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
import { mock } from 'jest-mock-extended';
|
||||
import type { INode, ISupplyDataFunctions } from 'n8n-workflow';
|
||||
import { NodeConnectionTypes } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { N8nStructuredOutputParser } from './N8nStructuredOutputParser';
|
||||
|
||||
describe('N8nStructuredOutputParser', () => {
|
||||
let mockContext: jest.Mocked<ISupplyDataFunctions>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockContext = mock<ISupplyDataFunctions>();
|
||||
mockContext.addInputData.mockReturnValue({ index: 0 });
|
||||
mockContext.addOutputData.mockReturnValue(undefined);
|
||||
mockContext.getNode.mockReturnValue({
|
||||
name: 'Test Node',
|
||||
type: 'n8n-nodes-langchain.outputParserStructured',
|
||||
typeVersion: 1.3,
|
||||
position: [0, 0],
|
||||
} as INode);
|
||||
});
|
||||
|
||||
// Bug AI-1852
|
||||
describe('Backticks in JSON string values', () => {
|
||||
it('should parse JSON containing markdown code blocks in string values', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
status: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
// This is the exact problematic output from the bug report
|
||||
// Valid JSON wrapped in code fence, but contains backticks INSIDE the message field
|
||||
const problematicOutput = `\`\`\`json
|
||||
{
|
||||
"output": {
|
||||
"message": "## Example\\n\`\`\`bash\\n--set globals.enable=false\\n\`\`\`\\n",
|
||||
"status": "completed"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(problematicOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
message: '## Example\n```bash\n--set globals.enable=false\n```\n',
|
||||
status: 'completed',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse JSON containing multiple code blocks in string values', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
content: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const multipleCodeBlocksOutput = `\`\`\`json
|
||||
{
|
||||
"output": {
|
||||
"content": "First example:\\n\`\`\`javascript\\nconst x = 1;\\n\`\`\`\\n\\nSecond example:\\n\`\`\`python\\nprint('hello')\\n\`\`\`"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(multipleCodeBlocksOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
content:
|
||||
"First example:\n```javascript\nconst x = 1;\n```\n\nSecond example:\n```python\nprint('hello')\n```",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse nested markdown in complex JSON structures', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
steps: z.array(
|
||||
z.object({
|
||||
title: z.string(),
|
||||
description: z.string(),
|
||||
code: z.string(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const complexOutput = `\`\`\`json
|
||||
{
|
||||
"output": {
|
||||
"steps": [
|
||||
{
|
||||
"title": "Step 1",
|
||||
"description": "Run this command:\\n\`\`\`bash\\nnpm install\\n\`\`\`",
|
||||
"code": "npm install"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(complexOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
steps: [
|
||||
{
|
||||
title: 'Step 1',
|
||||
description: 'Run this command:\n```bash\nnpm install\n```',
|
||||
code: 'npm install',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Valid JSON parsing', () => {
|
||||
it('should parse valid JSON without code fence', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
status: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = `{
|
||||
"output": {
|
||||
"message": "Simple message",
|
||||
"status": "completed"
|
||||
}
|
||||
}`;
|
||||
|
||||
const result = await parser.parse(validOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
message: 'Simple message',
|
||||
status: 'completed',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse valid JSON wrapped in code fence without internal backticks', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
status: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = `\`\`\`json
|
||||
{
|
||||
"output": {
|
||||
"message": "Simple message",
|
||||
"status": "completed"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(validOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
message: 'Simple message',
|
||||
status: 'completed',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should parse JSON with escaped quotes and newlines', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = `{
|
||||
"output": {
|
||||
"message": "Line 1\\nLine 2\\n\\"quoted\\""
|
||||
}
|
||||
}`;
|
||||
|
||||
const result = await parser.parse(validOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
message: 'Line 1\nLine 2\n"quoted"',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle code fence with json language marker', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
data: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = `\`\`\`json
|
||||
{
|
||||
"output": {
|
||||
"data": "test"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(validOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
data: 'test',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle code fence without language marker', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
data: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = `\`\`\`
|
||||
{
|
||||
"output": {
|
||||
"data": "test"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
const result = await parser.parse(validOutput);
|
||||
|
||||
expect(result).toEqual({
|
||||
output: {
|
||||
data: 'test',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error handling', () => {
|
||||
it('should handle invalid JSON gracefully', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const invalidOutput = 'not valid json';
|
||||
|
||||
await expect(parser.parse(invalidOutput)).rejects.toThrow(
|
||||
"Model output doesn't fit required format",
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle empty output', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const emptyOutput = '{}';
|
||||
|
||||
await expect(parser.parse(emptyOutput)).rejects.toThrow(
|
||||
"Model output doesn't fit required format",
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle schema mismatch', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
requiredField: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const mismatchOutput = `{
|
||||
"output": {
|
||||
"message": "Test"
|
||||
}
|
||||
}`;
|
||||
|
||||
await expect(parser.parse(mismatchOutput)).rejects.toThrow(
|
||||
"Model output doesn't fit required format",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Context integration', () => {
|
||||
it('should call addInputData with correct parameters', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = '{"output": {"message": "test"}}';
|
||||
|
||||
await parser.parse(validOutput);
|
||||
|
||||
expect(mockContext.addInputData).toHaveBeenCalledWith(NodeConnectionTypes.AiOutputParser, [
|
||||
[{ json: { action: 'parse', text: validOutput } }],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should call addOutputData with parsed result', async () => {
|
||||
const schema = z.object({
|
||||
output: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const parser = new N8nStructuredOutputParser(mockContext, schema);
|
||||
|
||||
const validOutput = '{"output": {"message": "test"}}';
|
||||
|
||||
await parser.parse(validOutput);
|
||||
|
||||
expect(mockContext.addOutputData).toHaveBeenCalledWith(
|
||||
NodeConnectionTypes.AiOutputParser,
|
||||
0,
|
||||
[[{ json: { action: 'parse', response: { output: { message: 'test' } } } }]],
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
import type { Callbacks } from '@langchain/core/callbacks/manager';
|
||||
import { StructuredOutputParser } from '@langchain/classic/output_parsers';
|
||||
import get from 'lodash/get';
|
||||
import type { ISupplyDataFunctions } from 'n8n-workflow';
|
||||
import { NodeConnectionTypes, NodeOperationError } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { logAiEvent } from '@n8n/ai-utilities';
|
||||
import { unwrapNestedOutput } from '../helpers';
|
||||
|
||||
const STRUCTURED_OUTPUT_KEY = '__structured__output';
|
||||
const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object';
|
||||
const STRUCTURED_OUTPUT_ARRAY_KEY = '__structured__output__array';
|
||||
|
||||
export class N8nStructuredOutputParser extends StructuredOutputParser<
|
||||
z.ZodType<object, z.ZodTypeDef, object>
|
||||
> {
|
||||
constructor(
|
||||
private context: ISupplyDataFunctions,
|
||||
zodSchema: z.ZodSchema<object>,
|
||||
) {
|
||||
super(zodSchema);
|
||||
}
|
||||
|
||||
lc_namespace = ['langchain', 'output_parsers', 'structured'];
|
||||
|
||||
async parse(
|
||||
text: string,
|
||||
_callbacks?: Callbacks,
|
||||
errorMapper?: (error: Error) => Error,
|
||||
): Promise<object> {
|
||||
const { index } = this.context.addInputData(NodeConnectionTypes.AiOutputParser, [
|
||||
[{ json: { action: 'parse', text } }],
|
||||
]);
|
||||
|
||||
try {
|
||||
// Extract JSON from markdown code fence if present
|
||||
// Use line-based approach to avoid matching backticks inside JSON content
|
||||
let jsonString = text.trim();
|
||||
|
||||
// Look for markdown code fence by finding lines that start with ```
|
||||
const lines = jsonString.split('\n');
|
||||
let fenceStartIndex = -1;
|
||||
let fenceEndIndex = -1;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const trimmedLine = lines[i].trim();
|
||||
// Opening fence: line starting with ``` optionally followed by language identifier
|
||||
if (fenceStartIndex === -1 && trimmedLine.match(/^```(?:json)?$/)) {
|
||||
fenceStartIndex = i;
|
||||
} else if (fenceStartIndex !== -1 && trimmedLine === '```') {
|
||||
// Closing fence: line with just ```
|
||||
fenceEndIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If we found both opening and closing fences, extract the content between them
|
||||
if (fenceStartIndex !== -1 && fenceEndIndex !== -1) {
|
||||
jsonString = lines.slice(fenceStartIndex + 1, fenceEndIndex).join('\n');
|
||||
}
|
||||
|
||||
const json = JSON.parse(jsonString.trim());
|
||||
const parsed = await this.schema.parseAsync(json);
|
||||
|
||||
let result = (get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_OBJECT_KEY]) ??
|
||||
get(parsed, [STRUCTURED_OUTPUT_KEY, STRUCTURED_OUTPUT_ARRAY_KEY]) ??
|
||||
get(parsed, STRUCTURED_OUTPUT_KEY) ??
|
||||
parsed) as Record<string, unknown>;
|
||||
|
||||
// Unwrap any doubly-nested output structures (e.g., {output: {output: {...}}})
|
||||
result = unwrapNestedOutput(result);
|
||||
|
||||
logAiEvent(this.context, 'ai-output-parsed', { text, response: result });
|
||||
|
||||
this.context.addOutputData(NodeConnectionTypes.AiOutputParser, index, [
|
||||
[{ json: { action: 'parse', response: result } }],
|
||||
]);
|
||||
|
||||
return result;
|
||||
} catch (e) {
|
||||
const nodeError = new NodeOperationError(
|
||||
this.context.getNode(),
|
||||
"Model output doesn't fit required format",
|
||||
{
|
||||
description:
|
||||
"To continue the execution when this happens, change the 'On Error' parameter in the root node's settings",
|
||||
},
|
||||
);
|
||||
|
||||
// Add additional context to the error
|
||||
if (e instanceof SyntaxError) {
|
||||
nodeError.context.outputParserFailReason = 'Invalid JSON in model output';
|
||||
} else if (
|
||||
(typeof text === 'string' && text.trim() === '{}') ||
|
||||
(e instanceof z.ZodError &&
|
||||
e.issues?.[0] &&
|
||||
e.issues?.[0].code === 'invalid_type' &&
|
||||
e.issues?.[0].path?.[0] === 'output' &&
|
||||
e.issues?.[0].expected === 'object' &&
|
||||
e.issues?.[0].received === 'undefined')
|
||||
) {
|
||||
nodeError.context.outputParserFailReason = 'Model output wrapper is an empty object';
|
||||
} else if (e instanceof z.ZodError) {
|
||||
nodeError.context.outputParserFailReason =
|
||||
'Model output does not match the expected schema';
|
||||
}
|
||||
|
||||
logAiEvent(this.context, 'ai-output-parsed', {
|
||||
text,
|
||||
response: e.message ?? e,
|
||||
});
|
||||
|
||||
this.context.addOutputData(NodeConnectionTypes.AiOutputParser, index, nodeError);
|
||||
if (errorMapper) {
|
||||
throw errorMapper(e);
|
||||
}
|
||||
|
||||
throw nodeError;
|
||||
}
|
||||
}
|
||||
|
||||
static async fromZodJsonSchema(
|
||||
zodSchema: z.ZodSchema<object>,
|
||||
nodeVersion: number,
|
||||
context: ISupplyDataFunctions,
|
||||
): Promise<N8nStructuredOutputParser> {
|
||||
let returnSchema: z.ZodType<object, z.ZodTypeDef, object>;
|
||||
if (nodeVersion === 1) {
|
||||
returnSchema = z.object({
|
||||
[STRUCTURED_OUTPUT_KEY]: z
|
||||
.object({
|
||||
[STRUCTURED_OUTPUT_OBJECT_KEY]: zodSchema.optional(),
|
||||
[STRUCTURED_OUTPUT_ARRAY_KEY]: z.array(zodSchema).optional(),
|
||||
})
|
||||
.describe(
|
||||
`Wrapper around the output data. It can only contain ${STRUCTURED_OUTPUT_OBJECT_KEY} or ${STRUCTURED_OUTPUT_ARRAY_KEY} but never both.`,
|
||||
)
|
||||
.refine(
|
||||
(data) => {
|
||||
// Validate that one and only one of the properties exists
|
||||
return (
|
||||
Boolean(data[STRUCTURED_OUTPUT_OBJECT_KEY]) !==
|
||||
Boolean(data[STRUCTURED_OUTPUT_ARRAY_KEY])
|
||||
);
|
||||
},
|
||||
{
|
||||
message:
|
||||
'One and only one of __structured__output__object and __structured__output__array should be present.',
|
||||
path: [STRUCTURED_OUTPUT_KEY],
|
||||
},
|
||||
),
|
||||
});
|
||||
} else if (nodeVersion < 1.3) {
|
||||
returnSchema = z.object({
|
||||
output: zodSchema.optional(),
|
||||
});
|
||||
} else {
|
||||
returnSchema = z.object({
|
||||
output: zodSchema,
|
||||
});
|
||||
}
|
||||
|
||||
return new N8nStructuredOutputParser(context, returnSchema);
|
||||
}
|
||||
|
||||
getSchema() {
|
||||
return this.schema;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { PromptTemplate } from '@langchain/core/prompts';
|
||||
|
||||
export const NAIVE_FIX_TEMPLATE = `Instructions:
|
||||
--------------
|
||||
{instructions}
|
||||
--------------
|
||||
Completion:
|
||||
--------------
|
||||
{completion}
|
||||
--------------
|
||||
|
||||
Above, the Completion did not satisfy the constraints given in the Instructions.
|
||||
Error:
|
||||
--------------
|
||||
{error}
|
||||
--------------
|
||||
|
||||
Please try again. Please only respond with an answer that satisfies the constraints laid out in the Instructions:`;
|
||||
|
||||
export const NAIVE_FIX_PROMPT = PromptTemplate.fromTemplate(NAIVE_FIX_TEMPLATE);
|
||||
Reference in New Issue
Block a user