first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Security: Sync from Public / sync-from-public (push) Has been cancelled
Test: Benchmark Nightly / build (push) Has been cancelled
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
CI: Python / Checks (push) Has been cancelled
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Test: Benchmark Destroy Env / build (push) Has been cancelled
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
import type { INodeProperties } from 'n8n-workflow';
|
||||
|
||||
import * as executeQuery from './executeQuery.operation';
|
||||
import * as insert from './insert.operation';
|
||||
import { datasetRLC, projectRLC, tableRLC } from '../commonDescriptions/RLC.description';
|
||||
|
||||
export { executeQuery, insert };
|
||||
|
||||
export const description: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Operation',
|
||||
name: 'operation',
|
||||
type: 'options',
|
||||
noDataExpression: true,
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
name: 'Execute Query',
|
||||
value: 'executeQuery',
|
||||
description: 'Execute a SQL query',
|
||||
action: 'Execute a SQL query',
|
||||
},
|
||||
{
|
||||
name: 'Insert',
|
||||
value: 'insert',
|
||||
description: 'Insert rows in a table',
|
||||
action: 'Insert rows in a table',
|
||||
},
|
||||
],
|
||||
default: 'executeQuery',
|
||||
},
|
||||
{
|
||||
...projectRLC,
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['executeQuery', 'insert'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
...datasetRLC,
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['insert'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
...tableRLC,
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['insert'],
|
||||
},
|
||||
},
|
||||
},
|
||||
...executeQuery.description,
|
||||
...insert.description,
|
||||
];
|
||||
+468
@@ -0,0 +1,468 @@
|
||||
import type {
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
} from 'n8n-workflow';
|
||||
import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow';
|
||||
|
||||
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import type { ResponseWithJobReference } from '../../helpers/interfaces';
|
||||
import { prepareOutput } from '../../helpers/utils';
|
||||
import { googleBigQueryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport';
|
||||
|
||||
interface IQueryParameterOptions {
|
||||
namedParameters: Array<{
|
||||
name: string;
|
||||
value: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
const properties: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'SQL Query',
|
||||
name: 'sqlQuery',
|
||||
type: 'string',
|
||||
noDataExpression: true,
|
||||
typeOptions: {
|
||||
editor: 'sqlEditor',
|
||||
},
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/options.useLegacySql': [true],
|
||||
},
|
||||
},
|
||||
default: '',
|
||||
placeholder: 'SELECT * FROM dataset.table LIMIT 100',
|
||||
description:
|
||||
'SQL query to execute, you can find more information <a href="https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax" target="_blank">here</a>. Standard SQL syntax used by default, but you can also use Legacy SQL syntax by using optinon \'Use Legacy SQL\'.',
|
||||
},
|
||||
{
|
||||
displayName: 'SQL Query',
|
||||
name: 'sqlQuery',
|
||||
type: 'string',
|
||||
noDataExpression: true,
|
||||
typeOptions: {
|
||||
editor: 'sqlEditor',
|
||||
},
|
||||
displayOptions: {
|
||||
show: {
|
||||
'/options.useLegacySql': [true],
|
||||
},
|
||||
},
|
||||
default: '',
|
||||
placeholder: 'SELECT * FROM [project:dataset.table] LIMIT 100;',
|
||||
hint: 'Legacy SQL syntax',
|
||||
description:
|
||||
'SQL query to execute, you can find more information about Legacy SQL syntax <a href="https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax" target="_blank">here</a>',
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Default Dataset Name or ID',
|
||||
name: 'defaultDataset',
|
||||
type: 'options',
|
||||
typeOptions: {
|
||||
loadOptionsMethod: 'getDatasets',
|
||||
loadOptionsDependsOn: ['projectId.value'],
|
||||
},
|
||||
default: '',
|
||||
description:
|
||||
'If not set, all table names in the query string must be qualified in the format \'datasetId.tableId\'. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
|
||||
},
|
||||
{
|
||||
displayName: 'Dry Run',
|
||||
name: 'dryRun',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether set to true BigQuery doesn't run the job. Instead, if the query is valid, BigQuery returns statistics about the job such as how many bytes would be processed. If the query is invalid, an error returns.",
|
||||
},
|
||||
{
|
||||
displayName: 'Include Schema in Output',
|
||||
name: 'includeSchema',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether to include the schema in the output. If set to true, the output will contain key '_schema' with the schema of the table.",
|
||||
displayOptions: {
|
||||
hide: {
|
||||
rawOutput: [true],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Location (Region)',
|
||||
name: 'location',
|
||||
type: 'string',
|
||||
default: '',
|
||||
placeholder: 'e.g. europe-west3',
|
||||
description:
|
||||
'Location or the region where data would be stored and processed. Pricing for storage and analysis is also defined by location of data and reservations, more information <a href="https://cloud.google.com/bigquery/docs/locations" target="_blank">here</a>.',
|
||||
},
|
||||
{
|
||||
displayName: 'Maximum Bytes Billed',
|
||||
name: 'maximumBytesBilled',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in <a href="https://developers.google.com/discovery/v1/type-format?utm_source=cloud.google.com&utm_medium=referral" target="_blank">Int64Value</a> format',
|
||||
},
|
||||
{
|
||||
displayName: 'Max Results Per Page',
|
||||
name: 'maxResults',
|
||||
type: 'number',
|
||||
default: 1000,
|
||||
description:
|
||||
'Maximum number of results to return per page of results. This is particularly useful when dealing with large datasets. It will not affect the total number of results returned, e.g. rows in a table. You can use LIMIT in your SQL query to limit the number of rows returned.',
|
||||
},
|
||||
{
|
||||
displayName: 'Timeout',
|
||||
name: 'timeoutMs',
|
||||
type: 'number',
|
||||
default: 10000,
|
||||
hint: 'How long to wait for the query to complete, in milliseconds',
|
||||
description:
|
||||
'Specifies the maximum amount of time, in milliseconds, that the client is willing to wait for the query to complete. Be aware that the call is not guaranteed to wait for the specified timeout; it typically returns after around 200 seconds (200,000 milliseconds), even if the query is not complete.',
|
||||
},
|
||||
{
|
||||
displayName: 'Raw Output',
|
||||
name: 'rawOutput',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
displayOptions: {
|
||||
hide: {
|
||||
dryRun: [true],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Use Legacy SQL',
|
||||
name: 'useLegacySql',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
"Whether to use BigQuery's legacy SQL dialect for this query. If set to false, the query will use BigQuery's standard SQL.",
|
||||
},
|
||||
{
|
||||
displayName: 'Return Integers as Numbers',
|
||||
name: 'returnAsNumbers',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether all integer values will be returned as numbers. If set to false, all integer values will be returned as strings.',
|
||||
},
|
||||
{
|
||||
displayName: 'Query Parameters (Named)',
|
||||
name: 'queryParameters',
|
||||
type: 'fixedCollection',
|
||||
description:
|
||||
'Use <a href="https://cloud.google.com/bigquery/docs/parameterized-queries#using_structs_in_parameterized_queries" target="_blank">parameterized queries</a> to prevent SQL injections. Positional arguments are not supported at the moment. This feature won\'t be available when using legacy SQL.',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
'/options.useLegacySql': [true],
|
||||
},
|
||||
},
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
placeholder: 'Add Parameter',
|
||||
default: {
|
||||
namedParameters: [
|
||||
{
|
||||
name: '',
|
||||
value: '',
|
||||
},
|
||||
],
|
||||
},
|
||||
options: [
|
||||
{
|
||||
name: 'namedParameters',
|
||||
displayName: 'Named Parameter',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Name',
|
||||
name: 'name',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Name of the parameter',
|
||||
},
|
||||
{
|
||||
displayName: 'Value',
|
||||
name: 'value',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The substitute value. It must be a string. Arrays, dates and struct types mentioned in <a href="https://cloud.google.com/bigquery/docs/parameterized-queries#using_structs_in_parameterized_queries" target="_blank">the official documentation</a> are not yet supported.',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['executeQuery'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
||||
const abortSignal = this.getExecutionCancelSignal();
|
||||
const items = this.getInputData();
|
||||
const length = items.length;
|
||||
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
|
||||
let jobs = [];
|
||||
let maxResults = 1000;
|
||||
let timeoutMs = 10000;
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
try {
|
||||
let sqlQuery = this.getNodeParameter('sqlQuery', i) as string;
|
||||
|
||||
const options = this.getNodeParameter('options', i) as {
|
||||
defaultDataset?: string;
|
||||
dryRun?: boolean;
|
||||
includeSchema?: boolean;
|
||||
location?: string;
|
||||
maximumBytesBilled?: string;
|
||||
maxResults?: number;
|
||||
timeoutMs?: number;
|
||||
rawOutput?: boolean;
|
||||
useLegacySql?: boolean;
|
||||
returnAsNumbers?: boolean;
|
||||
queryParameters?: IQueryParameterOptions;
|
||||
};
|
||||
|
||||
const projectId = this.getNodeParameter('projectId', i, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
|
||||
for (const resolvable of getResolvables(sqlQuery)) {
|
||||
sqlQuery = sqlQuery.replace(resolvable, this.evaluateExpression(resolvable, i) as string);
|
||||
}
|
||||
|
||||
let rawOutput = false;
|
||||
let includeSchema = false;
|
||||
|
||||
if (options.rawOutput !== undefined) {
|
||||
rawOutput = options.rawOutput;
|
||||
delete options.rawOutput;
|
||||
}
|
||||
|
||||
if (options.includeSchema !== undefined) {
|
||||
includeSchema = options.includeSchema;
|
||||
delete options.includeSchema;
|
||||
}
|
||||
|
||||
if (options.maxResults) {
|
||||
maxResults = options.maxResults;
|
||||
delete options.maxResults;
|
||||
}
|
||||
|
||||
if (options.timeoutMs) {
|
||||
timeoutMs = options.timeoutMs;
|
||||
delete options.timeoutMs;
|
||||
}
|
||||
|
||||
const body: IDataObject = { ...options };
|
||||
|
||||
body.query = sqlQuery;
|
||||
|
||||
if (body.defaultDataset) {
|
||||
body.defaultDataset = {
|
||||
datasetId: options.defaultDataset,
|
||||
projectId,
|
||||
};
|
||||
}
|
||||
|
||||
if (body.useLegacySql === undefined) {
|
||||
body.useLegacySql = false;
|
||||
}
|
||||
|
||||
if (typeof body.queryParameters === 'object') {
|
||||
const { namedParameters } = body.queryParameters as IQueryParameterOptions;
|
||||
|
||||
body.parameterMode = 'NAMED';
|
||||
|
||||
body.queryParameters = namedParameters.map(({ name, value }) => {
|
||||
// BigQuery type descriptors are very involved, and it would be hard to support all possible
|
||||
// options, that's why the only supported type here is "STRING".
|
||||
//
|
||||
// If we switch this node to the official JS SDK from Google, we should be able to use `getTypeDescriptorFromValue`
|
||||
// at runtime, which would infer BQ type descriptors of any valid JS value automatically:
|
||||
//
|
||||
// https://github.com/googleapis/nodejs-bigquery/blob/22021957f697ce67491bd50535f6fb43a99feea0/src/bigquery.ts#L1111
|
||||
//
|
||||
// Another, less user-friendly option, would be to allow users to specify the types manually.
|
||||
return { name, parameterType: { type: 'STRING' }, parameterValue: { value } };
|
||||
});
|
||||
}
|
||||
|
||||
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
|
||||
const response: ResponseWithJobReference = await googleBigQueryApiRequest.call(
|
||||
this,
|
||||
'POST',
|
||||
`/v2/projects/${projectId}/jobs`,
|
||||
{
|
||||
configuration: {
|
||||
query: body,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (!response?.jobReference?.jobId) {
|
||||
throw new NodeOperationError(this.getNode(), `No job ID returned, item ${i}`, {
|
||||
description: `sql: ${sqlQuery}`,
|
||||
itemIndex: i,
|
||||
});
|
||||
}
|
||||
|
||||
const jobId = response?.jobReference?.jobId;
|
||||
const raw = rawOutput || options.dryRun || false;
|
||||
const location = options.location || response.jobReference.location;
|
||||
|
||||
if (response.status?.state === 'DONE') {
|
||||
const qs = { location, maxResults, timeoutMs };
|
||||
|
||||
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
|
||||
const queryResponse: IDataObject = await googleBigQueryApiRequestAllItems.call(
|
||||
this,
|
||||
'GET',
|
||||
`/v2/projects/${projectId}/queries/${jobId}`,
|
||||
undefined,
|
||||
qs,
|
||||
);
|
||||
|
||||
if (body.returnAsNumbers === true) {
|
||||
const numericDataTypes = ['INTEGER', 'NUMERIC', 'FLOAT', 'BIGNUMERIC']; // https://cloud.google.com/bigquery/docs/schemas#standard_sql_data_types
|
||||
const schema: IDataObject = queryResponse?.schema as IDataObject;
|
||||
const schemaFields: IDataObject[] = schema.fields as IDataObject[];
|
||||
const schemaDataTypes: string[] = schemaFields?.map(
|
||||
(field: IDataObject) => field.type as string,
|
||||
);
|
||||
const rows: IDataObject[] = queryResponse.rows as IDataObject[];
|
||||
|
||||
for (const row of rows) {
|
||||
if (!row?.f || !Array.isArray(row.f)) continue;
|
||||
row.f.forEach((entry: IDataObject, index: number) => {
|
||||
if (entry && typeof entry === 'object' && 'v' in entry) {
|
||||
// Skip this row if it's null or doesn't have 'f' as an array
|
||||
const value = entry.v;
|
||||
if (numericDataTypes.includes(schemaDataTypes[index])) {
|
||||
entry.v = Number(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
returnData.push(...prepareOutput.call(this, queryResponse, i, raw, includeSchema));
|
||||
} else {
|
||||
jobs.push({ jobId, projectId, i, raw, includeSchema, location });
|
||||
}
|
||||
} catch (error) {
|
||||
if (this.continueOnFail()) {
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: error.message }),
|
||||
{ itemData: { item: i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
if ((error.message as string).includes('location') || error.httpCode === '404') {
|
||||
error.description =
|
||||
"Are you sure your table is in that region? You can specify the region using the 'Location' parameter from options.";
|
||||
}
|
||||
|
||||
if (error.httpCode === '403' && error.message.includes('Drive')) {
|
||||
error.description =
|
||||
'If your table(s) pull from a document in Google Drive, make sure that document is shared with your user';
|
||||
}
|
||||
|
||||
throw new NodeOperationError(this.getNode(), error as Error, {
|
||||
itemIndex: i,
|
||||
description: error.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let waitTime = 1000;
|
||||
outerLoop: while (jobs.length > 0) {
|
||||
const settledJobs: string[] = [];
|
||||
|
||||
for (const job of jobs) {
|
||||
if (abortSignal?.aborted) {
|
||||
break outerLoop;
|
||||
}
|
||||
try {
|
||||
const qs: IDataObject = job.location ? { location: job.location } : {};
|
||||
|
||||
qs.maxResults = maxResults;
|
||||
qs.timeoutMs = timeoutMs;
|
||||
|
||||
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
|
||||
const response: IDataObject = await googleBigQueryApiRequestAllItems.call(
|
||||
this,
|
||||
'GET',
|
||||
`/v2/projects/${job.projectId}/queries/${job.jobId}`,
|
||||
undefined,
|
||||
qs,
|
||||
);
|
||||
|
||||
if (response.jobComplete) {
|
||||
settledJobs.push(job.jobId);
|
||||
|
||||
returnData.push(...prepareOutput.call(this, response, job.i, job.raw, job.includeSchema));
|
||||
}
|
||||
if ((response?.errors as IDataObject[])?.length) {
|
||||
const errorMessages = (response.errors as IDataObject[]).map((error) => error.message);
|
||||
throw new ApplicationError(
|
||||
`Error(s) ocurring while executing query from item ${job.i.toString()}: ${errorMessages.join(
|
||||
', ',
|
||||
)}`,
|
||||
{ level: 'warning' },
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (this.continueOnFail()) {
|
||||
settledJobs.push(job.jobId);
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: error.message }),
|
||||
{ itemData: { item: job.i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
throw new NodeOperationError(this.getNode(), error as Error, {
|
||||
itemIndex: job.i,
|
||||
description: error.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
jobs = jobs.filter((job) => !settledJobs.includes(job.jobId));
|
||||
|
||||
if (jobs.length > 0) {
|
||||
await sleep(waitTime);
|
||||
if (waitTime < 30000) {
|
||||
waitTime = waitTime * 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
import type {
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
} from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
import { generatePairedItemData, updateDisplayOptions } from '@utils/utilities';
|
||||
|
||||
import type { TableSchema } from '../../helpers/interfaces';
|
||||
import { checkSchema, wrapData } from '../../helpers/utils';
|
||||
import { googleBigQueryApiRequest } from '../../transport';
|
||||
|
||||
const properties: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Data Mode',
|
||||
name: 'dataMode',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Auto-Map Input Data',
|
||||
value: 'autoMap',
|
||||
description: 'Use when node input properties match destination field names',
|
||||
},
|
||||
{
|
||||
name: 'Map Each Field Below',
|
||||
value: 'define',
|
||||
description: 'Set the value for each destination field',
|
||||
},
|
||||
],
|
||||
default: 'autoMap',
|
||||
description: 'Whether to insert the input data this node receives in the new row',
|
||||
},
|
||||
{
|
||||
displayName:
|
||||
"In this mode, make sure the incoming data fields are named the same as the columns in BigQuery. (Use an 'Edit Fields' node before this node to change them if required.)",
|
||||
name: 'info',
|
||||
type: 'notice',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['autoMap'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Send',
|
||||
name: 'fieldsUi',
|
||||
placeholder: 'Add Field',
|
||||
type: 'fixedCollection',
|
||||
typeOptions: {
|
||||
multipleValueButtonText: 'Add Field',
|
||||
multipleValues: true,
|
||||
},
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Field Name or ID',
|
||||
name: 'fieldId',
|
||||
type: 'options',
|
||||
description:
|
||||
'Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>',
|
||||
typeOptions: {
|
||||
loadOptionsDependsOn: ['projectId.value', 'datasetId.value', 'tableId.value'],
|
||||
loadOptionsMethod: 'getSchema',
|
||||
},
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'Field Value',
|
||||
name: 'fieldValue',
|
||||
type: 'string',
|
||||
default: '',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['define'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Batch Size',
|
||||
name: 'batchSize',
|
||||
type: 'number',
|
||||
default: 100,
|
||||
typeOptions: {
|
||||
minValue: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Ignore Unknown Values',
|
||||
name: 'ignoreUnknownValues',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to gnore row values that do not match the schema',
|
||||
},
|
||||
{
|
||||
displayName: 'Skip Invalid Rows',
|
||||
name: 'skipInvalidRows',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to skip rows with values that do not match the schema',
|
||||
},
|
||||
{
|
||||
displayName: 'Template Suffix',
|
||||
name: 'templateSuffix',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
|
||||
},
|
||||
{
|
||||
displayName: 'Trace ID',
|
||||
name: 'traceId',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['insert'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
||||
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
|
||||
const projectId = this.getNodeParameter('projectId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
const datasetId = this.getNodeParameter('datasetId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
const tableId = this.getNodeParameter('tableId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
|
||||
const options = this.getNodeParameter('options', 0);
|
||||
const dataMode = this.getNodeParameter('dataMode', 0) as string;
|
||||
|
||||
let batchSize = 100;
|
||||
if (options.batchSize) {
|
||||
batchSize = options.batchSize as number;
|
||||
delete options.batchSize;
|
||||
}
|
||||
|
||||
const items = this.getInputData();
|
||||
const length = items.length;
|
||||
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
const rows: IDataObject[] = [];
|
||||
const body: IDataObject = {};
|
||||
|
||||
Object.assign(body, options);
|
||||
if (body.traceId === undefined) {
|
||||
body.traceId = uuid();
|
||||
}
|
||||
|
||||
const schema = (
|
||||
await googleBigQueryApiRequest.call(
|
||||
this,
|
||||
'GET',
|
||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
||||
{},
|
||||
)
|
||||
).schema as TableSchema;
|
||||
|
||||
if (schema === undefined) {
|
||||
throw new NodeOperationError(this.getNode(), 'The destination table has no defined schema');
|
||||
}
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
try {
|
||||
const record: IDataObject = {};
|
||||
|
||||
if (dataMode === 'autoMap') {
|
||||
schema.fields.forEach(({ name }) => {
|
||||
record[name] = items[i].json[name];
|
||||
});
|
||||
}
|
||||
|
||||
if (dataMode === 'define') {
|
||||
const fields = this.getNodeParameter('fieldsUi.values', i, []) as IDataObject[];
|
||||
|
||||
fields.forEach(({ fieldId, fieldValue }) => {
|
||||
record[`${fieldId}`] = fieldValue;
|
||||
});
|
||||
}
|
||||
|
||||
rows.push({ json: checkSchema.call(this, schema, record, i) });
|
||||
} catch (error) {
|
||||
if (this.continueOnFail()) {
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: error.message }),
|
||||
{ itemData: { item: i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
throw new NodeOperationError(this.getNode(), error.message as string, {
|
||||
itemIndex: i,
|
||||
description: error?.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const itemData = generatePairedItemData(items.length);
|
||||
for (let i = 0; i < rows.length; i += batchSize) {
|
||||
const batch = rows.slice(i, i + batchSize);
|
||||
body.rows = batch;
|
||||
|
||||
const responseData = await googleBigQueryApiRequest.call(
|
||||
this,
|
||||
'POST',
|
||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
|
||||
body,
|
||||
);
|
||||
|
||||
if (responseData?.insertErrors && !options.skipInvalidRows) {
|
||||
const errors: string[] = [];
|
||||
const failedRows: number[] = [];
|
||||
const stopedRows: number[] = [];
|
||||
|
||||
(responseData.insertErrors as IDataObject[]).forEach((entry) => {
|
||||
const invalidRows = (entry.errors as IDataObject[]).filter(
|
||||
(error) => error.reason !== 'stopped',
|
||||
);
|
||||
if (invalidRows.length) {
|
||||
const entryIndex = (entry.index as number) + i;
|
||||
errors.push(
|
||||
`Row ${entryIndex} failed with error: ${invalidRows
|
||||
.map((error) => error.message)
|
||||
.join(', ')}`,
|
||||
);
|
||||
failedRows.push(entryIndex);
|
||||
} else {
|
||||
const entryIndex = (entry.index as number) + i;
|
||||
stopedRows.push(entryIndex);
|
||||
}
|
||||
});
|
||||
|
||||
if (this.continueOnFail()) {
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: errors.join('\n, ') }),
|
||||
{ itemData: { item: i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
|
||||
const failedMessage = `Problem inserting item(s) [${failedRows.join(', ')}]`;
|
||||
const stoppedMessage = stopedRows.length
|
||||
? `, nothing was inserted item(s) [${stopedRows.join(', ')}]`
|
||||
: '';
|
||||
throw new NodeOperationError(this.getNode(), `${failedMessage}${stoppedMessage}`, {
|
||||
description: errors.join('\n, '),
|
||||
itemIndex: i,
|
||||
});
|
||||
}
|
||||
|
||||
const executionData = this.helpers.constructExecutionMetaData(
|
||||
wrapData(responseData as IDataObject[]),
|
||||
{ itemData },
|
||||
);
|
||||
|
||||
returnData.push(...executionData);
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
Reference in New Issue
Block a user