alighasami/n8n
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

first commit
Security: Sync from Public / sync-from-public (push) Has been cancelled
Details
Test: Benchmark Nightly / build (push) Has been cancelled
Details
Test: Benchmark Nightly / Notify Cats on failure (push) Has been cancelled
Details
CI: Python / Checks (push) Has been cancelled
Details
Test: Evals Python / Workflow Comparison Python (push) Has been cancelled
Details
Util: Check Docs URLs / check-docs-urls (push) Has been cancelled
Details
Test: Visual Storybook / Cloudflare Pages (push) Has been cancelled
Details
Test: E2E Performance / build-and-test-performance (push) Has been cancelled
Details
Test: Workflows Nightly / Run Workflow Tests (push) Has been cancelled
Details
Util: Cleanup CI Docker Images / Delete stale CI images (push) Has been cancelled
Details
Test: Benchmark Destroy Env / build (push) Has been cancelled
Details
Util: Update Node Popularity / update-popularity (push) Has been cancelled
Details
Test: E2E Coverage Weekly / Coverage Tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
alighasami
2026-03-17 16:22:57 +03:30
commit 3d5eaf9445
15349 changed files with 2847338 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.json
+18
View File
@@ -0,0 +1,18 @@
{
"node": "n8n-nodes-base.googleBigQuery",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"categories": ["Data & Storage", "Development"],
"resources": {
"credentialDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/credentials/google/oauth-single-service/"
}
],
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.googlebigquery/"
}
]
}
}
packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts
+27
View File
@@ -0,0 +1,27 @@
import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow';
import { VersionedNodeType } from 'n8n-workflow';
import { GoogleBigQueryV1 } from './v1/GoogleBigQueryV1.node';
import { GoogleBigQueryV2 } from './v2/GoogleBigQueryV2.node';
export class GoogleBigQuery extends VersionedNodeType {
constructor() {
const baseDescription: INodeTypeBaseDescription = {
displayName: 'Google BigQuery',
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
description: 'Consume Google BigQuery API',
defaultVersion: 2.1,
};
const nodeVersions: IVersionedNodeType['nodeVersions'] = {
1: new GoogleBigQueryV1(baseDescription),
2: new GoogleBigQueryV2(baseDescription),
2.1: new GoogleBigQueryV2(baseDescription),
};
super(nodeVersions, baseDescription);
}
}
packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" fill="#fff" fill-rule="evenodd" stroke="#000" stroke-linecap="round" stroke-linejoin="round" viewBox="0 0 66 58"><use xlink:href="#a" x=".5" y=".5"/><symbol id="a" overflow="visible"><g fill-rule="nonzero" stroke="none"><path fill="#4386fa" d="M14.48 54.473.558 30.359a4.16 4.16 0 0 1 0-4.164L14.48 2.082A4.16 4.16 0 0 1 18.08 0h27.857c1.48.007 2.845.8 3.585 2.082l13.92 24.113a4.16 4.16 0 0 1 0 4.164L49.52 54.473a4.16 4.16 0 0 1-3.6 2.082H18.07a4.17 4.17 0 0 1-3.593-2.082z"/><path fill="#000" d="M40.697 20.512s3.87 9.283-1.406 14.545-14.883 1.894-14.883 1.894L43.95 56.547h1.984a4.16 4.16 0 0 0 3.6-2.082l9.216-15.958z" opacity=".1"/><path d="M45.266 39.507 41 35.23a.7.7 0 0 0-.158-.12 11.63 11.63 0 0 0-1.499-15.859 11.63 11.63 0 0 0-16.396 16.436 11.63 11.63 0 0 0 15.863 1.46.7.7 0 0 0 .113.15l4.277 4.277a.67.67 0 0 0 .947 0l1.12-1.12a.67.67 0 0 0 0-.947zM31.64 36.741a8.75 8.75 0 0 1-6.188-14.937 8.75 8.75 0 0 1 14.937 6.188 8.75 8.75 0 0 1-8.749 8.749m-5.593-9.216v3.616a6.4 6.4 0 0 0 2.338 2.375v-6.013zm4.375-2.998v9.772a6.5 6.5 0 0 0 2.338 0v-9.772zm6.764 6.606v-2.142H34.85v4.5a6.43 6.43 0 0 0 2.338-2.368z"/></g></symbol></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.queryParameters.test.ts
+61
View File
@@ -0,0 +1,61 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import nock from 'nock';
jest.mock('jsonwebtoken', () => ({
sign: jest.fn().mockReturnValue('signature'),
}));
describe('Test Google BigQuery V2, executeQuery with named parameters', () => {
nock('https://oauth2.googleapis.com')
.persist()
.post(
'/token',
'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=signature',
)
.reply(200, { access_token: 'token' });
nock('https://bigquery.googleapis.com/bigquery')
.post('/v2/projects/test-project/jobs', {
configuration: {
query: {
queryParameters: [
{
name: 'email',
parameterType: { type: 'STRING' },
parameterValue: { value: 'test@n8n.io' },
},
{
name: 'name',
parameterType: { type: 'STRING' },
parameterValue: { value: 'Test Testerson' },
},
{
name: 'n8n_variable',
parameterType: { type: 'STRING' },
parameterValue: { value: 42 },
},
],
query:
'SELECT * FROM bigquery_node_dev_test_dataset.test_json WHERE email = @email AND name = @name AND n8n_variable = @n8n_variable;',
useLegacySql: false,
parameterMode: 'NAMED',
},
},
})
.reply(200, {
jobReference: {
jobId: 'job_123',
},
status: {
state: 'DONE',
},
})
.get('/v2/projects/test-project/queries/job_123')
.reply(200)
.get('/v2/projects/test-project/queries/job_123?maxResults=1000&timeoutMs=10000')
.reply(200, { rows: [], schema: {} });
new NodeTestHarness().setupTests({
workflowFiles: ['executeQuery.queryParameters.workflow.json'],
});
});
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.queryParameters.workflow.json
+79
View File
@@ -0,0 +1,79 @@
{
"name": "My workflow 12",
"nodes": [
{
"parameters": {},
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [360, 340]
},
{
"parameters": {
"authentication": "serviceAccount",
"projectId": {
"__rl": true,
"value": "test-project",
"mode": "list",
"cachedResultName": "test-project",
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
},
"sqlQuery": "SELECT * FROM bigquery_node_dev_test_dataset.test_json WHERE email = @email AND name = @name AND n8n_variable = @n8n_variable;",
"options": {
"queryParameters": {
"namedParameters": [
{
"name": "email",
"value": "test@n8n.io"
},
{
"name": "name",
"value": "Test Testerson"
},
{
"name": "n8n_variable",
"value": "={{ 40 + 2 }}"
}
]
}
}
},
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
"name": "Google BigQuery",
"type": "n8n-nodes-base.googleBigQuery",
"typeVersion": 2,
"position": [620, 340],
"credentials": {
"googleApi": {
"id": "66",
"name": "Google account 5"
}
}
}
],
"pinData": {
"Google BigQuery": []
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Google BigQuery",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "be2fc126-5d71-4e86-9a4e-eb62ad266860",
"id": "156",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts
+42
View File
@@ -0,0 +1,42 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import nock from 'nock';
jest.mock('jsonwebtoken', () => ({
sign: jest.fn().mockReturnValue('signature'),
}));
describe('Test Google BigQuery V2, executeQuery', () => {
nock('https://oauth2.googleapis.com')
.persist()
.post(
'/token',
'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=signature',
)
.reply(200, { access_token: 'token' });
nock('https://bigquery.googleapis.com/bigquery')
.post('/v2/projects/test-project/jobs', {
configuration: {
query: {
query: 'SELECT * FROM bigquery_node_dev_test_dataset.test_json;',
useLegacySql: false,
},
},
})
.reply(200, {
jobReference: {
jobId: 'job_123',
},
status: {
state: 'DONE',
},
})
.get('/v2/projects/test-project/queries/job_123')
.reply(200)
.get('/v2/projects/test-project/queries/job_123?maxResults=1000&timeoutMs=10000')
.reply(200, { rows: [], schema: {} });
new NodeTestHarness().setupTests({
workflowFiles: ['executeQuery.workflow.json'],
});
});
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.workflow.json
+62
View File
@@ -0,0 +1,62 @@
{
"name": "My workflow 12",
"nodes": [
{
"parameters": {},
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [360, 340]
},
{
"parameters": {
"authentication": "serviceAccount",
"projectId": {
"__rl": true,
"value": "test-project",
"mode": "list",
"cachedResultName": "test-project",
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
},
"sqlQuery": "SELECT * FROM bigquery_node_dev_test_dataset.test_json;",
"options": {}
},
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
"name": "Google BigQuery",
"type": "n8n-nodes-base.googleBigQuery",
"typeVersion": 2,
"position": [620, 340],
"credentials": {
"googleApi": {
"id": "66",
"name": "Google account 5"
}
}
}
],
"pinData": {
"Google BigQuery": []
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Google BigQuery",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "be2fc126-5d71-4e86-9a4e-eb62ad266860",
"id": "156",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQueryContinueOnJobFail.test.ts
+40
View File
@@ -0,0 +1,40 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import nock from 'nock';
jest.mock('jsonwebtoken', () => ({
sign: jest.fn().mockReturnValue('signature'),
}));
describe('Test Google BigQuery V2, executeQuery', () => {
nock('https://oauth2.googleapis.com')
.persist()
.post(
'/token',
'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=signature',
)
.reply(200, { access_token: 'token' });
nock('https://bigquery.googleapis.com/bigquery')
.post('/v2/projects/test-project/jobs', {
configuration: {
query: {
query: 'SELECT * FROM bigquery_node_dev_test_dataset.test_json;',
useLegacySql: false,
},
},
})
.reply(200, {
jobReference: {
jobId: 'job_123',
},
status: {
state: 'RUNNING',
},
})
.get('/v2/projects/test-project/queries/job_123?maxResults=1000&timeoutMs=10000')
.replyWithError('Internal server error');
new NodeTestHarness().setupTests({
workflowFiles: ['executeQueryContinueOnJobFail.workflow.json'],
});
});
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQueryContinueOnJobFail.workflow.json
+69
View File
@@ -0,0 +1,69 @@
{
"name": "My workflow 12",
"nodes": [
{
"parameters": {},
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [360, 340]
},
{
"parameters": {
"authentication": "serviceAccount",
"projectId": {
"__rl": true,
"value": "test-project",
"mode": "list",
"cachedResultName": "test-project",
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
},
"sqlQuery": "SELECT * FROM bigquery_node_dev_test_dataset.test_json;",
"options": {}
},
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
"name": "Google BigQuery",
"type": "n8n-nodes-base.googleBigQuery",
"typeVersion": 2,
"position": [620, 340],
"credentials": {
"googleApi": {
"id": "66",
"name": "Google account 5"
}
},
"onError": "continueRegularOutput"
}
],
"pinData": {
"Google BigQuery": [
{
"json": {
"error": "Internal server error"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Google BigQuery",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "be2fc126-5d71-4e86-9a4e-eb62ad266860",
"id": "156",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts
+47
View File
@@ -0,0 +1,47 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import nock from 'nock';
jest.mock('jsonwebtoken', () => ({
sign: jest.fn().mockReturnValue('signature'),
}));
describe('Test Google BigQuery V2, insert auto map', () => {
nock('https://oauth2.googleapis.com')
.persist()
.post(
'/token',
'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=signature',
)
.reply(200, { access_token: 'token' });
nock('https://bigquery.googleapis.com/bigquery')
.get('/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text')
.reply(200, {
schema: {
fields: [
{ name: 'id', type: 'INT' },
{ name: 'test', type: 'STRING' },
],
},
})
.post(
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll',
{
rows: [
{ json: { id: 1, test: '111' } },
{ json: { id: 2, test: '222' } },
{ json: { id: 3, test: '333' } },
],
traceId: 'trace_id',
},
)
.reply(200, [
{ kind: 'bigquery#tableDataInsertAllResponse' },
{ kind: 'bigquery#tableDataInsertAllResponse' },
{ kind: 'bigquery#tableDataInsertAllResponse' },
]);
new NodeTestHarness().setupTests({
workflowFiles: ['insert.autoMapMode.workflow.json'],
});
});
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.workflow.json
+146
View File
@@ -0,0 +1,146 @@
{
"name": "My workflow 12",
"nodes": [
{
"parameters": {},
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [20, 340]
},
{
"parameters": {
"authentication": "serviceAccount",
"operation": "insert",
"projectId": {
"__rl": true,
"value": "test-project",
"mode": "list",
"cachedResultName": "test-project",
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
},
"datasetId": {
"__rl": true,
"value": "bigquery_node_dev_test_dataset",
"mode": "list",
"cachedResultName": "bigquery_node_dev_test_dataset"
},
"tableId": {
"__rl": true,
"value": "num_text",
"mode": "list",
"cachedResultName": "num_text"
},
"options": {
"traceId": "trace_id"
}
},
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
"name": "Google BigQuery",
"type": "n8n-nodes-base.googleBigQuery",
"typeVersion": 2,
"position": [500, 340],
"credentials": {
"googleApi": {
"id": "66",
"name": "Google account 5"
}
}
},
{
"parameters": {
"data": [
{
"id": 1,
"test": "111"
},
{
"id": 2,
"test": "222"
},
{
"id": 3,
"test": "333"
}
]
},
"id": "11d06660-cbd3-4bd2-9619-68e82438a0e3",
"name": "Code",
"type": "n8n-nodes-testing.testData",
"typeVersion": 1,
"position": [240, 340]
}
],
"pinData": {
"Code": [
{
"json": {
"id": 1,
"test": "111"
}
},
{
"json": {
"id": 2,
"test": "222"
}
},
{
"json": {
"id": 3,
"test": "333"
}
}
],
"Google BigQuery": [
{
"json": {
"kind": "bigquery#tableDataInsertAllResponse"
}
},
{
"json": {
"kind": "bigquery#tableDataInsertAllResponse"
}
},
{
"json": {
"kind": "bigquery#tableDataInsertAllResponse"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Google BigQuery",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "30d3e38a-b5a4-4999-816d-7c05a68f31c8",
"id": "156",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts
+40
View File
@@ -0,0 +1,40 @@
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
import nock from 'nock';
jest.mock('jsonwebtoken', () => ({
sign: jest.fn().mockReturnValue('signature'),
}));
describe('Test Google BigQuery V2, insert define manually', () => {
nock('https://oauth2.googleapis.com')
.persist()
.post(
'/token',
'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=signature',
)
.reply(200, { access_token: 'token' });
nock('https://bigquery.googleapis.com/bigquery')
.get('/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json')
.reply(200, {
schema: {
fields: [
{ name: 'json', type: 'JSON' },
{ name: 'name with space', type: 'STRING' },
{ name: 'active', type: 'BOOLEAN' },
],
},
})
.post(
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll',
{
rows: [{ json: { active: 'true', json: '{"test": 1}', 'name with space': 'some name' } }],
traceId: 'trace_id',
},
)
.reply(200, [{ kind: 'bigquery#tableDataInsertAllResponse' }]);
new NodeTestHarness().setupTests({
workflowFiles: ['insert.manualMode.workflow.json'],
});
});
packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.workflow.json
+99
View File
@@ -0,0 +1,99 @@
{
"name": "My workflow 12",
"nodes": [
{
"parameters": {},
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [360, 340]
},
{
"parameters": {
"authentication": "serviceAccount",
"operation": "insert",
"projectId": {
"__rl": true,
"value": "test-project",
"mode": "list",
"cachedResultName": "test-project",
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
},
"datasetId": {
"__rl": true,
"value": "bigquery_node_dev_test_dataset",
"mode": "list",
"cachedResultName": "bigquery_node_dev_test_dataset"
},
"tableId": {
"__rl": true,
"value": "test_json",
"mode": "list",
"cachedResultName": "test_json"
},
"dataMode": "define",
"fieldsUi": {
"values": [
{
"fieldId": "active",
"fieldValue": "true"
},
{
"fieldId": "name with space",
"fieldValue": "some name"
},
{
"fieldId": "json",
"fieldValue": "{\"test\": 1}"
}
]
},
"options": {
"traceId": "trace_id"
}
},
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
"name": "Google BigQuery",
"type": "n8n-nodes-base.googleBigQuery",
"typeVersion": 2,
"position": [620, 340],
"credentials": {
"googleApi": {
"id": "66",
"name": "Google account 5"
}
}
}
],
"pinData": {
"Google BigQuery": [
{
"json": {
"kind": "bigquery#tableDataInsertAllResponse"
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Google BigQuery",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "abd49f26-184d-4f9b-95f0-389ea20df809",
"id": "156",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}
packages/nodes-base/nodes/Google/BigQuery/test/v2/utils/utils.test.ts
+272
View File
@@ -0,0 +1,272 @@
import { mock } from 'jest-mock-extended';
import { constructExecutionMetaData } from 'n8n-core';
import type { IDataObject, IExecuteFunctions, INode } from 'n8n-workflow';
import { prepareOutput } from '../../../v2/helpers/utils';
describe('Google BigQuery v2 Utils', () => {
it('should prepareOutput', () => {
const thisArg = mock<IExecuteFunctions>({
getNode: () => ({ typeVersion: 2.1 }) as INode,
helpers: mock({ constructExecutionMetaData }),
});
const response: IDataObject = {
kind: 'bigquery#getQueryResultsResponse',
etag: 'e_tag',
schema: {
fields: [
{
name: 'nodes',
type: 'RECORD',
mode: 'REPEATED',
fields: [
{
name: 'webhookId',
type: 'STRING',
mode: 'NULLABLE',
},
{
name: 'position',
type: 'INTEGER',
mode: 'REPEATED',
},
{
name: 'name',
type: 'STRING',
mode: 'NULLABLE',
},
{
name: 'typeVersion',
type: 'INTEGER',
mode: 'NULLABLE',
},
{
name: 'credentials',
type: 'RECORD',
mode: 'NULLABLE',
fields: [
{
name: 'zendeskApi',
type: 'RECORD',
mode: 'NULLABLE',
fields: [
{
name: 'name',
type: 'STRING',
mode: 'NULLABLE',
},
{
name: 'id',
type: 'INTEGER',
mode: 'NULLABLE',
},
],
},
],
},
{
name: 'type',
type: 'STRING',
mode: 'NULLABLE',
},
{
name: 'parameters',
type: 'RECORD',
mode: 'NULLABLE',
fields: [
{
name: 'conditions',
type: 'RECORD',
mode: 'NULLABLE',
fields: [
{
name: 'all',
type: 'RECORD',
mode: 'REPEATED',
fields: [
{
name: 'value',
type: 'STRING',
mode: 'NULLABLE',
},
],
},
],
},
{
name: 'options',
type: 'RECORD',
mode: 'NULLABLE',
fields: [
{
name: 'fields',
type: 'STRING',
mode: 'REPEATED',
},
],
},
],
},
],
},
],
},
jobReference: {
projectId: 'project_id',
jobId: 'job_ref',
location: 'US',
},
totalRows: '1',
rows: [
{
f: [
{
v: [
{
v: {
f: [
{
v: 'web_hook_id',
},
{
v: [
{
v: '100',
},
{
v: '100',
},
],
},
{
v: 'Zendesk Trigger',
},
{
v: '1',
},
{
v: {
f: [
{
v: {
f: [
{
v: 'Zendesk account',
},
{
v: '8',
},
],
},
},
],
},
},
{
v: 'n8n-nodes-base.zendeskTrigger',
},
{
v: {
f: [
{
v: {
f: [
{
v: [
{
v: {
f: [
{
v: 'closed',
},
],
},
},
],
},
],
},
},
{
v: {
f: [
{
v: [
{
v: 'ticket.title',
},
{
v: 'ticket.description',
},
],
},
],
},
},
],
},
},
],
},
},
],
},
],
},
],
totalBytesProcessed: '0',
jobComplete: true,
cacheHit: true,
};
const returnData = prepareOutput.call(thisArg, response, 0, false, false);
expect(returnData).toBeDefined();
// expect(returnData).toHaveProperty('nodes');
expect(returnData).toEqual([
{
json: {
nodes: [
{
webhookId: 'web_hook_id',
position: ['100', '100'],
name: 'Zendesk Trigger',
typeVersion: '1',
credentials: [
{
zendeskApi: [
{
name: 'Zendesk account',
id: '8',
},
],
},
],
type: 'n8n-nodes-base.zendeskTrigger',
parameters: [
{
conditions: [
{
all: [
{
value: 'closed',
},
],
},
],
options: [
{
fields: ['ticket.title', 'ticket.description'],
},
],
},
],
},
],
},
pairedItem: {
item: 0,
},
},
]);
});
});
packages/nodes-base/nodes/Google/BigQuery/v1/GenericFunctions.ts
+102
View File
@@ -0,0 +1,102 @@
import type {
IExecuteFunctions,
ILoadOptionsFunctions,
IDataObject,
JsonObject,
IHttpRequestMethods,
IRequestOptions,
} from 'n8n-workflow';
import { NodeApiError, NodeOperationError } from 'n8n-workflow';
import { getGoogleAccessToken } from '../../GenericFunctions';
export async function googleApiRequest(
this: IExecuteFunctions | ILoadOptionsFunctions,
method: IHttpRequestMethods,
resource: string,
body: any = {},
qs: IDataObject = {},
uri?: string,
headers: IDataObject = {},
): Promise<any> {
const authenticationMethod = this.getNodeParameter(
'authentication',
0,
'serviceAccount',
) as string;
const options: IRequestOptions = {
headers: {
'Content-Type': 'application/json',
},
method,
body,
qs,
uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
json: true,
};
try {
if (Object.keys(headers).length !== 0) {
options.headers = Object.assign({}, options.headers, headers);
}
if (Object.keys(body as IDataObject).length === 0) {
delete options.body;
}
if (authenticationMethod === 'serviceAccount') {
const credentials = await this.getCredentials('googleApi');
if (credentials === undefined) {
throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
}
const { access_token } = await getGoogleAccessToken.call(this, credentials, 'bigquery');
options.headers!.Authorization = `Bearer ${access_token}`;
return await this.helpers.request(options);
} else {
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
}
} catch (error) {
if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
error.statusCode = '401';
}
throw new NodeApiError(this.getNode(), error as JsonObject);
}
}
export async function googleApiRequestAllItems(
this: IExecuteFunctions | ILoadOptionsFunctions,
propertyName: string,
method: IHttpRequestMethods,
endpoint: string,
body: any = {},
query: IDataObject = {},
): Promise<any> {
const returnData: IDataObject[] = [];
let responseData;
query.maxResults = 100;
do {
responseData = await googleApiRequest.call(this, method, endpoint, body, query);
query.pageToken = responseData.pageToken;
returnData.push.apply(returnData, responseData[propertyName] as IDataObject[]);
} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
return returnData;
}
export function simplify(rows: IDataObject[], fields: string[]) {
const results = [];
for (const row of rows) {
const record: IDataObject = {};
for (const [index, field] of fields.entries()) {
record[field] = (row.f as IDataObject[])[index].v;
}
results.push(record);
}
return results;
}
packages/nodes-base/nodes/Google/BigQuery/v1/GoogleBigQueryV1.node.ts
+305
View File
@@ -0,0 +1,305 @@
import type {
IDataObject,
IExecuteFunctions,
ILoadOptionsFunctions,
INodeExecutionData,
INodePropertyOptions,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
JsonObject,
} from 'n8n-workflow';
import { NodeConnectionTypes, NodeApiError } from 'n8n-workflow';
import { v4 as uuid } from 'uuid';
import { oldVersionNotice } from '@utils/descriptions';
import { googleApiRequest, googleApiRequestAllItems, simplify } from './GenericFunctions';
import { recordFields, recordOperations } from './RecordDescription';
import { generatePairedItemData } from '../../../../utils/utilities';
const versionDescription: INodeTypeDescription = {
displayName: 'Google BigQuery',
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
version: 1,
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
description: 'Consume Google BigQuery API',
defaults: {
name: 'Google BigQuery',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
credentials: [
{
name: 'googleApi',
required: true,
displayOptions: {
show: {
authentication: ['serviceAccount'],
},
},
},
{
name: 'googleBigQueryOAuth2Api',
required: true,
displayOptions: {
show: {
authentication: ['oAuth2'],
},
},
},
],
properties: [
oldVersionNotice,
{
displayName: 'Authentication',
name: 'authentication',
type: 'options',
noDataExpression: true,
options: [
{
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
name: 'OAuth2 (recommended)',
value: 'oAuth2',
},
{
name: 'Service Account',
value: 'serviceAccount',
},
],
default: 'oAuth2',
},
{
displayName: 'Resource',
name: 'resource',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Record',
value: 'record',
},
],
default: 'record',
},
...recordOperations,
...recordFields,
],
};
export class GoogleBigQueryV1 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
methods = {
loadOptions: {
async getProjects(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const returnData: INodePropertyOptions[] = [];
const { projects } = await googleApiRequest.call(this, 'GET', '/v2/projects');
for (const project of projects) {
returnData.push({
name: project.friendlyName as string,
value: project.id,
});
}
return returnData;
},
async getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const projectId = this.getCurrentNodeParameter('projectId');
const returnData: INodePropertyOptions[] = [];
const { datasets } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets`,
);
for (const dataset of datasets) {
returnData.push({
name: dataset.datasetReference.datasetId as string,
value: dataset.datasetReference.datasetId,
});
}
return returnData;
},
async getTables(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const projectId = this.getCurrentNodeParameter('projectId');
const datasetId = this.getCurrentNodeParameter('datasetId');
const returnData: INodePropertyOptions[] = [];
const { tables } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
);
for (const table of tables) {
returnData.push({
name: table.tableReference.tableId as string,
value: table.tableReference.tableId,
});
}
return returnData;
},
},
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const returnData: INodeExecutionData[] = [];
const length = items.length;
const qs: IDataObject = {};
let responseData;
const resource = this.getNodeParameter('resource', 0);
const operation = this.getNodeParameter('operation', 0);
if (resource === 'record') {
// *********************************************************************
// record
// *********************************************************************
if (operation === 'create') {
// ----------------------------------
// record: create
// ----------------------------------
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
const projectId = this.getNodeParameter('projectId', 0) as string;
const datasetId = this.getNodeParameter('datasetId', 0) as string;
const tableId = this.getNodeParameter('tableId', 0) as string;
const rows: IDataObject[] = [];
const body: IDataObject = {};
for (let i = 0; i < length; i++) {
const options = this.getNodeParameter('options', i);
Object.assign(body, options);
if (body.traceId === undefined) {
body.traceId = uuid();
}
const columns = this.getNodeParameter('columns', i) as string;
const columnList = columns.split(',').map((column) => column.trim());
const record: IDataObject = {};
for (const key of Object.keys(items[i].json)) {
if (columnList.includes(key)) {
record[`${key}`] = items[i].json[key];
}
}
rows.push({ json: record });
}
body.rows = rows;
const itemData = generatePairedItemData(items.length);
try {
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
const executionData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray(responseData as IDataObject[]),
{ itemData },
);
returnData.push(...executionData);
} catch (error) {
if (this.continueOnFail()) {
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: error.message }),
{ itemData },
);
returnData.push(...executionErrorData);
}
throw new NodeApiError(this.getNode(), error as JsonObject, { itemIndex: 0 });
}
} else if (operation === 'getAll') {
// ----------------------------------
// record: getAll
// ----------------------------------
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get
const returnAll = this.getNodeParameter('returnAll', 0);
const projectId = this.getNodeParameter('projectId', 0) as string;
const datasetId = this.getNodeParameter('datasetId', 0) as string;
const tableId = this.getNodeParameter('tableId', 0) as string;
const simple = this.getNodeParameter('simple', 0) as boolean;
let fields;
if (simple) {
const { schema } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
{},
);
fields = (schema.fields || []).map((field: IDataObject) => field.name);
}
for (let i = 0; i < length; i++) {
try {
const options = this.getNodeParameter('options', i);
Object.assign(qs, options);
if (qs.selectedFields) {
fields = (qs.selectedFields as string).split(',');
}
if (returnAll) {
responseData = await googleApiRequestAllItems.call(
this,
'rows',
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
} else {
qs.maxResults = this.getNodeParameter('limit', i);
responseData = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
}
if (!returnAll) {
responseData = responseData.rows;
}
responseData = simple
? simplify(responseData as IDataObject[], fields as string[])
: responseData;
const executionData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray(responseData as IDataObject[]),
{ itemData: { item: i } },
);
returnData.push(...executionData);
} catch (error) {
if (this.continueOnFail()) {
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: error.message }),
{ itemData: { item: i } },
);
returnData.push(...executionErrorData);
continue;
}
throw new NodeApiError(this.getNode(), error as JsonObject, { itemIndex: i });
}
}
}
}
return [returnData];
}
}
packages/nodes-base/nodes/Google/BigQuery/v1/RecordDescription.ts
+286
View File
@@ -0,0 +1,286 @@
import type { INodeProperties } from 'n8n-workflow';
export const recordOperations: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
displayOptions: {
show: {
resource: ['record'],
},
},
options: [
{
name: 'Create',
value: 'create',
description: 'Create a new record',
action: 'Create a record',
},
{
name: 'Get Many',
value: 'getAll',
description: 'Retrieve many records',
action: 'Get many records',
},
],
default: 'create',
},
];
export const recordFields: INodeProperties[] = [
// ----------------------------------
// record: create
// ----------------------------------
{
displayName: 'Project Name or ID',
name: 'projectId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getProjects',
},
required: true,
displayOptions: {
show: {
operation: ['create'],
resource: ['record'],
},
},
default: '',
description:
'ID of the project to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Dataset Name or ID',
name: 'datasetId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getDatasets',
loadOptionsDependsOn: ['projectId'],
},
required: true,
displayOptions: {
show: {
operation: ['create'],
resource: ['record'],
},
},
default: '',
description:
'ID of the dataset to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Table Name or ID',
name: 'tableId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getTables',
loadOptionsDependsOn: ['projectId', 'datasetId'],
},
required: true,
displayOptions: {
show: {
operation: ['create'],
resource: ['record'],
},
},
default: '',
description:
'ID of the table to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Columns',
name: 'columns',
type: 'string',
displayOptions: {
show: {
resource: ['record'],
operation: ['create'],
},
},
default: '',
required: true,
placeholder: 'id,name,description',
description: 'Comma-separated list of the item properties to use as columns',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add option',
default: {},
displayOptions: {
show: {
operation: ['create'],
resource: ['record'],
},
},
options: [
{
displayName: 'Ignore Unknown Values',
name: 'ignoreUnknownValues',
type: 'boolean',
default: false,
description: 'Whether to ignore row values that do not match the schema',
},
{
displayName: 'Skip Invalid Rows',
name: 'skipInvalidRows',
type: 'boolean',
default: false,
description: 'Whether to skip rows with values that do not match the schema',
},
{
displayName: 'Template Suffix',
name: 'templateSuffix',
type: 'string',
default: '',
description:
'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
},
{
displayName: 'Trace ID',
name: 'traceId',
type: 'string',
default: '',
description:
'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
},
],
},
// ----------------------------------
// record: getAll
// ----------------------------------
{
displayName: 'Project Name or ID',
name: 'projectId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getProjects',
},
required: true,
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
},
},
default: '',
description:
'ID of the project to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Dataset Name or ID',
name: 'datasetId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getDatasets',
loadOptionsDependsOn: ['projectId'],
},
required: true,
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
},
},
default: '',
description:
'ID of the dataset to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Table Name or ID',
name: 'tableId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getTables',
loadOptionsDependsOn: ['projectId', 'datasetId'],
},
required: true,
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
},
},
default: '',
description:
'ID of the table to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Return All',
name: 'returnAll',
type: 'boolean',
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
},
},
default: false,
description: 'Whether to return all results or only up to a given limit',
},
{
displayName: 'Limit',
name: 'limit',
type: 'number',
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
returnAll: [false],
},
},
typeOptions: {
minValue: 1,
maxValue: 500,
},
default: 50,
description: 'Max number of results to return',
},
{
displayName: 'Simplify',
name: 'simple',
type: 'boolean',
displayOptions: {
show: {
resource: ['record'],
operation: ['getAll'],
},
},
default: true,
description: 'Whether to return a simplified version of the response instead of the raw data',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add option',
default: {},
displayOptions: {
show: {
operation: ['getAll'],
resource: ['record'],
},
},
options: [
{
displayName: 'Fields',
name: 'selectedFields',
type: 'string',
default: '',
description:
'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>',
},
// {
// displayName: 'Use Int64 Timestamp',
// name: 'useInt64Timestamp',
// type: 'boolean',
// default: false,
// description: 'Output timestamp as usec int64.',
// },
],
},
];
packages/nodes-base/nodes/Google/BigQuery/v2/GoogleBigQueryV2.node.ts
+29
View File
@@ -0,0 +1,29 @@
import type {
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
} from 'n8n-workflow';
import { router } from './actions/router';
import { versionDescription } from './actions/versionDescription';
import { loadOptions, listSearch } from './methods';
export class GoogleBigQueryV2 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
usableAsTool: true,
};
}
methods = { loadOptions, listSearch };
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
return await router.call(this);
}
}
packages/nodes-base/nodes/Google/BigQuery/v2/actions/commonDescriptions/RLC.description.ts
+121
View File
@@ -0,0 +1,121 @@
import type { INodeProperties } from 'n8n-workflow';
export const projectRLC: INodeProperties = {
displayName: 'Project',
name: 'projectId',
type: 'resourceLocator',
default: { mode: 'list', value: '' },
required: true,
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'searchProjects',
searchable: true,
},
},
{
displayName: 'By URL',
name: 'url',
type: 'string',
extractValue: {
type: 'regex',
regex: 'https:\\/\\/console.cloud.google.com\\/bigquery\\?project=([0-9a-zA-Z\\-_]+).{0,}',
},
validation: [
{
type: 'regex',
properties: {
regex:
'https:\\/\\/console.cloud.google.com\\/bigquery\\?project=([0-9a-zA-Z\\-_]+).{0,}',
errorMessage: 'Not a valid BigQuery Project URL',
},
},
],
},
{
displayName: 'By ID',
name: 'id',
type: 'string',
validation: [
{
type: 'regex',
properties: {
regex: '[a-zA-Z0-9\\-_]{2,}',
errorMessage: 'Not a valid BigQuery Project ID',
},
},
],
url: '=https://console.cloud.google.com/bigquery?project={{$value}}',
},
],
description: 'Projects to which you have been granted any project role',
};
export const datasetRLC: INodeProperties = {
displayName: 'Dataset',
name: 'datasetId',
type: 'resourceLocator',
default: { mode: 'list', value: '' },
required: true,
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'searchDatasets',
searchable: true,
},
},
{
displayName: 'By ID',
name: 'id',
type: 'string',
validation: [
{
type: 'regex',
properties: {
regex: '[a-zA-Z0-9\\-_]{2,}',
errorMessage: 'Not a valid Dataset ID',
},
},
],
},
],
};
export const tableRLC: INodeProperties = {
displayName: 'Table',
name: 'tableId',
type: 'resourceLocator',
default: { mode: 'list', value: '' },
required: true,
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'searchTables',
searchable: true,
},
},
{
displayName: 'By ID',
name: 'id',
type: 'string',
validation: [
{
type: 'regex',
properties: {
regex: '[a-zA-Z0-9\\-_]{2,}',
errorMessage: 'Not a valid Table ID',
},
},
],
},
],
};
packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/Database.resource.ts
+65
View File
@@ -0,0 +1,65 @@
import type { INodeProperties } from 'n8n-workflow';
import * as executeQuery from './executeQuery.operation';
import * as insert from './insert.operation';
import { datasetRLC, projectRLC, tableRLC } from '../commonDescriptions/RLC.description';
export { executeQuery, insert };
export const description: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
displayOptions: {
show: {
resource: ['database'],
},
},
options: [
{
name: 'Execute Query',
value: 'executeQuery',
description: 'Execute a SQL query',
action: 'Execute a SQL query',
},
{
name: 'Insert',
value: 'insert',
description: 'Insert rows in a table',
action: 'Insert rows in a table',
},
],
default: 'executeQuery',
},
{
...projectRLC,
displayOptions: {
show: {
resource: ['database'],
operation: ['executeQuery', 'insert'],
},
},
},
{
...datasetRLC,
displayOptions: {
show: {
resource: ['database'],
operation: ['insert'],
},
},
},
{
...tableRLC,
displayOptions: {
show: {
resource: ['database'],
operation: ['insert'],
},
},
},
...executeQuery.description,
...insert.description,
];
packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts
+468
View File
@@ -0,0 +1,468 @@
import type {
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow';
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
import type { ResponseWithJobReference } from '../../helpers/interfaces';
import { prepareOutput } from '../../helpers/utils';
import { googleBigQueryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport';
interface IQueryParameterOptions {
namedParameters: Array<{
name: string;
value: string;
}>;
}
const properties: INodeProperties[] = [
{
displayName: 'SQL Query',
name: 'sqlQuery',
type: 'string',
noDataExpression: true,
typeOptions: {
editor: 'sqlEditor',
},
displayOptions: {
hide: {
'/options.useLegacySql': [true],
},
},
default: '',
placeholder: 'SELECT * FROM dataset.table LIMIT 100',
description:
'SQL query to execute, you can find more information <a href="https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax" target="_blank">here</a>. Standard SQL syntax used by default, but you can also use Legacy SQL syntax by using optinon \'Use Legacy SQL\'.',
},
{
displayName: 'SQL Query',
name: 'sqlQuery',
type: 'string',
noDataExpression: true,
typeOptions: {
editor: 'sqlEditor',
},
displayOptions: {
show: {
'/options.useLegacySql': [true],
},
},
default: '',
placeholder: 'SELECT * FROM [project:dataset.table] LIMIT 100;',
hint: 'Legacy SQL syntax',
description:
'SQL query to execute, you can find more information about Legacy SQL syntax <a href="https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax" target="_blank">here</a>',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add option',
default: {},
options: [
{
displayName: 'Default Dataset Name or ID',
name: 'defaultDataset',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getDatasets',
loadOptionsDependsOn: ['projectId.value'],
},
default: '',
description:
'If not set, all table names in the query string must be qualified in the format \'datasetId.tableId\'. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
},
{
displayName: 'Dry Run',
name: 'dryRun',
type: 'boolean',
default: false,
description:
"Whether set to true BigQuery doesn't run the job. Instead, if the query is valid, BigQuery returns statistics about the job such as how many bytes would be processed. If the query is invalid, an error returns.",
},
{
displayName: 'Include Schema in Output',
name: 'includeSchema',
type: 'boolean',
default: false,
description:
"Whether to include the schema in the output. If set to true, the output will contain key '_schema' with the schema of the table.",
displayOptions: {
hide: {
rawOutput: [true],
},
},
},
{
displayName: 'Location (Region)',
name: 'location',
type: 'string',
default: '',
placeholder: 'e.g. europe-west3',
description:
'Location or the region where data would be stored and processed. Pricing for storage and analysis is also defined by location of data and reservations, more information <a href="https://cloud.google.com/bigquery/docs/locations" target="_blank">here</a>.',
},
{
displayName: 'Maximum Bytes Billed',
name: 'maximumBytesBilled',
type: 'string',
default: '',
description:
'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in <a href="https://developers.google.com/discovery/v1/type-format?utm_source=cloud.google.com&utm_medium=referral" target="_blank">Int64Value</a> format',
},
{
displayName: 'Max Results Per Page',
name: 'maxResults',
type: 'number',
default: 1000,
description:
'Maximum number of results to return per page of results. This is particularly useful when dealing with large datasets. It will not affect the total number of results returned, e.g. rows in a table. You can use LIMIT in your SQL query to limit the number of rows returned.',
},
{
displayName: 'Timeout',
name: 'timeoutMs',
type: 'number',
default: 10000,
hint: 'How long to wait for the query to complete, in milliseconds',
description:
'Specifies the maximum amount of time, in milliseconds, that the client is willing to wait for the query to complete. Be aware that the call is not guaranteed to wait for the specified timeout; it typically returns after around 200 seconds (200,000 milliseconds), even if the query is not complete.',
},
{
displayName: 'Raw Output',
name: 'rawOutput',
type: 'boolean',
default: false,
displayOptions: {
hide: {
dryRun: [true],
},
},
},
{
displayName: 'Use Legacy SQL',
name: 'useLegacySql',
type: 'boolean',
default: false,
description:
"Whether to use BigQuery's legacy SQL dialect for this query. If set to false, the query will use BigQuery's standard SQL.",
},
{
displayName: 'Return Integers as Numbers',
name: 'returnAsNumbers',
type: 'boolean',
default: false,
description:
'Whether all integer values will be returned as numbers. If set to false, all integer values will be returned as strings.',
},
{
displayName: 'Query Parameters (Named)',
name: 'queryParameters',
type: 'fixedCollection',
description:
'Use <a href="https://cloud.google.com/bigquery/docs/parameterized-queries#using_structs_in_parameterized_queries" target="_blank">parameterized queries</a> to prevent SQL injections. Positional arguments are not supported at the moment. This feature won\'t be available when using legacy SQL.',
displayOptions: {
hide: {
'/options.useLegacySql': [true],
},
},
typeOptions: {
multipleValues: true,
},
placeholder: 'Add Parameter',
default: {
namedParameters: [
{
name: '',
value: '',
},
],
},
options: [
{
name: 'namedParameters',
displayName: 'Named Parameter',
values: [
{
displayName: 'Name',
name: 'name',
type: 'string',
default: '',
description: 'Name of the parameter',
},
{
displayName: 'Value',
name: 'value',
type: 'string',
default: '',
description:
'The substitute value. It must be a string. Arrays, dates and struct types mentioned in <a href="https://cloud.google.com/bigquery/docs/parameterized-queries#using_structs_in_parameterized_queries" target="_blank">the official documentation</a> are not yet supported.',
},
],
},
],
},
],
},
];
const displayOptions = {
show: {
resource: ['database'],
operation: ['executeQuery'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
const abortSignal = this.getExecutionCancelSignal();
const items = this.getInputData();
const length = items.length;
const returnData: INodeExecutionData[] = [];
let jobs = [];
let maxResults = 1000;
let timeoutMs = 10000;
for (let i = 0; i < length; i++) {
try {
let sqlQuery = this.getNodeParameter('sqlQuery', i) as string;
const options = this.getNodeParameter('options', i) as {
defaultDataset?: string;
dryRun?: boolean;
includeSchema?: boolean;
location?: string;
maximumBytesBilled?: string;
maxResults?: number;
timeoutMs?: number;
rawOutput?: boolean;
useLegacySql?: boolean;
returnAsNumbers?: boolean;
queryParameters?: IQueryParameterOptions;
};
const projectId = this.getNodeParameter('projectId', i, undefined, {
extractValue: true,
});
for (const resolvable of getResolvables(sqlQuery)) {
sqlQuery = sqlQuery.replace(resolvable, this.evaluateExpression(resolvable, i) as string);
}
let rawOutput = false;
let includeSchema = false;
if (options.rawOutput !== undefined) {
rawOutput = options.rawOutput;
delete options.rawOutput;
}
if (options.includeSchema !== undefined) {
includeSchema = options.includeSchema;
delete options.includeSchema;
}
if (options.maxResults) {
maxResults = options.maxResults;
delete options.maxResults;
}
if (options.timeoutMs) {
timeoutMs = options.timeoutMs;
delete options.timeoutMs;
}
const body: IDataObject = { ...options };
body.query = sqlQuery;
if (body.defaultDataset) {
body.defaultDataset = {
datasetId: options.defaultDataset,
projectId,
};
}
if (body.useLegacySql === undefined) {
body.useLegacySql = false;
}
if (typeof body.queryParameters === 'object') {
const { namedParameters } = body.queryParameters as IQueryParameterOptions;
body.parameterMode = 'NAMED';
body.queryParameters = namedParameters.map(({ name, value }) => {
// BigQuery type descriptors are very involved, and it would be hard to support all possible
// options, that's why the only supported type here is "STRING".
//
// If we switch this node to the official JS SDK from Google, we should be able to use `getTypeDescriptorFromValue`
// at runtime, which would infer BQ type descriptors of any valid JS value automatically:
//
// https://github.com/googleapis/nodejs-bigquery/blob/22021957f697ce67491bd50535f6fb43a99feea0/src/bigquery.ts#L1111
//
// Another, less user-friendly option, would be to allow users to specify the types manually.
return { name, parameterType: { type: 'STRING' }, parameterValue: { value } };
});
}
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
const response: ResponseWithJobReference = await googleBigQueryApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/jobs`,
{
configuration: {
query: body,
},
},
);
if (!response?.jobReference?.jobId) {
throw new NodeOperationError(this.getNode(), `No job ID returned, item ${i}`, {
description: `sql: ${sqlQuery}`,
itemIndex: i,
});
}
const jobId = response?.jobReference?.jobId;
const raw = rawOutput || options.dryRun || false;
const location = options.location || response.jobReference.location;
if (response.status?.state === 'DONE') {
const qs = { location, maxResults, timeoutMs };
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
const queryResponse: IDataObject = await googleBigQueryApiRequestAllItems.call(
this,
'GET',
`/v2/projects/${projectId}/queries/${jobId}`,
undefined,
qs,
);
if (body.returnAsNumbers === true) {
const numericDataTypes = ['INTEGER', 'NUMERIC', 'FLOAT', 'BIGNUMERIC']; // https://cloud.google.com/bigquery/docs/schemas#standard_sql_data_types
const schema: IDataObject = queryResponse?.schema as IDataObject;
const schemaFields: IDataObject[] = schema.fields as IDataObject[];
const schemaDataTypes: string[] = schemaFields?.map(
(field: IDataObject) => field.type as string,
);
const rows: IDataObject[] = queryResponse.rows as IDataObject[];
for (const row of rows) {
if (!row?.f || !Array.isArray(row.f)) continue;
row.f.forEach((entry: IDataObject, index: number) => {
if (entry && typeof entry === 'object' && 'v' in entry) {
// Skip this row if it's null or doesn't have 'f' as an array
const value = entry.v;
if (numericDataTypes.includes(schemaDataTypes[index])) {
entry.v = Number(value);
}
}
});
}
}
returnData.push(...prepareOutput.call(this, queryResponse, i, raw, includeSchema));
} else {
jobs.push({ jobId, projectId, i, raw, includeSchema, location });
}
} catch (error) {
if (this.continueOnFail()) {
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: error.message }),
{ itemData: { item: i } },
);
returnData.push(...executionErrorData);
continue;
}
if ((error.message as string).includes('location') || error.httpCode === '404') {
error.description =
"Are you sure your table is in that region? You can specify the region using the 'Location' parameter from options.";
}
if (error.httpCode === '403' && error.message.includes('Drive')) {
error.description =
'If your table(s) pull from a document in Google Drive, make sure that document is shared with your user';
}
throw new NodeOperationError(this.getNode(), error as Error, {
itemIndex: i,
description: error.description,
});
}
}
let waitTime = 1000;
outerLoop: while (jobs.length > 0) {
const settledJobs: string[] = [];
for (const job of jobs) {
if (abortSignal?.aborted) {
break outerLoop;
}
try {
const qs: IDataObject = job.location ? { location: job.location } : {};
qs.maxResults = maxResults;
qs.timeoutMs = timeoutMs;
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
const response: IDataObject = await googleBigQueryApiRequestAllItems.call(
this,
'GET',
`/v2/projects/${job.projectId}/queries/${job.jobId}`,
undefined,
qs,
);
if (response.jobComplete) {
settledJobs.push(job.jobId);
returnData.push(...prepareOutput.call(this, response, job.i, job.raw, job.includeSchema));
}
if ((response?.errors as IDataObject[])?.length) {
const errorMessages = (response.errors as IDataObject[]).map((error) => error.message);
throw new ApplicationError(
`Error(s) ocurring while executing query from item ${job.i.toString()}: ${errorMessages.join(
', ',
)}`,
{ level: 'warning' },
);
}
} catch (error) {
if (this.continueOnFail()) {
settledJobs.push(job.jobId);
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: error.message }),
{ itemData: { item: job.i } },
);
returnData.push(...executionErrorData);
continue;
}
throw new NodeOperationError(this.getNode(), error as Error, {
itemIndex: job.i,
description: error.description,
});
}
}
jobs = jobs.filter((job) => !settledJobs.includes(job.jobId));
if (jobs.length > 0) {
await sleep(waitTime);
if (waitTime < 30000) {
waitTime = waitTime * 2;
}
}
}
return returnData;
}
packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/insert.operation.ts
+293
View File
@@ -0,0 +1,293 @@
import type {
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { v4 as uuid } from 'uuid';
import { generatePairedItemData, updateDisplayOptions } from '@utils/utilities';
import type { TableSchema } from '../../helpers/interfaces';
import { checkSchema, wrapData } from '../../helpers/utils';
import { googleBigQueryApiRequest } from '../../transport';
const properties: INodeProperties[] = [
{
displayName: 'Data Mode',
name: 'dataMode',
type: 'options',
options: [
{
name: 'Auto-Map Input Data',
value: 'autoMap',
description: 'Use when node input properties match destination field names',
},
{
name: 'Map Each Field Below',
value: 'define',
description: 'Set the value for each destination field',
},
],
default: 'autoMap',
description: 'Whether to insert the input data this node receives in the new row',
},
{
displayName:
"In this mode, make sure the incoming data fields are named the same as the columns in BigQuery. (Use an 'Edit Fields' node before this node to change them if required.)",
name: 'info',
type: 'notice',
default: '',
displayOptions: {
show: {
dataMode: ['autoMap'],
},
},
},
{
displayName: 'Fields to Send',
name: 'fieldsUi',
placeholder: 'Add Field',
type: 'fixedCollection',
typeOptions: {
multipleValueButtonText: 'Add Field',
multipleValues: true,
},
default: {},
options: [
{
displayName: 'Field',
name: 'values',
values: [
{
displayName: 'Field Name or ID',
name: 'fieldId',
type: 'options',
description:
'Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>',
typeOptions: {
loadOptionsDependsOn: ['projectId.value', 'datasetId.value', 'tableId.value'],
loadOptionsMethod: 'getSchema',
},
default: '',
},
{
displayName: 'Field Value',
name: 'fieldValue',
type: 'string',
default: '',
},
],
},
],
displayOptions: {
show: {
dataMode: ['define'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add option',
default: {},
options: [
{
displayName: 'Batch Size',
name: 'batchSize',
type: 'number',
default: 100,
typeOptions: {
minValue: 1,
},
},
{
displayName: 'Ignore Unknown Values',
name: 'ignoreUnknownValues',
type: 'boolean',
default: false,
description: 'Whether to gnore row values that do not match the schema',
},
{
displayName: 'Skip Invalid Rows',
name: 'skipInvalidRows',
type: 'boolean',
default: false,
description: 'Whether to skip rows with values that do not match the schema',
},
{
displayName: 'Template Suffix',
name: 'templateSuffix',
type: 'string',
default: '',
description:
'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
},
{
displayName: 'Trace ID',
name: 'traceId',
type: 'string',
default: '',
description:
'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
},
],
},
];
const displayOptions = {
show: {
resource: ['database'],
operation: ['insert'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
const projectId = this.getNodeParameter('projectId', 0, undefined, {
extractValue: true,
});
const datasetId = this.getNodeParameter('datasetId', 0, undefined, {
extractValue: true,
});
const tableId = this.getNodeParameter('tableId', 0, undefined, {
extractValue: true,
});
const options = this.getNodeParameter('options', 0);
const dataMode = this.getNodeParameter('dataMode', 0) as string;
let batchSize = 100;
if (options.batchSize) {
batchSize = options.batchSize as number;
delete options.batchSize;
}
const items = this.getInputData();
const length = items.length;
const returnData: INodeExecutionData[] = [];
const rows: IDataObject[] = [];
const body: IDataObject = {};
Object.assign(body, options);
if (body.traceId === undefined) {
body.traceId = uuid();
}
const schema = (
await googleBigQueryApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
{},
)
).schema as TableSchema;
if (schema === undefined) {
throw new NodeOperationError(this.getNode(), 'The destination table has no defined schema');
}
for (let i = 0; i < length; i++) {
try {
const record: IDataObject = {};
if (dataMode === 'autoMap') {
schema.fields.forEach(({ name }) => {
record[name] = items[i].json[name];
});
}
if (dataMode === 'define') {
const fields = this.getNodeParameter('fieldsUi.values', i, []) as IDataObject[];
fields.forEach(({ fieldId, fieldValue }) => {
record[`${fieldId}`] = fieldValue;
});
}
rows.push({ json: checkSchema.call(this, schema, record, i) });
} catch (error) {
if (this.continueOnFail()) {
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: error.message }),
{ itemData: { item: i } },
);
returnData.push(...executionErrorData);
continue;
}
throw new NodeOperationError(this.getNode(), error.message as string, {
itemIndex: i,
description: error?.description,
});
}
}
const itemData = generatePairedItemData(items.length);
for (let i = 0; i < rows.length; i += batchSize) {
const batch = rows.slice(i, i + batchSize);
body.rows = batch;
const responseData = await googleBigQueryApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
if (responseData?.insertErrors && !options.skipInvalidRows) {
const errors: string[] = [];
const failedRows: number[] = [];
const stopedRows: number[] = [];
(responseData.insertErrors as IDataObject[]).forEach((entry) => {
const invalidRows = (entry.errors as IDataObject[]).filter(
(error) => error.reason !== 'stopped',
);
if (invalidRows.length) {
const entryIndex = (entry.index as number) + i;
errors.push(
`Row ${entryIndex} failed with error: ${invalidRows
.map((error) => error.message)
.join(', ')}`,
);
failedRows.push(entryIndex);
} else {
const entryIndex = (entry.index as number) + i;
stopedRows.push(entryIndex);
}
});
if (this.continueOnFail()) {
const executionErrorData = this.helpers.constructExecutionMetaData(
this.helpers.returnJsonArray({ error: errors.join('\n, ') }),
{ itemData: { item: i } },
);
returnData.push(...executionErrorData);
continue;
}
const failedMessage = `Problem inserting item(s) [${failedRows.join(', ')}]`;
const stoppedMessage = stopedRows.length
? `, nothing was inserted item(s) [${stopedRows.join(', ')}]`
: '';
throw new NodeOperationError(this.getNode(), `${failedMessage}${stoppedMessage}`, {
description: errors.join('\n, '),
itemIndex: i,
});
}
const executionData = this.helpers.constructExecutionMetaData(
wrapData(responseData as IDataObject[]),
{ itemData },
);
returnData.push(...executionData);
}
return returnData;
}
packages/nodes-base/nodes/Google/BigQuery/v2/actions/node.type.ts
+9
View File
@@ -0,0 +1,9 @@
import type { AllEntities, Entity } from 'n8n-workflow';
type GoogleBigQueryMap = {
database: 'executeQuery' | 'insert';
};
export type GoogleBigQuery = AllEntities<GoogleBigQueryMap>;
export type GoogleBigQueryDatabase = Entity<GoogleBigQueryMap, 'database'>;
packages/nodes-base/nodes/Google/BigQuery/v2/actions/router.ts
+27
View File
@@ -0,0 +1,27 @@
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import * as record from './database/Database.resource';
import type { GoogleBigQuery } from './node.type';
export async function router(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const resource = this.getNodeParameter<GoogleBigQuery>('resource', 0);
const operation = this.getNodeParameter('operation', 0);
let returnData: INodeExecutionData[] = [];
const googleBigQuery = {
resource,
operation,
} as GoogleBigQuery;
switch (googleBigQuery.resource) {
case 'database':
returnData = await record[googleBigQuery.operation].execute.call(this);
break;
default:
throw new NodeOperationError(this.getNode(), `The resource "${resource}" is not known`);
}
return [returnData];
}
packages/nodes-base/nodes/Google/BigQuery/v2/actions/versionDescription.ts
+73
View File
@@ -0,0 +1,73 @@
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
import { NodeConnectionTypes, type INodeTypeDescription } from 'n8n-workflow';
import * as database from './database/Database.resource';
export const versionDescription: INodeTypeDescription = {
displayName: 'Google BigQuery',
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
version: [2, 2.1],
subtitle: '={{$parameter["operation"]}}',
description: 'Consume Google BigQuery API',
defaults: {
name: 'Google BigQuery',
},
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
credentials: [
{
name: 'googleApi',
required: true,
displayOptions: {
show: {
authentication: ['serviceAccount'],
},
},
},
{
name: 'googleBigQueryOAuth2Api',
required: true,
displayOptions: {
show: {
authentication: ['oAuth2'],
},
},
},
],
properties: [
{
displayName: 'Authentication',
name: 'authentication',
type: 'options',
noDataExpression: true,
options: [
{
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
name: 'OAuth2 (recommended)',
value: 'oAuth2',
},
{
name: 'Service Account',
value: 'serviceAccount',
},
],
default: 'oAuth2',
},
{
displayName: 'Resource',
name: 'resource',
type: 'hidden',
noDataExpression: true,
options: [
{
name: 'Database',
value: 'database',
},
],
default: 'database',
},
...database.description,
],
};
packages/nodes-base/nodes/Google/BigQuery/v2/helpers/interfaces.ts
+31
View File
@@ -0,0 +1,31 @@
import type { IDataObject } from 'n8n-workflow';
export type SchemaField = {
name: string;
type: string;
mode: string;
fields?: SchemaField[];
};
export type TableSchema = {
fields: SchemaField[];
};
export type TableRawData = {
f: Array<{ v: IDataObject | TableRawData }>;
};
export type JobReference = {
projectId: string;
jobId: string;
location: string;
};
export type ResponseWithJobReference = {
kind: string;
id: string;
jobReference: JobReference;
status: {
state: 'PENDING' | 'RUNNING' | 'DONE';
};
};
packages/nodes-base/nodes/Google/BigQuery/v2/helpers/utils.ts
+156
View File
@@ -0,0 +1,156 @@
import { DateTime } from 'luxon';
import type { IDataObject, IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { jsonParse, NodeOperationError } from 'n8n-workflow';
import type { SchemaField, TableRawData, TableSchema } from './interfaces';
function getFieldValue(schemaField: SchemaField, field: IDataObject, parseTimestamps = false) {
if (schemaField.type === 'RECORD') {
return simplify([field.v as TableRawData], schemaField.fields as unknown as SchemaField[]);
} else {
let value = field.v;
if (schemaField.type === 'JSON') {
try {
value = jsonParse(value as string);
} catch (error) {}
} else if (schemaField.type === 'TIMESTAMP' && parseTimestamps) {
const dt = DateTime.fromSeconds(Number(value));
value = dt.isValid ? dt.toISO() : value;
}
return value;
}
}
export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[] {
if (!Array.isArray(data)) {
return [{ json: data }];
}
return data.map((item) => ({
json: item,
}));
}
export function simplify(
data: TableRawData[],
schema: SchemaField[],
includeSchema = false,
parseTimestamps = false,
) {
const returnData: IDataObject[] = [];
for (const entry of data) {
const record: IDataObject = {};
for (const [index, field] of entry.f.entries()) {
if (schema[index].mode !== 'REPEATED') {
record[schema[index].name] = getFieldValue(schema[index], field, parseTimestamps);
} else {
record[schema[index].name] = (field.v as unknown as IDataObject[]).flatMap(
(repeatedField) => {
return getFieldValue(
schema[index],
repeatedField as unknown as IDataObject,
parseTimestamps,
);
},
);
}
}
if (includeSchema) {
record._schema = schema;
}
returnData.push(record);
}
return returnData;
}
export function prepareOutput(
this: IExecuteFunctions,
response: IDataObject,
itemIndex: number,
rawOutput: boolean,
includeSchema = false,
) {
let responseData;
if (response === undefined) return [];
if (rawOutput) {
responseData = response;
} else {
const { rows, schema } = response;
const parseTimestamps = this.getNode().typeVersion >= 2.1;
if (rows !== undefined && schema !== undefined) {
const fields = (schema as TableSchema).fields;
responseData = rows;
responseData = simplify(
responseData as TableRawData[],
fields,
includeSchema,
parseTimestamps,
);
} else if (schema && includeSchema) {
responseData = { success: true, _schema: schema };
} else {
responseData = { success: true };
}
}
const executionData = this.helpers.constructExecutionMetaData(
wrapData(responseData as IDataObject[]),
{
itemData: { item: itemIndex },
},
);
return executionData;
}
export function checkSchema(
this: IExecuteFunctions,
schema: TableSchema,
record: IDataObject,
i: number,
) {
const returnData = { ...record };
schema.fields.forEach(({ name, mode, type, fields }) => {
if (mode === 'REQUIRED' && returnData[name] === undefined) {
throw new NodeOperationError(
this.getNode(),
`The property '${name}' is required, please define it in the 'Fields to Send'`,
{ itemIndex: i },
);
}
if (type !== 'STRING' && returnData[name] === '') {
returnData[name] = null;
}
if (type === 'JSON') {
let value = returnData[name];
if (typeof value === 'object') {
value = JSON.stringify(value);
}
returnData[name] = value;
}
if (type === 'RECORD' && typeof returnData[name] !== 'object') {
let parsedField;
try {
parsedField = jsonParse(returnData[name] as string);
} catch (error) {
const recordField = fields ? `Field Schema:\n ${JSON.stringify(fields)}` : '';
throw new NodeOperationError(
this.getNode(),
`The property '${name}' is a RECORD type, but the value is nor an object nor a valid JSON string`,
{ itemIndex: i, description: recordField },
);
}
returnData[name] = parsedField as IDataObject;
}
});
return returnData;
}
packages/nodes-base/nodes/Google/BigQuery/v2/methods/index.ts
+2
View File
@@ -0,0 +1,2 @@
export * as loadOptions from './loadOptions';
export * as listSearch from './listSearch';
packages/nodes-base/nodes/Google/BigQuery/v2/methods/listSearch.ts
+116
View File
@@ -0,0 +1,116 @@
import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
import { googleBigQueryApiRequest } from '../transport';
export async function searchProjects(
this: ILoadOptionsFunctions,
filter?: string,
paginationToken?: string,
): Promise<INodeListSearchResult> {
const qs = {
pageToken: (paginationToken as string) || undefined,
};
const response = await googleBigQueryApiRequest.call(this, 'GET', '/v2/projects', undefined, qs);
let { projects } = response;
if (filter) {
projects = projects.filter(
(project: IDataObject) =>
(project.friendlyName as string).includes(filter) ||
(project.id as string).includes(filter),
);
}
return {
results: projects.map((project: IDataObject) => ({
name: project.friendlyName as string,
value: project.id,
url: `https://console.cloud.google.com/bigquery?project=${project.id as string}`,
})),
paginationToken: response.nextPageToken,
};
}
export async function searchDatasets(
this: ILoadOptionsFunctions,
filter?: string,
paginationToken?: string,
): Promise<INodeListSearchResult> {
const projectId = this.getNodeParameter('projectId', undefined, {
extractValue: true,
});
const qs = {
pageToken: (paginationToken as string) || undefined,
};
const response = await googleBigQueryApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets`,
undefined,
qs,
);
let { datasets } = response;
if (filter) {
datasets = datasets.filter((dataset: { datasetReference: IDataObject }) =>
(dataset.datasetReference.datasetId as string).includes(filter),
);
}
return {
results: datasets.map((dataset: { datasetReference: IDataObject }) => ({
name: dataset.datasetReference.datasetId as string,
value: dataset.datasetReference.datasetId,
})),
paginationToken: response.nextPageToken,
};
}
export async function searchTables(
this: ILoadOptionsFunctions,
filter?: string,
paginationToken?: string,
): Promise<INodeListSearchResult> {
const projectId = this.getNodeParameter('projectId', undefined, {
extractValue: true,
});
const datasetId = this.getNodeParameter('datasetId', undefined, {
extractValue: true,
});
const qs = {
pageToken: (paginationToken as string) || undefined,
};
const response = await googleBigQueryApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
undefined,
qs,
);
let { tables } = response;
if (filter) {
tables = tables.filter((table: { tableReference: IDataObject }) =>
(table.tableReference.tableId as string).includes(filter),
);
}
const returnData = {
results: tables.map((table: { tableReference: IDataObject }) => ({
name: table.tableReference.tableId as string,
value: table.tableReference.tableId,
})),
paginationToken: response.nextPageToken,
};
return returnData;
}
packages/nodes-base/nodes/Google/BigQuery/v2/methods/loadOptions.ts
+54
View File
@@ -0,0 +1,54 @@
import type { IDataObject, ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
import { googleBigQueryApiRequest } from '../transport';
export async function getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const projectId = this.getNodeParameter('projectId', undefined, {
extractValue: true,
});
const returnData: INodePropertyOptions[] = [];
const { datasets } = await googleBigQueryApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets`,
);
for (const dataset of datasets) {
returnData.push({
name: dataset.datasetReference.datasetId as string,
value: dataset.datasetReference.datasetId,
});
}
return returnData;
}
export async function getSchema(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const projectId = this.getNodeParameter('projectId', undefined, {
extractValue: true,
});
const datasetId = this.getNodeParameter('datasetId', undefined, {
extractValue: true,
});
const tableId = this.getNodeParameter('tableId', undefined, {
extractValue: true,
});
const returnData: INodePropertyOptions[] = [];
const { schema } = await googleBigQueryApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
{},
);
for (const field of schema.fields as IDataObject[]) {
returnData.push({
name: field.name as string,
value: field.name as string,
description:
`type: ${field.type as string}` + (field.mode ? ` mode: ${field.mode as string}` : ''),
});
}
return returnData;
}
packages/nodes-base/nodes/Google/BigQuery/v2/transport/index.ts
+93
View File
@@ -0,0 +1,93 @@
import type {
IDataObject,
IExecuteFunctions,
IHttpRequestMethods,
ILoadOptionsFunctions,
IRequestOptions,
JsonObject,
} from 'n8n-workflow';
import { NodeApiError, NodeOperationError } from 'n8n-workflow';
import { getGoogleAccessToken } from '../../../GenericFunctions';
export async function googleBigQueryApiRequest(
this: IExecuteFunctions | ILoadOptionsFunctions,
method: IHttpRequestMethods,
resource: string,
body: IDataObject = {},
qs: IDataObject = {},
uri?: string,
headers: IDataObject = {},
) {
const authenticationMethod = this.getNodeParameter(
'authentication',
0,
'serviceAccount',
) as string;
const options: IRequestOptions = {
headers: {
'Content-Type': 'application/json',
},
method,
body,
qs,
uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
json: true,
};
try {
if (Object.keys(headers).length !== 0) {
options.headers = Object.assign({}, options.headers, headers);
}
if (Object.keys(body).length === 0) {
delete options.body;
}
if (authenticationMethod === 'serviceAccount') {
const credentials = await this.getCredentials('googleApi');
if (credentials === undefined) {
throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
}
const { access_token } = await getGoogleAccessToken.call(this, credentials, 'bigquery');
options.headers!.Authorization = `Bearer ${access_token}`;
return await this.helpers.request(options);
} else {
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
}
} catch (error) {
if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
error.statusCode = '401';
}
throw new NodeApiError(this.getNode(), error as JsonObject, {
message: error?.error?.error?.message || error.message,
});
}
}
export async function googleBigQueryApiRequestAllItems(
this: IExecuteFunctions | ILoadOptionsFunctions,
method: IHttpRequestMethods,
endpoint: string,
body: IDataObject = {},
query: IDataObject = {},
) {
let rows: IDataObject[] = [];
let responseData;
if (query.maxResults === undefined) {
query.maxResults = 1000;
}
do {
responseData = await googleBigQueryApiRequest.call(this, method, endpoint, body, query);
query.pageToken = responseData.pageToken;
rows = rows.concat((responseData.rows as IDataObject[]) ?? []);
} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
return { ...(responseData || {}), rows };
}