first commit

packages/nodes-base/nodes/Google/BigQuery/v1/GenericFunctions.ts

@@ -0,0 +1,102 @@
+import type {
+	IExecuteFunctions,
+	ILoadOptionsFunctions,
+	IDataObject,
+	JsonObject,
+	IHttpRequestMethods,
+	IRequestOptions,
+} from 'n8n-workflow';
+import { NodeApiError, NodeOperationError } from 'n8n-workflow';
+
+import { getGoogleAccessToken } from '../../GenericFunctions';
+
+export async function googleApiRequest(
+	this: IExecuteFunctions | ILoadOptionsFunctions,
+	method: IHttpRequestMethods,
+	resource: string,
+	body: any = {},
+	qs: IDataObject = {},
+	uri?: string,
+	headers: IDataObject = {},
+): Promise<any> {
+	const authenticationMethod = this.getNodeParameter(
+		'authentication',
+		0,
+		'serviceAccount',
+	) as string;
+
+	const options: IRequestOptions = {
+		headers: {
+			'Content-Type': 'application/json',
+		},
+		method,
+		body,
+		qs,
+		uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
+		json: true,
+	};
+	try {
+		if (Object.keys(headers).length !== 0) {
+			options.headers = Object.assign({}, options.headers, headers);
+		}
+		if (Object.keys(body as IDataObject).length === 0) {
+			delete options.body;
+		}
+
+		if (authenticationMethod === 'serviceAccount') {
+			const credentials = await this.getCredentials('googleApi');
+
+			if (credentials === undefined) {
+				throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
+			}
+
+			const { access_token } = await getGoogleAccessToken.call(this, credentials, 'bigquery');
+
+			options.headers!.Authorization = `Bearer ${access_token}`;
+			return await this.helpers.request(options);
+		} else {
+			return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
+		}
+	} catch (error) {
+		if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
+			error.statusCode = '401';
+		}
+
+		throw new NodeApiError(this.getNode(), error as JsonObject);
+	}
+}
+
+export async function googleApiRequestAllItems(
+	this: IExecuteFunctions | ILoadOptionsFunctions,
+	propertyName: string,
+	method: IHttpRequestMethods,
+	endpoint: string,
+
+	body: any = {},
+	query: IDataObject = {},
+): Promise<any> {
+	const returnData: IDataObject[] = [];
+
+	let responseData;
+	query.maxResults = 100;
+
+	do {
+		responseData = await googleApiRequest.call(this, method, endpoint, body, query);
+		query.pageToken = responseData.pageToken;
+		returnData.push.apply(returnData, responseData[propertyName] as IDataObject[]);
+	} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
+
+	return returnData;
+}
+
+export function simplify(rows: IDataObject[], fields: string[]) {
+	const results = [];
+	for (const row of rows) {
+		const record: IDataObject = {};
+		for (const [index, field] of fields.entries()) {
+			record[field] = (row.f as IDataObject[])[index].v;
+		}
+		results.push(record);
+	}
+	return results;
+}

packages/nodes-base/nodes/Google/BigQuery/v1/GoogleBigQueryV1.node.ts

@@ -0,0 +1,305 @@
+import type {
+	IDataObject,
+	IExecuteFunctions,
+	ILoadOptionsFunctions,
+	INodeExecutionData,
+	INodePropertyOptions,
+	INodeType,
+	INodeTypeBaseDescription,
+	INodeTypeDescription,
+	JsonObject,
+} from 'n8n-workflow';
+import { NodeConnectionTypes, NodeApiError } from 'n8n-workflow';
+import { v4 as uuid } from 'uuid';
+
+import { oldVersionNotice } from '@utils/descriptions';
+
+import { googleApiRequest, googleApiRequestAllItems, simplify } from './GenericFunctions';
+import { recordFields, recordOperations } from './RecordDescription';
+import { generatePairedItemData } from '../../../../utils/utilities';
+
+const versionDescription: INodeTypeDescription = {
+	displayName: 'Google BigQuery',
+	name: 'googleBigQuery',
+	icon: 'file:googleBigQuery.svg',
+	group: ['input'],
+	version: 1,
+	subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+	description: 'Consume Google BigQuery API',
+	defaults: {
+		name: 'Google BigQuery',
+	},
+	inputs: [NodeConnectionTypes.Main],
+	outputs: [NodeConnectionTypes.Main],
+	credentials: [
+		{
+			name: 'googleApi',
+			required: true,
+			displayOptions: {
+				show: {
+					authentication: ['serviceAccount'],
+				},
+			},
+		},
+		{
+			name: 'googleBigQueryOAuth2Api',
+			required: true,
+			displayOptions: {
+				show: {
+					authentication: ['oAuth2'],
+				},
+			},
+		},
+	],
+	properties: [
+		oldVersionNotice,
+		{
+			displayName: 'Authentication',
+			name: 'authentication',
+			type: 'options',
+			noDataExpression: true,
+			options: [
+				{
+					// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
+					name: 'OAuth2 (recommended)',
+					value: 'oAuth2',
+				},
+				{
+					name: 'Service Account',
+					value: 'serviceAccount',
+				},
+			],
+			default: 'oAuth2',
+		},
+		{
+			displayName: 'Resource',
+			name: 'resource',
+			type: 'options',
+			noDataExpression: true,
+			options: [
+				{
+					name: 'Record',
+					value: 'record',
+				},
+			],
+			default: 'record',
+		},
+		...recordOperations,
+		...recordFields,
+	],
+};
+
+export class GoogleBigQueryV1 implements INodeType {
+	description: INodeTypeDescription;
+
+	constructor(baseDescription: INodeTypeBaseDescription) {
+		this.description = {
+			...baseDescription,
+			...versionDescription,
+		};
+	}
+
+	methods = {
+		loadOptions: {
+			async getProjects(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+				const returnData: INodePropertyOptions[] = [];
+				const { projects } = await googleApiRequest.call(this, 'GET', '/v2/projects');
+				for (const project of projects) {
+					returnData.push({
+						name: project.friendlyName as string,
+						value: project.id,
+					});
+				}
+				return returnData;
+			},
+			async getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+				const projectId = this.getCurrentNodeParameter('projectId');
+				const returnData: INodePropertyOptions[] = [];
+				const { datasets } = await googleApiRequest.call(
+					this,
+					'GET',
+					`/v2/projects/${projectId}/datasets`,
+				);
+				for (const dataset of datasets) {
+					returnData.push({
+						name: dataset.datasetReference.datasetId as string,
+						value: dataset.datasetReference.datasetId,
+					});
+				}
+				return returnData;
+			},
+			async getTables(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+				const projectId = this.getCurrentNodeParameter('projectId');
+				const datasetId = this.getCurrentNodeParameter('datasetId');
+				const returnData: INodePropertyOptions[] = [];
+				const { tables } = await googleApiRequest.call(
+					this,
+					'GET',
+					`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
+				);
+				for (const table of tables) {
+					returnData.push({
+						name: table.tableReference.tableId as string,
+						value: table.tableReference.tableId,
+					});
+				}
+				return returnData;
+			},
+		},
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+		const returnData: INodeExecutionData[] = [];
+		const length = items.length;
+		const qs: IDataObject = {};
+		let responseData;
+		const resource = this.getNodeParameter('resource', 0);
+		const operation = this.getNodeParameter('operation', 0);
+
+		if (resource === 'record') {
+			// *********************************************************************
+			//                               record
+			// *********************************************************************
+
+			if (operation === 'create') {
+				// ----------------------------------
+				//         record: create
+				// ----------------------------------
+
+				// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
+
+				const projectId = this.getNodeParameter('projectId', 0) as string;
+				const datasetId = this.getNodeParameter('datasetId', 0) as string;
+				const tableId = this.getNodeParameter('tableId', 0) as string;
+				const rows: IDataObject[] = [];
+				const body: IDataObject = {};
+
+				for (let i = 0; i < length; i++) {
+					const options = this.getNodeParameter('options', i);
+					Object.assign(body, options);
+					if (body.traceId === undefined) {
+						body.traceId = uuid();
+					}
+					const columns = this.getNodeParameter('columns', i) as string;
+					const columnList = columns.split(',').map((column) => column.trim());
+					const record: IDataObject = {};
+
+					for (const key of Object.keys(items[i].json)) {
+						if (columnList.includes(key)) {
+							record[`${key}`] = items[i].json[key];
+						}
+					}
+					rows.push({ json: record });
+				}
+
+				body.rows = rows;
+
+				const itemData = generatePairedItemData(items.length);
+
+				try {
+					responseData = await googleApiRequest.call(
+						this,
+						'POST',
+						`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
+						body,
+					);
+
+					const executionData = this.helpers.constructExecutionMetaData(
+						this.helpers.returnJsonArray(responseData as IDataObject[]),
+						{ itemData },
+					);
+					returnData.push(...executionData);
+				} catch (error) {
+					if (this.continueOnFail()) {
+						const executionErrorData = this.helpers.constructExecutionMetaData(
+							this.helpers.returnJsonArray({ error: error.message }),
+							{ itemData },
+						);
+						returnData.push(...executionErrorData);
+					}
+					throw new NodeApiError(this.getNode(), error as JsonObject, { itemIndex: 0 });
+				}
+			} else if (operation === 'getAll') {
+				// ----------------------------------
+				//         record: getAll
+				// ----------------------------------
+
+				// https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get
+
+				const returnAll = this.getNodeParameter('returnAll', 0);
+				const projectId = this.getNodeParameter('projectId', 0) as string;
+				const datasetId = this.getNodeParameter('datasetId', 0) as string;
+				const tableId = this.getNodeParameter('tableId', 0) as string;
+				const simple = this.getNodeParameter('simple', 0) as boolean;
+				let fields;
+
+				if (simple) {
+					const { schema } = await googleApiRequest.call(
+						this,
+						'GET',
+						`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
+						{},
+					);
+					fields = (schema.fields || []).map((field: IDataObject) => field.name);
+				}
+
+				for (let i = 0; i < length; i++) {
+					try {
+						const options = this.getNodeParameter('options', i);
+						Object.assign(qs, options);
+
+						if (qs.selectedFields) {
+							fields = (qs.selectedFields as string).split(',');
+						}
+
+						if (returnAll) {
+							responseData = await googleApiRequestAllItems.call(
+								this,
+								'rows',
+								'GET',
+								`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
+								{},
+								qs,
+							);
+						} else {
+							qs.maxResults = this.getNodeParameter('limit', i);
+							responseData = await googleApiRequest.call(
+								this,
+								'GET',
+								`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
+								{},
+								qs,
+							);
+						}
+
+						if (!returnAll) {
+							responseData = responseData.rows;
+						}
+						responseData = simple
+							? simplify(responseData as IDataObject[], fields as string[])
+							: responseData;
+
+						const executionData = this.helpers.constructExecutionMetaData(
+							this.helpers.returnJsonArray(responseData as IDataObject[]),
+							{ itemData: { item: i } },
+						);
+						returnData.push(...executionData);
+					} catch (error) {
+						if (this.continueOnFail()) {
+							const executionErrorData = this.helpers.constructExecutionMetaData(
+								this.helpers.returnJsonArray({ error: error.message }),
+								{ itemData: { item: i } },
+							);
+							returnData.push(...executionErrorData);
+							continue;
+						}
+						throw new NodeApiError(this.getNode(), error as JsonObject, { itemIndex: i });
+					}
+				}
+			}
+		}
+
+		return [returnData];
+	}
+}

packages/nodes-base/nodes/Google/BigQuery/v1/RecordDescription.ts

@@ -0,0 +1,286 @@
+import type { INodeProperties } from 'n8n-workflow';
+
+export const recordOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		noDataExpression: true,
+		displayOptions: {
+			show: {
+				resource: ['record'],
+			},
+		},
+		options: [
+			{
+				name: 'Create',
+				value: 'create',
+				description: 'Create a new record',
+				action: 'Create a record',
+			},
+			{
+				name: 'Get Many',
+				value: 'getAll',
+				description: 'Retrieve many records',
+				action: 'Get many records',
+			},
+		],
+		default: 'create',
+	},
+];
+
+export const recordFields: INodeProperties[] = [
+	// ----------------------------------
+	//         record: create
+	// ----------------------------------
+	{
+		displayName: 'Project Name or ID',
+		name: 'projectId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getProjects',
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['create'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the project to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Dataset Name or ID',
+		name: 'datasetId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getDatasets',
+			loadOptionsDependsOn: ['projectId'],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['create'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the dataset to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Table Name or ID',
+		name: 'tableId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getTables',
+			loadOptionsDependsOn: ['projectId', 'datasetId'],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['create'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the table to create the record in. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Columns',
+		name: 'columns',
+		type: 'string',
+		displayOptions: {
+			show: {
+				resource: ['record'],
+				operation: ['create'],
+			},
+		},
+		default: '',
+		required: true,
+		placeholder: 'id,name,description',
+		description: 'Comma-separated list of the item properties to use as columns',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add option',
+		default: {},
+		displayOptions: {
+			show: {
+				operation: ['create'],
+				resource: ['record'],
+			},
+		},
+		options: [
+			{
+				displayName: 'Ignore Unknown Values',
+				name: 'ignoreUnknownValues',
+				type: 'boolean',
+				default: false,
+				description: 'Whether to ignore row values that do not match the schema',
+			},
+			{
+				displayName: 'Skip Invalid Rows',
+				name: 'skipInvalidRows',
+				type: 'boolean',
+				default: false,
+				description: 'Whether to skip rows with values that do not match the schema',
+			},
+			{
+				displayName: 'Template Suffix',
+				name: 'templateSuffix',
+				type: 'string',
+				default: '',
+				description:
+					'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
+			},
+			{
+				displayName: 'Trace ID',
+				name: 'traceId',
+				type: 'string',
+				default: '',
+				description:
+					'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
+			},
+		],
+	},
+
+	// ----------------------------------
+	//         record: getAll
+	// ----------------------------------
+	{
+		displayName: 'Project Name or ID',
+		name: 'projectId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getProjects',
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the project to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Dataset Name or ID',
+		name: 'datasetId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getDatasets',
+			loadOptionsDependsOn: ['projectId'],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the dataset to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Table Name or ID',
+		name: 'tableId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getTables',
+			loadOptionsDependsOn: ['projectId', 'datasetId'],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+			},
+		},
+		default: '',
+		description:
+			'ID of the table to retrieve all rows from. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+	},
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+			},
+		},
+		default: false,
+		description: 'Whether to return all results or only up to a given limit',
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+				returnAll: [false],
+			},
+		},
+		typeOptions: {
+			minValue: 1,
+			maxValue: 500,
+		},
+		default: 50,
+		description: 'Max number of results to return',
+	},
+	{
+		displayName: 'Simplify',
+		name: 'simple',
+		type: 'boolean',
+		displayOptions: {
+			show: {
+				resource: ['record'],
+				operation: ['getAll'],
+			},
+		},
+		default: true,
+		description: 'Whether to return a simplified version of the response instead of the raw data',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add option',
+		default: {},
+		displayOptions: {
+			show: {
+				operation: ['getAll'],
+				resource: ['record'],
+			},
+		},
+		options: [
+			{
+				displayName: 'Fields',
+				name: 'selectedFields',
+				type: 'string',
+				default: '',
+				description:
+					'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>',
+			},
+			// {
+			// 	displayName: 'Use Int64 Timestamp',
+			// 	name: 'useInt64Timestamp',
+			// 	type: 'boolean',
+			// 	default: false,
+			// 	description: 'Output timestamp as usec int64.',
+			// },
+		],
+	},
+];