first commit

packages/nodes-base/nodes/UrlScanIo/GenericFunctions.ts

@@ -0,0 +1,63 @@
+import type { IExecuteFunctions, IDataObject, IRequestOptions } from 'n8n-workflow';
+
+export async function urlScanIoApiRequest(
+	this: IExecuteFunctions,
+	method: 'GET' | 'POST',
+	endpoint: string,
+	body: IDataObject = {},
+	qs: IDataObject = {},
+) {
+	const options: IRequestOptions = {
+		method,
+		body,
+		qs,
+		uri: `https://urlscan.io/api/v1${endpoint}`,
+		json: true,
+	};
+
+	if (!Object.keys(body).length) {
+		delete options.body;
+	}
+
+	if (!Object.keys(qs).length) {
+		delete options.qs;
+	}
+
+	return await this.helpers.requestWithAuthentication.call(this, 'urlScanIoApi', options);
+}
+
+export async function handleListing(
+	this: IExecuteFunctions,
+	endpoint: string,
+	qs: IDataObject = {},
+): Promise<IDataObject[]> {
+	const returnData: IDataObject[] = [];
+	let responseData;
+
+	qs.size = 100;
+
+	const returnAll = this.getNodeParameter('returnAll', 0, false);
+	const limit = this.getNodeParameter('limit', 0, 0);
+
+	do {
+		responseData = await urlScanIoApiRequest.call(this, 'GET', endpoint, {}, qs);
+		returnData.push(...(responseData.results as IDataObject[]));
+
+		if (!returnAll && returnData.length > limit) {
+			return returnData.slice(0, limit);
+		}
+
+		if (responseData.results.length) {
+			const lastResult = responseData.results[responseData.results.length - 1];
+			qs.search_after = lastResult.sort;
+		}
+	} while (responseData.total > returnData.length);
+
+	return returnData;
+}
+
+export const normalizeId = ({ _id, uuid, ...rest }: IDataObject) => {
+	if (_id) return { scanId: _id, ...rest };
+	if (uuid) return { scanId: uuid, ...rest };
+	return rest;
+};

packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.json

@@ -0,0 +1,19 @@
+{
+	"node": "n8n-nodes-base.urlScanIo",
+	"nodeVersion": "1.0",
+	"codexVersion": "1.0",
+	"categories": ["Development", "Utility"],
+	"resources": {
+		"credentialDocumentation": [
+			{
+				"url": "https://docs.n8n.io/integrations/builtin/credentials/urlscanio/"
+			}
+		],
+		"primaryDocumentation": [
+			{
+				"url": "https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.urlscanio/"
+			}
+		]
+	},
+	"alias": ["Scrape"]
+}

packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.ts

@@ -0,0 +1,145 @@
+import type {
+	IExecuteFunctions,
+	IDataObject,
+	INodeExecutionData,
+	INodeType,
+	INodeTypeDescription,
+} from 'n8n-workflow';
+import { NodeConnectionTypes, NodeOperationError } from 'n8n-workflow';
+
+import { scanFields, scanOperations } from './descriptions';
+import { handleListing, normalizeId, urlScanIoApiRequest } from './GenericFunctions';
+
+export class UrlScanIo implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'urlscan.io',
+		name: 'urlScanIo',
+		icon: 'file:urlScanIo.svg',
+		group: ['transform'],
+		version: 1,
+		subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+		description:
+			'Provides various utilities for monitoring websites like health checks or screenshots',
+		defaults: {
+			name: 'urlscan.io',
+		},
+		usableAsTool: true,
+		inputs: [NodeConnectionTypes.Main],
+		outputs: [NodeConnectionTypes.Main],
+		credentials: [
+			{
+				name: 'urlScanIoApi',
+				required: true,
+			},
+		],
+		properties: [
+			{
+				displayName: 'Resource',
+				name: 'resource',
+				noDataExpression: true,
+				type: 'options',
+				options: [
+					{
+						name: 'Scan',
+						value: 'scan',
+					},
+				],
+				default: 'scan',
+			},
+			...scanOperations,
+			...scanFields,
+		],
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+		const returnData: IDataObject[] = [];
+
+		const resource = this.getNodeParameter('resource', 0) as 'scan';
+		const operation = this.getNodeParameter('operation', 0) as 'perform' | 'get' | 'getAll';
+
+		let responseData;
+
+		for (let i = 0; i < items.length; i++) {
+			try {
+				if (resource === 'scan') {
+					// **********************************************************************
+					//                               scan
+					// **********************************************************************
+
+					if (operation === 'get') {
+						// ----------------------------------------
+						//               scan: get
+						// ----------------------------------------
+
+						const scanId = this.getNodeParameter('scanId', i) as string;
+						responseData = await urlScanIoApiRequest.call(this, 'GET', `/result/${scanId}`);
+					} else if (operation === 'getAll') {
+						// ----------------------------------------
+						//             scan: getAll
+						// ----------------------------------------
+
+						// https://urlscan.io/docs/search
+
+						const filters = this.getNodeParameter('filters', i) as { query?: string };
+
+						const qs: IDataObject = {};
+
+						if (filters?.query) {
+							qs.q = filters.query;
+						}
+
+						responseData = await handleListing.call(this, '/search', qs);
+						responseData = responseData.map(normalizeId);
+					} else if (operation === 'perform') {
+						// ----------------------------------------
+						//             scan: perform
+						// ----------------------------------------
+
+						// https://urlscan.io/docs/search
+
+						const { tags: rawTags, ...rest } = this.getNodeParameter('additionalFields', i) as {
+							customAgent?: string;
+							visibility?: 'public' | 'private' | 'unlisted';
+							tags?: string;
+							referer?: string;
+							overrideSafety: string;
+						};
+
+						const body: IDataObject = {
+							url: this.getNodeParameter('url', i) as string,
+							...rest,
+						};
+
+						if (rawTags) {
+							const tags = rawTags.split(',').map((tag) => tag.trim());
+
+							if (tags.length > 10) {
+								throw new NodeOperationError(this.getNode(), 'Please enter at most 10 tags', {
+									itemIndex: i,
+								});
+							}
+
+							body.tags = tags;
+						}
+
+						responseData = await urlScanIoApiRequest.call(this, 'POST', '/scan', body);
+						responseData = normalizeId(responseData as IDataObject);
+					}
+				}
+
+				Array.isArray(responseData)
+					? returnData.push(...(responseData as IDataObject[]))
+					: returnData.push(responseData as IDataObject);
+			} catch (error) {
+				if (this.continueOnFail()) {
+					returnData.push({ error: error.message });
+					continue;
+				}
+				throw error;
+			}
+		}
+
+		return [this.helpers.returnJsonArray(returnData)];
+	}
+}

packages/nodes-base/nodes/UrlScanIo/__schema__/v1.0.0/scan/perform.json

@@ -0,0 +1,24 @@
+{
+    "type": "object",
+    "properties": {
+        "api": {
+            "type": "string"
+        },
+        "message": {
+            "type": "string"
+        },
+        "result": {
+            "type": "string"
+        },
+        "scanId": {
+            "type": "string"
+        },
+        "url": {
+            "type": "string"
+        },
+        "visibility": {
+            "type": "string"
+        }
+    },
+    "version": 1
+}

packages/nodes-base/nodes/UrlScanIo/descriptions/ScanDescription.ts

@@ -0,0 +1,195 @@
+import type { INodeProperties } from 'n8n-workflow';
+
+export const scanOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		noDataExpression: true,
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+			},
+		},
+		options: [
+			{
+				name: 'Get',
+				value: 'get',
+				action: 'Get a scan',
+			},
+			{
+				name: 'Get Many',
+				value: 'getAll',
+				action: 'Get many scans',
+			},
+			{
+				name: 'Perform',
+				value: 'perform',
+				action: 'Perform a scan',
+			},
+		],
+		default: 'perform',
+	},
+];
+
+export const scanFields: INodeProperties[] = [
+	// ----------------------------------------
+	//               scan: get
+	// ----------------------------------------
+	{
+		displayName: 'Scan ID',
+		name: 'scanId',
+		type: 'string',
+		default: '',
+		description: 'ID of the scan to retrieve',
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['get'],
+			},
+		},
+	},
+
+	// ----------------------------------------
+	//             scan: getAll
+	// ----------------------------------------
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		default: false,
+		description: 'Whether to return all results or only up to a given limit',
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['getAll'],
+			},
+		},
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		default: 50,
+		description: 'Max number of results to return',
+		typeOptions: {
+			minValue: 1,
+		},
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['getAll'],
+				returnAll: [false],
+			},
+		},
+	},
+	{
+		displayName: 'Filters',
+		name: 'filters',
+		type: 'collection',
+		placeholder: 'Add Filter',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['getAll'],
+			},
+		},
+		options: [
+			{
+				displayName: 'Query',
+				name: 'query',
+				type: 'string',
+				description:
+					'Query using the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-dsl-query-string-query">Elastic Search Query String syntax</a>. See <a href="https://urlscan.io/docs/search/">supported fields</a> in the documentation.',
+				default: '',
+				placeholder: 'domain:n8n.io',
+			},
+		],
+	},
+
+	// ----------------------------------------
+	//             scan: perform
+	// ----------------------------------------
+	{
+		displayName: 'URL',
+		name: 'url',
+		type: 'string',
+		default: '',
+		placeholder: 'https://n8n.io',
+		description: 'URL to scan',
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['perform'],
+			},
+		},
+	},
+	{
+		displayName: 'Additional Fields',
+		name: 'additionalFields',
+		type: 'collection',
+		placeholder: 'Add Field',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: ['scan'],
+				operation: ['perform'],
+			},
+		},
+		options: [
+			{
+				displayName: 'Custom Agent',
+				name: 'customAgent',
+				description:
+					'<code>User-Agent</code> header to set for this scan. Defaults to <code>n8n</code>',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Override Safety',
+				name: 'overrideSafety',
+				description: 'Disable reclassification of URLs with potential PII in them',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Referer',
+				name: 'referer',
+				description: 'HTTP referer to set for this scan',
+				type: 'string',
+				placeholder: 'https://n8n.io',
+				default: '',
+			},
+			{
+				displayName: 'Tags',
+				name: 'tags',
+				description:
+					'Comma-separated list of user-defined tags to add to this scan. Limited to 10 tags.',
+				placeholder: 'phishing, malicious',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Visibility',
+				name: 'visibility',
+				type: 'options',
+				default: 'private',
+				options: [
+					{
+						name: 'Private',
+						value: 'private',
+					},
+					{
+						name: 'Public',
+						value: 'public',
+					},
+					{
+						name: 'Unlisted',
+						value: 'unlisted',
+					},
+				],
+			},
+		],
+	},
+];

packages/nodes-base/nodes/UrlScanIo/descriptions/index.ts

@@ -0,0 +1 @@
+export * from './ScanDescription';

packages/nodes-base/nodes/UrlScanIo/urlScanIo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="2500" height="2500" viewBox="70 70 884 884"><path fill="#e35946" d="M512 70c244 0 442 198 442 442S756 954 512 954 70 756 70 512 268 70 512 70"/><path fill="#b74837" d="M772 730c10 9 16 22 16 37 0 29-24 53-53 53-15 0-28-6-37-16L548 655c-34 23-76 37-121 37-120 0-218-98-218-218s98-218 218-218 218 98 218 218c0 37-9 72-26 102z"/><path fill="#294658" d="M789 721c0 29-24 53-53 53-15 0-28-6-37-16L504 564c32-18 57-46 70-80l199 200c10 9 16 22 16 37"/><path fill="#26495d" d="M428 272c86 0 156 70 156 156s-70 156-156 156-156-70-156-156 70-156 156-156"/><path fill="#3b637d" d="M428 606c-82 0-148-66-148-148s66-148 148-148 148 66 148 148-66 148-148 148"/><path fill="#9db2c2" d="M403 334c23 0 41 18 41 41s-18 41-41 41-41-18-41-41 18-41 41-41"/><path fill="#e5e9ec" d="M428 646c-120 0-218-98-218-218s98-218 218-218 218 98 218 218-98 218-218 218m0-366c-82 0-148 66-148 148s66 148 148 148 148-66 148-148-66-148-148-148"/></svg>