Skip to content

Commit c0028b8

Browse files
Karek2001Karek2001
committed
Replaced basic logging with n8n's official logging methods | Switched from standard error throwing to n8n error handling | Improved item processing | Enhanced logging for URL processing and expression evaluation | Removed affiliate Link from credentials
1 parent aa1be03 commit c0028b8

10 files changed

Lines changed: 206 additions & 151 deletions

File tree

‎credentials/ScrappeyApi.credentials.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,6 @@ export class ScrappeyApi implements ICredentialType {
66
icon = 'file:Scrappey.svg' as const;
77
documentationUrl = 'https://wiki.scrappey.com';
88
properties: INodeProperties[] = [
9-
{
10-
displayName:
11-
'Get 750 Direct (bot-bypass) & 150 GUI Browser requests free—sign up now 👉 <a href="https://nodes.n8n.community/scrappey/signup" target="_blank">Start scraping</a>',
12-
name: 'affiliateMessage',
13-
type: 'notice',
14-
default: '',
15-
},
16-
179
{
1810
displayName: 'API Key',
1911
name: 'apiKey',

‎nodes/Scrappey/GenericFunctions.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ export async function genericHttpRequest<T>(
240240
},
241241
};
242242

243-
// Special handling for proxy error with code-0007
243+
244244
if (
245245
(error.message && error.message.includes('ERR_TUNNEL_CONNECTION_FAILED')) ||
246246
error.message.includes('ERR_EMPTY_RESPONSE')

‎nodes/Scrappey/Scrappey.node.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"node": "n8n-nodes-scrappey",
3-
"nodeVersion": "0.3.11",
4-
"codexVersion": "0.3.11",
3+
"nodeVersion": "0.3.12",
4+
"codexVersion": "0.3.12",
55
"categories": [
66
"Development",
77
"Web Scraping"

‎nodes/Scrappey/Scrappey.node.ts‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,73 @@ import { INodeType, INodeTypeDescription } from 'n8n-workflow';
22
import { AdvancedSettingsForBrowser, publicFields } from './fields';
33
import { executeScrappey } from './execute';
44
import { scrappeyOperators } from './operators';
5-
import { IExecuteFunctions, INodeExecutionData, IDataObject } from 'n8n-workflow';
5+
import { IExecuteFunctions, INodeExecutionData, IDataObject, NodeOperationError } from 'n8n-workflow';
6+
67
export class Scrappey implements INodeType {
78
public async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
9+
// Get all input items - this is crucial for item linking
10+
const items = this.getInputData();
11+
const returnData: INodeExecutionData[] = [];
12+
813
const operation = this.getNodeParameter('scrappeyOperations', 0) as string;
9-
const responseData = await executeScrappey.call(this, operation);
10-
11-
if (
12-
Array.isArray(responseData) &&
13-
responseData.length > 0 &&
14-
responseData[0].hasOwnProperty('json')
15-
) {
16-
return [responseData as INodeExecutionData[]];
14+
15+
// Process each input item individually to maintain item relationships
16+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
17+
try {
18+
// This ensures that expressions like {{ $json.field }} work correctly
19+
const responseData = await executeScrappey.call(this, operation, itemIndex);
20+
21+
if (Array.isArray(responseData)) {
22+
if (responseData.length > 0 && responseData[0].hasOwnProperty('json')) {
23+
responseData.forEach((item) => {
24+
returnData.push({
25+
json: item.json,
26+
pairedItem: { item: itemIndex }
27+
});
28+
});
29+
} else {
30+
responseData.forEach((item) => {
31+
returnData.push({
32+
json: item as IDataObject,
33+
pairedItem: { item: itemIndex }
34+
});
35+
});
36+
}
37+
} else {
38+
returnData.push({
39+
json: responseData as IDataObject,
40+
pairedItem: { item: itemIndex }
41+
});
42+
}
43+
} catch (error) {
44+
// This allows n8n to track which input item caused the error
45+
if (this.continueOnFail()) {
46+
returnData.push({
47+
json: {
48+
error: error.message,
49+
// Include the original input data so it's not lost
50+
originalInput: items[itemIndex].json
51+
},
52+
pairedItem: { item: itemIndex },
53+
error: new NodeOperationError(this.getNode(), error as Error)
54+
});
55+
} else {
56+
throw new NodeOperationError(
57+
this.getNode(),
58+
error as Error,
59+
{
60+
message: `Failed to process item ${itemIndex}`,
61+
description: `Error occurred while processing input item at index ${itemIndex}`,
62+
itemIndex
63+
}
64+
);
65+
}
66+
}
1767
}
18-
return [[{ json: responseData as unknown as IDataObject }]];
68+
69+
return [returnData];
1970
}
71+
2072
description: INodeTypeDescription = {
2173
displayName: 'Scrappey',
2274
name: 'scrappey',
@@ -47,4 +99,4 @@ export class Scrappey implements INodeType {
4799
},
48100
properties: [...scrappeyOperators, ...publicFields, ...AdvancedSettingsForBrowser],
49101
};
50-
}
102+
}

‎nodes/Scrappey/execute.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
1-
import { AutoRetryTypeBrowser, PostRequest, AutoRetryTypeRequest } from './RequestMethods';
2-
import { IExecuteFunctions } from 'n8n-workflow';
1+
import { AutoRetryTypeBrowser, PostRequest, AutoRetryTypeRequest } from './methods';
2+
import { IExecuteFunctions, NodeOperationError } from 'n8n-workflow';
33

4-
export async function executeScrappey(this: IExecuteFunctions, operation: string) {
4+
export async function executeScrappey(this: IExecuteFunctions, operation: string, itemIndex: number = 0) {
55
switch (operation) {
66
case 'requestBuilder':
7-
return await PostRequest.call(this);
7+
return await PostRequest.call(this, itemIndex);
88
case 'httpRequestAutoRetry':
9-
return await AutoRetryTypeRequest.call(this);
9+
return await AutoRetryTypeRequest.call(this, itemIndex);
1010
case 'httpRequestAutoRetryBrowser':
11-
return await AutoRetryTypeBrowser.call(this);
11+
return await AutoRetryTypeBrowser.call(this, itemIndex);
1212
default:
13-
throw new Error(`Operation ${operation} is not supported`);
13+
throw new NodeOperationError(this.getNode(), `Operation "${operation}" is not supported`, {
14+
description: 'Please select a valid operation from the available options.',
15+
itemIndex // item index in error for better debugging
16+
});
1417
}
15-
}
18+
}

‎nodes/Scrappey/fields.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export const publicFields: INodeProperties[] = [
1818
},
1919
},
2020
},
21+
2122
{
2223
displayName: 'URL',
2324
name: 'url',

nodes/Scrappey/RequestMethods.ts renamed to nodes/Scrappey/methods.ts

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,19 @@ import { handleBody, HTTPRequest_Extract_Parameters } from './requestBodyBuilder
33
import type { ScrappeyRequestBody } from './types';
44
import { genericHttpRequest } from './GenericFunctions';
55

6-
export const PostRequest = async function (this: IExecuteFunctions) {
7-
const body = await handleBody(this);
6+
export const PostRequest = async function (this: IExecuteFunctions, itemIndex: number = 0) {
7+
const body = await handleBody(this, itemIndex);
88
const response = await genericHttpRequest.call(this, 'POST', '', { body });
99
return response;
1010
};
1111

12-
export const AutoRetryTypeBrowser = async function (this: IExecuteFunctions) {
13-
const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this);
14-
const proxyType = this.getNodeParameter('proxyType', 0, '') as string;
12+
export const AutoRetryTypeBrowser = async function (this: IExecuteFunctions, itemIndex: number = 0) {
13+
const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this, itemIndex);
14+
15+
const proxyType = this.getNodeParameter('proxyType', itemIndex, '') as string;
1516
const whichProxyToUse = this.getNodeParameter(
1617
'whichProxyToUse',
17-
0,
18+
itemIndex,
1819
'proxyFromCredentials',
1920
) as string;
2021

@@ -33,20 +34,18 @@ export const AutoRetryTypeBrowser = async function (this: IExecuteFunctions) {
3334

3435
// Handle proxy settings based on the selected proxy source
3536
if (whichProxyToUse === 'proxyFromScrappey') {
36-
// Apply proxy type
3737
if (proxyType && proxyType.trim() !== '') {
3838
body[proxyType] = true;
3939
}
4040

41-
// Check if custom proxy country is enabled
4241
const customProxyCountryBoolean = this.getNodeParameter(
4342
'customProxyCountryBoolean',
44-
0,
43+
itemIndex,
4544
false,
4645
) as boolean;
4746

4847
if (customProxyCountryBoolean) {
49-
const customProxyCountry = this.getNodeParameter('customProxyCountry', 0, '') as string;
48+
const customProxyCountry = this.getNodeParameter('customProxyCountry', itemIndex, '') as string;
5049
if (customProxyCountry && customProxyCountry.trim() !== '') {
5150
body.country = customProxyCountry;
5251
}
@@ -59,53 +58,65 @@ export const AutoRetryTypeBrowser = async function (this: IExecuteFunctions) {
5958
if (prev_HTTPRequest.processedPostData) {
6059
body.postData = prev_HTTPRequest.processedPostData;
6160

62-
if (prev_HTTPRequest.contentType) {
61+
if (prev_HTTPRequest.contentType && body.customHeaders) {
6362
body.customHeaders['content-type'] = prev_HTTPRequest.contentType;
6463
}
6564
}
65+
6666
const response = await genericHttpRequest.call(this, 'POST', '', { body });
6767
return response;
6868
};
6969

70-
export const AutoRetryTypeRequest = async function (this: IExecuteFunctions) {
71-
const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this);
72-
const customProxyCountry = this.getNodeParameter('customProxyCountry', 0, '') as string;
70+
export const AutoRetryTypeRequest = async function (this: IExecuteFunctions, itemIndex: number = 0) {
71+
const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this, itemIndex);
72+
73+
const customProxyCountry = this.getNodeParameter('customProxyCountry', itemIndex, '') as string;
7374
const customProxyCountryBoolean = this.getNodeParameter(
7475
'customProxyCountryBoolean',
75-
0,
76+
itemIndex,
7677
false,
7778
) as boolean;
78-
const proxyType = this.getNodeParameter('proxyType', 0, '') as string;
79+
const proxyType = this.getNodeParameter('proxyType', itemIndex, '') as string;
7980

8081
let body: ScrappeyRequestBody = {
8182
cmd: prev_HTTPRequest.cmd,
8283
url: prev_HTTPRequest.url as string,
84+
requestType: 'request', // Add this to ensure it's a request type
8385
};
8486

8587
if (prev_HTTPRequest.processedHeaders) {
8688
body.customHeaders = prev_HTTPRequest.processedHeaders;
8789
}
8890

89-
if (prev_HTTPRequest.processedProxy) {
91+
const whichProxyToUse = this.getNodeParameter(
92+
'whichProxyToUse',
93+
itemIndex,
94+
'proxyFromCredentials',
95+
) as string;
96+
97+
if (whichProxyToUse === 'proxyFromNode' && prev_HTTPRequest.processedProxy) {
9098
body.proxy = prev_HTTPRequest.processedProxy;
91-
} else {
99+
} else if (whichProxyToUse === 'proxyFromScrappey') {
92100
if (customProxyCountryBoolean) {
93101
body.proxyCountry = customProxyCountry;
94102
}
95-
}
96-
97-
if (proxyType && proxyType.trim() !== '') {
98-
body[proxyType] = true;
103+
104+
if (proxyType && proxyType.trim() !== '') {
105+
body[proxyType] = true;
106+
}
99107
}
100108

101109
if (prev_HTTPRequest.processedPostData) {
102110
body.postData = prev_HTTPRequest.processedPostData;
103111

104112
if (prev_HTTPRequest.contentType) {
113+
if (!body.customHeaders) {
114+
body.customHeaders = {};
115+
}
105116
body.customHeaders['content-type'] = prev_HTTPRequest.contentType;
106117
}
107118
}
119+
108120
const response = await genericHttpRequest.call(this, 'POST', '', { body });
109121
return response;
110-
// return { body };
111-
};
122+
};

0 commit comments

Comments
 (0)