Skip to content

Commit 5c1cc47

Browse files
KristjanESPERANTOKristjanESPERANTO
authored
[newsfeed] refactor: migrate to centralized HTTPFetcher (#4023)
This migrates the Newsfeed module to use the centralized HTTPFetcher class (introduced in #4016), following the same pattern as the Calendar module. This continues the refactoring effort to centralize HTTP error handling across all modules. ## Changes **NewsfeedFetcher:** - Refactored from function constructor to ES6 class (like the calendar module in #3959) - Replaced manual fetch() + timer handling with HTTPFetcher composition - Uses structured error objects with translation keys - Inherits smart retry strategies (401/403, 429, 5xx backoff) - Inherits timeout handling (30s) and AbortController **node_helper.js:** - Updated error handler to use `errorInfo.translationKey` - Simplified property access (`fetcher.url`, `fetcher.items`) **Cleanup:** - Removed `js/module_functions.js` (`scheduleTimer` no longer needed) - Removed `#module_functions` import from package.json ## Related Part of the HTTPFetcher migration effort started in #4016. Next candidate: Weather module (client-side → server-side migration).
1 parent 2b55b8e commit 5c1cc47

File tree

4 files changed

+108
-139
lines changed

4 files changed

+108
-139
lines changed

‎defaultmodules/newsfeed/newsfeedfetcher.js‎

Lines changed: 102 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -4,41 +4,71 @@ const FeedMe = require("feedme");
44
const iconv = require("iconv-lite");
55
const { htmlToText } = require("html-to-text");
66
const Log = require("logger");
7-
const NodeHelper = require("node_helper");
8-
const { getUserAgent } = require("#server_functions");
9-
const { scheduleTimer } = require("#module_functions");
7+
const HTTPFetcher = require("#http_fetcher");
108

119
/**
12-
* Responsible for requesting an update on the set interval and broadcasting the data.
13-
* @param {string} url URL of the news feed.
14-
* @param {number} reloadInterval Reload interval in milliseconds.
15-
* @param {string} encoding Encoding of the feed.
16-
* @param {boolean} logFeedWarnings If true log warnings when there is an error parsing a news article.
17-
* @param {boolean} useCorsProxy If true cors proxy is used for article url's.
10+
* NewsfeedFetcher - Fetches and parses RSS/Atom feed data
11+
* Uses HTTPFetcher for HTTP handling with intelligent error handling
1812
* @class
1913
*/
20-
const NewsfeedFetcher = function (url, reloadInterval, encoding, logFeedWarnings, useCorsProxy) {
21-
let reloadTimer = null;
22-
let items = [];
23-
let reloadIntervalMS = reloadInterval;
14+
class NewsfeedFetcher {
2415

25-
let fetchFailedCallback = function () {};
26-
let itemsReceivedCallback = function () {};
16+
/**
17+
* Creates a new NewsfeedFetcher instance
18+
* @param {string} url - The URL of the news feed to fetch
19+
* @param {number} reloadInterval - Time in ms between fetches
20+
* @param {string} encoding - Encoding of the feed (e.g., 'UTF-8')
21+
* @param {boolean} logFeedWarnings - If true log warnings when there is an error parsing a news article
22+
* @param {boolean} useCorsProxy - If true cors proxy is used for article url's
23+
*/
24+
constructor (url, reloadInterval, encoding, logFeedWarnings, useCorsProxy) {
25+
this.url = url;
26+
this.encoding = encoding;
27+
this.logFeedWarnings = logFeedWarnings;
28+
this.useCorsProxy = useCorsProxy;
29+
this.items = [];
30+
this.fetchFailedCallback = () => {};
31+
this.itemsReceivedCallback = () => {};
32+
33+
// Use HTTPFetcher for HTTP handling (Composition)
34+
this.httpFetcher = new HTTPFetcher(url, {
35+
reloadInterval: Math.max(reloadInterval, 1000),
36+
headers: {
37+
"Cache-Control": "max-age=0, no-cache, no-store, must-revalidate",
38+
Pragma: "no-cache"
39+
}
40+
});
2741

28-
if (reloadIntervalMS < 1000) {
29-
reloadIntervalMS = 1000;
42+
// Wire up HTTPFetcher events
43+
this.httpFetcher.on("response", (response) => this.#handleResponse(response));
44+
this.httpFetcher.on("error", (errorInfo) => this.fetchFailedCallback(this, errorInfo));
3045
}
3146

32-
/* private methods */
33-
3447
/**
35-
* Request the new items.
48+
* Creates a parse error info object
49+
* @param {string} message - Error message
50+
* @param {Error} error - Original error
51+
* @returns {object} Error info object
3652
*/
37-
const fetchNews = () => {
38-
clearTimeout(reloadTimer);
39-
reloadTimer = null;
40-
items = [];
53+
#createParseError (message, error) {
54+
return {
55+
message,
56+
status: null,
57+
errorType: "PARSE_ERROR",
58+
translationKey: "MODULE_ERROR_UNSPECIFIED",
59+
retryAfter: this.httpFetcher.reloadInterval,
60+
retryCount: 0,
61+
url: this.url,
62+
originalError: error
63+
};
64+
}
4165

66+
/**
67+
* Handles successful HTTP response
68+
* @param {Response} response - The fetch Response object
69+
*/
70+
#handleResponse (response) {
71+
this.items = [];
4272
const parser = new FeedMe();
4373

4474
parser.on("item", (item) => {
@@ -58,118 +88,80 @@ const NewsfeedFetcher = function (url, reloadInterval, encoding, logFeedWarnings
5888
]
5989
});
6090

61-
items.push({
62-
title: title,
63-
description: description,
64-
pubdate: pubdate,
65-
url: url,
66-
useCorsProxy: useCorsProxy,
91+
this.items.push({
92+
title,
93+
description,
94+
pubdate,
95+
url,
96+
useCorsProxy: this.useCorsProxy,
6797
hash: crypto.createHash("sha256").update(`${pubdate} :: ${title} :: ${url}`).digest("hex")
6898
});
69-
} else if (logFeedWarnings) {
99+
} else if (this.logFeedWarnings) {
70100
Log.warn("Can't parse feed item:", item);
71101
Log.warn(`Title: ${title}`);
72102
Log.warn(`Description: ${description}`);
73103
Log.warn(`Pubdate: ${pubdate}`);
74104
}
75105
});
76106

77-
parser.on("end", () => {
78-
this.broadcastItems();
79-
});
107+
parser.on("end", () => this.broadcastItems());
80108

81109
parser.on("error", (error) => {
82-
fetchFailedCallback(this, error);
83-
scheduleTimer(reloadTimer, reloadIntervalMS, fetchNews);
84-
});
85-
86-
//"end" event is not broadcast if the feed is empty but "finish" is used for both
87-
parser.on("finish", () => {
88-
scheduleTimer(reloadTimer, reloadIntervalMS, fetchNews);
110+
Log.error(`${this.url} - Feed parsing failed: ${error.message}`);
111+
this.fetchFailedCallback(this, this.#createParseError(`Feed parsing failed: ${error.message}`, error));
89112
});
90113

91114
parser.on("ttl", (minutes) => {
92-
try {
93-
// 86400000 = 24 hours is mentioned in the docs as maximum value:
94-
const ttlms = Math.min(minutes * 60 * 1000, 86400000);
95-
if (ttlms > reloadIntervalMS) {
96-
reloadIntervalMS = ttlms;
97-
Log.info(`reloadInterval set to ttl=${reloadIntervalMS} for url ${url}`);
98-
}
99-
} catch (error) {
100-
Log.warn(`feed ttl is no valid integer=${minutes} for url ${url}`);
115+
const ttlms = Math.min(minutes * 60 * 1000, 86400000);
116+
if (ttlms > this.httpFetcher.reloadInterval) {
117+
this.httpFetcher.reloadInterval = ttlms;
118+
Log.info(`reloadInterval set to ttl=${ttlms} for url ${this.url}`);
101119
}
102120
});
103121

104-
const headers = {
105-
"User-Agent": getUserAgent(),
106-
"Cache-Control": "max-age=0, no-cache, no-store, must-revalidate",
107-
Pragma: "no-cache"
108-
};
109-
110-
fetch(url, { headers: headers })
111-
.then(NodeHelper.checkFetchStatus)
112-
.then((response) => {
113-
let nodeStream;
114-
if (response.body instanceof stream.Readable) {
115-
nodeStream = response.body;
116-
} else {
117-
nodeStream = stream.Readable.fromWeb(response.body);
118-
}
119-
nodeStream.pipe(iconv.decodeStream(encoding)).pipe(parser);
120-
})
121-
.catch((error) => {
122-
fetchFailedCallback(this, error);
123-
scheduleTimer(reloadTimer, reloadIntervalMS, fetchNews);
124-
});
125-
};
126-
127-
/* public methods */
122+
try {
123+
const nodeStream = response.body instanceof stream.Readable
124+
? response.body
125+
: stream.Readable.fromWeb(response.body);
126+
nodeStream.pipe(iconv.decodeStream(this.encoding)).pipe(parser);
127+
} catch (error) {
128+
Log.error(`${this.url} - Stream processing failed: ${error.message}`);
129+
this.fetchFailedCallback(this, this.#createParseError(`Stream processing failed: ${error.message}`, error));
130+
}
131+
}
128132

129133
/**
130134
* Update the reload interval, but only if we need to increase the speed.
131-
* @param {number} interval Interval for the update in milliseconds.
135+
* @param {number} interval - Interval for the update in milliseconds.
132136
*/
133-
this.setReloadInterval = function (interval) {
134-
if (interval > 1000 && interval < reloadIntervalMS) {
135-
reloadIntervalMS = interval;
137+
setReloadInterval (interval) {
138+
if (interval > 1000 && interval < this.httpFetcher.reloadInterval) {
139+
this.httpFetcher.reloadInterval = interval;
136140
}
137-
};
141+
}
138142

139-
/**
140-
* Initiate fetchNews();
141-
*/
142-
this.startFetch = function () {
143-
fetchNews();
144-
};
143+
startFetch () {
144+
this.httpFetcher.startPeriodicFetch();
145+
}
145146

146-
/**
147-
* Broadcast the existing items.
148-
*/
149-
this.broadcastItems = function () {
150-
if (items.length <= 0) {
147+
broadcastItems () {
148+
if (this.items.length <= 0) {
151149
Log.info("No items to broadcast yet.");
152150
return;
153151
}
154-
Log.info(`Broadcasting ${items.length} items.`);
155-
itemsReceivedCallback(this);
156-
};
157-
158-
this.onReceive = function (callback) {
159-
itemsReceivedCallback = callback;
160-
};
161-
162-
this.onError = function (callback) {
163-
fetchFailedCallback = callback;
164-
};
165-
166-
this.url = function () {
167-
return url;
168-
};
169-
170-
this.items = function () {
171-
return items;
172-
};
173-
};
152+
Log.info(`Broadcasting ${this.items.length} items.`);
153+
this.itemsReceivedCallback(this);
154+
}
155+
156+
/** @param {function(NewsfeedFetcher): void} callback - Called when items are received */
157+
onReceive (callback) {
158+
this.itemsReceivedCallback = callback;
159+
}
160+
161+
/** @param {function(NewsfeedFetcher, object): void} callback - Called on fetch error */
162+
onError (callback) {
163+
this.fetchFailedCallback = callback;
164+
}
165+
}
174166

175167
module.exports = NewsfeedFetcher;

‎defaultmodules/newsfeed/node_helper.js‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ module.exports = NodeHelper.create({
2626
const url = feed.url || "";
2727
const encoding = feed.encoding || "UTF-8";
2828
const reloadInterval = feed.reloadInterval || config.reloadInterval || 5 * 60 * 1000;
29-
let useCorsProxy = feed.useCorsProxy;
30-
if (useCorsProxy === undefined) useCorsProxy = true;
29+
const useCorsProxy = feed.useCorsProxy ?? true;
3130

3231
try {
3332
new URL(url);
@@ -46,11 +45,10 @@ module.exports = NodeHelper.create({
4645
this.broadcastFeeds();
4746
});
4847

49-
fetcher.onError((fetcher, error) => {
50-
Log.error("Error: Could not fetch newsfeed: ", url, error);
51-
let error_type = NodeHelper.checkFetchError(error);
48+
fetcher.onError((fetcher, errorInfo) => {
49+
Log.error("Error: Could not fetch newsfeed: ", fetcher.url, errorInfo.message || errorInfo);
5250
this.sendSocketNotification("NEWSFEED_ERROR", {
53-
error_type
51+
error_type: errorInfo.translationKey
5452
});
5553
});
5654

@@ -71,8 +69,8 @@ module.exports = NodeHelper.create({
7169
*/
7270
broadcastFeeds () {
7371
const feeds = {};
74-
for (let f in this.fetchers) {
75-
feeds[f] = this.fetchers[f].items();
72+
for (const url in this.fetchers) {
73+
feeds[url] = this.fetchers[url].items;
7674
}
7775
this.sendSocketNotification("NEWS_ITEMS", feeds);
7876
}

‎js/module_functions.js‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎package.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@
2727
],
2828
"type": "commonjs",
2929
"imports": {
30-
"#module_functions": {
31-
"default": "./js/module_functions.js"
32-
},
3330
"#server_functions": {
3431
"default": "./js/server_functions.js"
3532
},

0 commit comments

Comments
 (0)