Skip to content

Commit 2061127

Browse files
committed
chore: add whateverorigin proxy
1 parent 4c0d413 commit 2061127

3 files changed

Lines changed: 24 additions & 1 deletion

File tree

src/app/helpers/proxy.helper.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ export const CORSProxies: Proxy[] = [
55
name: 'allOrigins',
66
url: 'https://api.allorigins.win/raw?url=',
77
},
8+
{
9+
name: 'Whatever Origin',
10+
url: 'http://www.whateverorigin.org/get?url=',
11+
options: {
12+
responseType: 'json',
13+
responseParser: (data) => data.contents,
14+
},
15+
},
816
{
917
name: 'cors-anywhere',
1018
url: 'https://cors-anywhere.herokuapp.com/',

src/app/models/proxy.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ export interface Proxy {
55
[key: string]: string;
66
};
77
options?: {
8+
responseType?: 'text' | 'json';
9+
responseParser?: (response: any) => string;
810
apiKey?: {
911
required: boolean;
1012
hint?: string;

src/app/services/scraper.service.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export class ScraperService {
2929
selector: SelectorList,
3030
filters?: FilterList
3131
): Observable<Episode[]> {
32-
return this.getRawHTML(url).pipe(
32+
return this.getResponse(url).pipe(
3333
map((html: string) => {
3434
const parsedData = this.htmlParser.parse(
3535
html,
@@ -42,6 +42,19 @@ export class ScraperService {
4242
);
4343
}
4444

45+
private getResponse(url: string) {
46+
if (!this.settings.proxy.enabled) {
47+
return this.getRawHTML(url);
48+
}
49+
const observable =
50+
this.proxy.options?.responseType === 'json'
51+
? this.getJSON(url)
52+
: this.getRawHTML(url);
53+
return this.proxy.options?.responseParser
54+
? observable.pipe(map(this.proxy.options.responseParser))
55+
: observable;
56+
}
57+
4558
getRawHTML(url: string, timeout?: number) {
4659
return this.get(this.resolveUrl(url), {
4760
responseType: 'text',

0 commit comments

Comments
 (0)