File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -5,6 +5,14 @@ export const CORSProxies: Proxy[] = [
55 name : 'allOrigins' ,
66 url : 'https://api.allorigins.win/raw?url=' ,
77 } ,
8+ {
9+ name : 'Whatever Origin' ,
10+ url : 'http://www.whateverorigin.org/get?url=' ,
11+ options : {
12+ responseType : 'json' ,
13+ responseParser : ( data ) => data . contents ,
14+ } ,
15+ } ,
816 {
917 name : 'cors-anywhere' ,
1018 url : 'https://cors-anywhere.herokuapp.com/' ,
Original file line number Diff line number Diff line change @@ -5,6 +5,8 @@ export interface Proxy {
55 [ key : string ] : string ;
66 } ;
77 options ?: {
8+ responseType ?: 'text' | 'json' ;
9+ responseParser ?: ( response : any ) => string ;
810 apiKey ?: {
911 required : boolean ;
1012 hint ?: string ;
Original file line number Diff line number Diff line change @@ -29,7 +29,7 @@ export class ScraperService {
2929 selector : SelectorList ,
3030 filters ?: FilterList
3131 ) : Observable < Episode [ ] > {
32- return this . getRawHTML ( url ) . pipe (
32+ return this . getResponse ( url ) . pipe (
3333 map ( ( html : string ) => {
3434 const parsedData = this . htmlParser . parse (
3535 html ,
@@ -42,6 +42,19 @@ export class ScraperService {
4242 ) ;
4343 }
4444
45+ private getResponse ( url : string ) {
46+ if ( ! this . settings . proxy . enabled ) {
47+ return this . getRawHTML ( url ) ;
48+ }
49+ const observable =
50+ this . proxy . options ?. responseType === 'json'
51+ ? this . getJSON ( url )
52+ : this . getRawHTML ( url ) ;
53+ return this . proxy . options ?. responseParser
54+ ? observable . pipe ( map ( this . proxy . options . responseParser ) )
55+ : observable ;
56+ }
57+
4558 getRawHTML ( url : string , timeout ?: number ) {
4659 return this . get ( this . resolveUrl ( url ) , {
4760 responseType : 'text' ,
You can’t perform that action at this time.
0 commit comments