|
| 1 | +from Requester import Requester |
| 2 | +from bs4 import BeautifulSoup |
| 3 | +from FileHandler import write, read |
| 4 | +from DataHandlers import get_unique |
| 5 | + |
| 6 | +class HtmlScraper: |
| 7 | + |
| 8 | + def __init__(self, url:str, setSessions:bool=False, set_header:bool=True, set_agent:bool=True, set_proxy:bool=False)->None: |
| 9 | + """ |
| 10 | + HtmlScraper |
| 11 | + =========== |
| 12 | +
|
| 13 | + This is a class that is to be used to scrape a website. |
| 14 | + """ |
| 15 | + self.url = url |
| 16 | + self.setSessions, self.sessions = setSessions, None |
| 17 | + self.req = Requester(set_header=set_header, set_agent=set_agent, set_proxy=set_proxy, agent_file='F:/Code Works/Python_works/storage/others/user-agent.txt') # proxy_file='F:/Code Works/Python_works/storage/others/proxies.txt') |
| 18 | + self.souped = None |
| 19 | + |
| 20 | + def _rectiftyPathway(self, pathway): |
| 21 | + |
| 22 | + if isinstance(pathway, list): |
| 23 | + pathway = [self._rectiftyPathway(p) for p in pathway] |
| 24 | + elif isinstance(pathway, dict): |
| 25 | + r = pathway.keys() |
| 26 | + if 'tag' in r: |
| 27 | + if 'attr' not in r: |
| 28 | + pathway['attr'] = {} |
| 29 | + if 'type' not in r: |
| 30 | + pathway['type'] = 'find' |
| 31 | + else: |
| 32 | + pathway = {k: self._rectiftyPathway(v) for k,v in pathway.items()} |
| 33 | + return pathway |
| 34 | + |
| 35 | + def _request(self, method:str='get', params:dict={}, ref:str='', response_code:int=200): |
| 36 | + reqVals = {'url': self.url, 'method': method,'params': params, 'ref': ref, 'response_code': response_code} |
| 37 | + req = None |
| 38 | + if self.setSessions: |
| 39 | + req, self.sessions = self.req.requestSessions(sessions=self.sessions, **reqVals) |
| 40 | + else: |
| 41 | + req = self.req.request(**reqVals) |
| 42 | + return req |
| 43 | + |
| 44 | + def _souper(self, data, parser:str='html.parser'): |
| 45 | + """ |
| 46 | + _souper() |
| 47 | + --------- |
| 48 | +
|
| 49 | + Converts a html document data into BeautifulSoup class value. |
| 50 | + """ |
| 51 | + self.souped = BeautifulSoup(data, parser) |
| 52 | + return self.souped |
| 53 | + |
| 54 | + def _getAtr(self, data, ty): |
| 55 | + if isinstance(data, list): |
| 56 | + return [self._getAtr(t, ty) for t in data if t is not None] |
| 57 | + else: |
| 58 | + r = None |
| 59 | + if ty=='<text>' or ty=='<stext>': |
| 60 | + r = data.getText(strip=(True if ty=='<stext>' else False)) |
| 61 | + else: |
| 62 | + r = data.get(ty) |
| 63 | + return r |
| 64 | + |
| 65 | + def _parser(self, selectorType:str='find', tagName='', attribute:dict={}, data=None)->(list|str|None): |
| 66 | + """ |
| 67 | + _parser() |
| 68 | + --------- |
| 69 | + This method is responsible for fetching the target element in the document. |
| 70 | +
|
| 71 | + Parameters: |
| 72 | + - `selectorType` str: The type of method that is to be used for fetching an element(s).Its values are |
| 73 | + - find: Finds a single element, the first element that matches the values (tagName & attribute). Also the default value. |
| 74 | + - find_all: Finds all the element of the same tag and atribute value. |
| 75 | + - select_one: Similar to find, the tagName used would be the JS query selector value. Selecting this value returns a single value. |
| 76 | + - select: Similar to find_all, the tagName used would be the JS query selector value. Selecting this value returns a list of value. |
| 77 | + - `tagName` str: This parameter determines where to lookand what to fetch. Depending of on the `selectorType`, the value can be a tagName(for `find` & `find_all`) or a JS querySelctor value (for `select` or `select_one`) |
| 78 | + - `attribbute` dict: This acts as a supporter for finding the target tag value. |
| 79 | + - `data`: This parameter is to pass the html data where to look. Default is `None` which means, the page that will be parsed will be the page got during the requesting of the page. |
| 80 | + |
| 81 | + Returns: |
| 82 | + - NoneType|list|str: Depending on the value passsed in `selectorType` parameter, the data type passed can be a list, str or a None type value. |
| 83 | + - `select_one` or `find`: Return str |
| 84 | + - `select` or `find_all`: Return list |
| 85 | + - If no data found: Return None |
| 86 | + """ |
| 87 | + if data==None: |
| 88 | + data=self.souped |
| 89 | + try: |
| 90 | + if selectorType == 'select': |
| 91 | + k = data.select(tagName, attr=attribute) |
| 92 | + elif selectorType == 'select_one': |
| 93 | + k = data.select_one(tagName, attr=attribute) |
| 94 | + elif selectorType == 'find': |
| 95 | + k = data.find(tagName, attr=attribute) |
| 96 | + elif selectorType == 'findall' or selectorType =='find_all': |
| 97 | + k = data.find_all(tagName, attr=attribute) |
| 98 | + return k |
| 99 | + except: |
| 100 | + return None |
| 101 | + |
| 102 | + # basic purpose |
| 103 | + def storePage(self, fileName:str, data=None, seperator:str='\n', prevEmpty:bool=True)->None: |
| 104 | + """ |
| 105 | + storePage() |
| 106 | + ----------- |
| 107 | + This method is to store the page or the data in a file. |
| 108 | +
|
| 109 | + Parameter: |
| 110 | + - fileName str: Name of the file. |
| 111 | + - data any: Takes the data that is to be inserted in the file. Default is `None`, which means the data stored will be the html data fetched during the request processes. |
| 112 | + - seperator str: THis paramerter specifies, how the data points will be seperated in the file. Default is `\n` (a line break). |
| 113 | + - prevEmpty bool: This parameter specifies if the existing data in the file should remain or be deleted. Default is `True`. Values:- |
| 114 | + - `True`: The file will be emptied before inserting new data. |
| 115 | + - `False`: The new data will be appended into the file with existing data. |
| 116 | + """ |
| 117 | + write(file_name=fileName, data=(self.souped if data==None else data), separator=seperator, emptyPervious=prevEmpty) |
| 118 | + |
| 119 | + # User use functions |
| 120 | + def getAllUrls(self, data=None)->list[str]: |
| 121 | + """ |
| 122 | + getAllUrls() |
| 123 | + ------------ |
| 124 | + This method is fo getting all the urls in the parsed page |
| 125 | + """ |
| 126 | + return get_unique([i.get('href') for i in self._parser(selectorType='find_all',tagName='a', data=data)]) |
| 127 | + |
| 128 | + def getAllImages(self, data=None): |
| 129 | + """ |
| 130 | + getAllImages() |
| 131 | + -------------- |
| 132 | + Returns all the images in the page. |
| 133 | + """ |
| 134 | + imgs = {'tag': 'img', 'attr': {}, 'type': 'select', 'inner':{'imgLnk': 'src', 'alt':'alt'}} |
| 135 | + return self.jsonParser(pathway=imgs, data=data) |
| 136 | + |
| 137 | + def getPageMeta(self, data=None): |
| 138 | + """ |
| 139 | + getPageMeta() |
| 140 | + ------------- |
| 141 | + Fetches the meta data of the page. |
| 142 | + """ |
| 143 | + pathway = {'title': {'tag':'title', 'get': '<stext>'}} |
| 144 | + return self.jsonParser(pathway, data) |
| 145 | + |
| 146 | + def jsonParser(self, pathway:dict, data=None)->dict|None|list: |
| 147 | + """ |
| 148 | + jsonParser() |
| 149 | + ------------ |
| 150 | + This method is responsible for parsing the websitein the given structure. |
| 151 | + """ |
| 152 | + if data==None: |
| 153 | + data = self._souper(self._request()) if self.souped==None else self.souped |
| 154 | + |
| 155 | + pathway = self._rectiftyPathway(pathway) |
| 156 | + try: |
| 157 | + if isinstance(pathway, str): |
| 158 | + return self._parser(selectorType='select', tag=pathway, data=data) |
| 159 | + elif isinstance(pathway, dict): |
| 160 | + ret:dict = {} |
| 161 | + if len(pathway) == 0: return None |
| 162 | + if 'tag' in pathway.keys(): |
| 163 | + k = self._parser(selectorType=pathway['type'], tagName=pathway['tag'], attribute=pathway['attr'], data=data) |
| 164 | + if k is not None: |
| 165 | + if 'get' in pathway.keys() and pathway['get'] != '' and pathway['get'] is not None: |
| 166 | + ret['get'] = self._getAtr(ty=pathway['get'], data=k) |
| 167 | + |
| 168 | + if 'inner' in pathway.keys() and pathway['inner']!={}: |
| 169 | + ret['inner'] = [self.jsonParser(data=n,pathway=pathway['inner']) for n in k] if isinstance(k, list) else self.jsonParser(data=k, pathway=pathway['inner']) |
| 170 | + elif 'get' in ret.keys(): |
| 171 | + return ret['get'] |
| 172 | + else: return k |
| 173 | + else: |
| 174 | + for k,v in pathway.items(): |
| 175 | + ret[k] = self._getAtr(ty=v, data=data) if isinstance(v, str) else self.jsonParser(pathway=v,data=data) |
| 176 | + |
| 177 | + return ret |
| 178 | + elif isinstance(pathway, list): |
| 179 | + return [self.jsonParser(pathway=i,data=data) for i in pathway] |
| 180 | + except: |
| 181 | + return None |
| 182 | + |
0 commit comments