Merge branch 'ThaiHipster-robert_branch'

Author: jhaber-zz

.build.sh

@@ -3,7 +3,7 @@
 python3 -m pip install --user virtualenv
 python3 -m venv .venv
 source .venv/bin/activate
-cd scrapy/schools
+cd schools
 pip install -r requirements.txt
 pip install schools --no-index --find-links .
 

docker-compose.yml

@@ -4,7 +4,7 @@
 version: '3.5'
 services:
   crawler_api:
-    build: scrapy/schools # Build the Scrapy Dockerfile.
+    build: schools # Build the Scrapy Dockerfile.
     depends_on:
       - mongodb_container
     logging:
@@ -38,7 +38,7 @@ services:
       - "7480:6379"
         #    command: rq worker crawling-tasks
   redis_worker:
-    build: scrapy/schools
+    build: schools
     depends_on:
       - mongodb_container
       - redis

requirements.txt schools/requirements.txtrequirements.txt renamed to schools/requirements.txt

@@ -162,6 +162,12 @@ def parse_items(self, response):
         if 'text/html' in str(response.headers['Content-Type']):
             for href in response.xpath('//a/@href').getall():
                 yield Request(response.urljoin(href), self.parse_items)
+            yield from response.follow_all(
+                css="a[href]" \
+                    + ":not([href^='javascript:'])" \
+                    + ":not([href^='tel:'])" \
+                    + ":not([href^='mailto:'])",
+                callback=self.parse_items)
 
     def init_from_school_list(self, school_list):
         """