Enable harvester in datagov

Author: akariv

.docker-compose.datagov-theme.yaml

@@ -5,6 +5,11 @@ services:
     image: viderum/ckan-cloud-docker:ckan-latest-datagov-theme
     build:
       args:
+        EXTRA_PACKAGES: supervisor cron
+        EXTRA_FILESYSTEM: "./overrides/datagov/filesystem/"
+        ROOT_INIT: |
+          mkdir -p /var/log/supervisor
+          chown ckan:ckan /var/log/supervisor
         POST_INSTALL: |
           install_standard_ckan_extension_github ckan/ckanext-spatial ckanext-spatial &&\
           install_standard_ckan_extension_github ckan/ckanext-xloader ckanext-xloader &&\
@@ -16,12 +21,15 @@ services:
           install_standard_ckan_extension_github GSA/ckanext-geodatagov ckanext-geodatagov &&\
           install_standard_ckan_extension_github ViderumGlobal/USMetadata ckanext-usmetadata
         POST_DOCKER_BUILD: |
-          mkdir -p /var/tmp/ckan/dynamic_menu
+          mkdir -p /var/tmp/ckan/dynamic_menu &&\
+          mkdir -p /var/log/ckan/std/
         CKAN_INIT: |
           ckan-paster --plugin=ckanext-archiver archiver init  -c "CKAN_CONFIG/production.ini"
           ckan-paster --plugin=ckanext-report report initdb  -c "CKAN_CONFIG/production.ini"
           ckan-paster --plugin=ckanext-harvest harvester initdb  -c "CKAN_CONFIG/production.ini"
           cp /var/lib/ckan/main.css /usr/lib/ckan/venv/src/ckan/ckan/public/base/css/main.min.css || true
+          crontab /etc/crontab-harvester
+          service supervisor start
     environment:
     - CKAN_CONFIG_TEMPLATE_PREFIX=datagov-theme-
   jobs:

ckan/Dockerfile

@@ -2,6 +2,8 @@
 
 FROM debian:jessie
 
+ARG EXTRA_PACKAGES
+
 # Install required system packages
 RUN apt-get -q -y update \
     && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \
@@ -22,27 +24,30 @@ RUN apt-get -q -y update \
         vim \
         wget \
         redis-tools \
+        ${EXTRA_PACKAGES} \
     && apt-get -q clean \
     && rm -rf /var/lib/apt/lists/*
 
+
 # Define environment variables
 ENV CKAN_HOME /usr/lib/ckan
 ENV CKAN_VENV $CKAN_HOME/venv
 ENV CKAN_CONFIG /etc/ckan
 ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV CKAN_LOGS_PATH=/var/log/ckan
 
 # Create ckan user
 RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan
 
 # Setup virtual environment for CKAN
-RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $CKAN_LOGS_PATH && \
     virtualenv $CKAN_VENV && \
     ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\
     ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster
 
 # Setup CKAN
 RUN ckan-pip install -U pip &&\
-    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $CKAN_LOGS_PATH
 
 USER ckan
 
@@ -78,6 +83,18 @@ RUN . /post_install_functions.sh && eval "${POST_DOCKER_BUILD}"
 ARG CKAN_INIT
 RUN echo "${CKAN_INIT}" | sed s@CKAN_CONFIG@${CKAN_CONFIG}@g > ${CKAN_CONFIG}/ckan_extra_init.sh
 
+USER root
+
+# Extra files in the filesystem
+ARG EXTRA_FILESYSTEM
+COPY ${EXTRA_FILESYSTEM} /
+
+# Initialization that should be done as root
+ARG ROOT_INIT
+RUN eval "${ROOT_INIT}"
+
+USER ckan
+
 ENTRYPOINT ["/ckan-entrypoint.sh"]
 
 EXPOSE 5000

ckan/overrides/datagov/filesystem/etc/crontab-harvester

@@ -0,0 +1,3 @@
+# m  h  dom mon dow   command
+*/15 *  *   *   *     /usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester run               --config=/etc/ckan/production.ini
+  0  5  *   *   *     /usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester clean_harvest_log --config=/etc/ckan/production.ini

ckan/overrides/datagov/filesystem/etc/supervisor/conf.d/ckan_harvesting.conf

@@ -0,0 +1,30 @@
+; ===============================
+; ckan harvester
+; ===============================
+[program:ckan_gather_consumer]
+
+command=/usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester gather_consumer --config=/etc/ckan/production.ini
+
+; user that owns virtual environment.
+user=ckan
+
+numprocs=1
+stdout_logfile=/var/log/ckan/std/gather_consumer.log
+stderr_logfile=/var/log/ckan/std/gather_consumer.log
+autostart=true
+autorestart=true
+startsecs=10
+
+[program:ckan_fetch_consumer]
+
+command=/usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester fetch_consumer --config=/etc/ckan/production.ini
+
+; user that owns virtual environment.
+user=ckan
+
+numprocs=1
+stdout_logfile=/var/log/ckan/std/fetch_consumer.log
+stderr_logfile=/var/log/ckan/std/fetch_consumer.log
+autostart=true
+autorestart=true
+startsecs=10

ckan/overrides/datagov/filesystem/etc/supervisor/supervisord.conf

@@ -0,0 +1,30 @@
+; supervisor config file
+
+[unix_http_server]
+file=/var/tmp/supervisord.sock
+chmod=0770
+chown=ckan:ckan
+
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/tmp/supervisor.sock
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = /etc/supervisor/conf.d/*.conf