Feature/corpus settings

backend/addcorpus/python_corpora/corpus.py

@@ -7,7 +7,6 @@
 from datetime import datetime, date
 from os.path import isdir
 import os
-
 from django.conf import settings
 
 from ianalyzer_readers.readers.core import Reader, Field

backend/addcorpus/python_corpora/load_corpus.py

@@ -17,13 +17,42 @@ def corpus_dir(corpus_name: str) -> str:
     Arguments:
         corpus_name {str} -- Key of the corpus in CORPORA object in settings
     """
-    corpus = load_corpus_definition(corpus_name)
-    return dirname(getabsfile(corpus.__class__))
+    corpus = import_corpus_class(corpus_name)
+    return dirname(getabsfile(corpus))
 
 
-def load_corpus_definition(corpus_name) -> Type[CorpusDefinition]:
+def import_corpus_class(corpus_name: str) -> Type[CorpusDefinition]:
+    '''Imports a corpus definition class'''
     import_path = settings.CORPORA.get(corpus_name)
-    return import_string(import_path)()
+    return import_string(import_path)
+
+
+def corpus_settings(corpus_name: str) -> Dict:
+    '''Attribute overrides for a corpus as configured in settings'''
+    corpora_settings = getattr(settings, 'CORPUS_SETTINGS', {})
+    return corpora_settings.get(corpus_name, dict())
+
+
+def apply_corpus_settings(corpus_class: Type[CorpusDefinition], settings: Dict):
+    '''
+    Creates a subclass of a corpus that overrides parent attributes according to settings.
+
+    Overrides are applied on a class instead of an instance, as this allows you to
+    override `@property` attributes with static values.
+    '''
+    class ConfiguredCorpus(corpus_class):
+        pass
+
+    for attr, value in settings.items():
+        setattr(ConfiguredCorpus, attr, value)
+    return ConfiguredCorpus
+
+
+def load_corpus_definition(corpus_name: str) -> CorpusDefinition:
+    '''Imports, configures and instantiates a corpus definition'''
+    corpus_class = import_corpus_class(corpus_name)
+    configured_class = apply_corpus_settings(corpus_class, corpus_settings(corpus_name))
+    return configured_class()
 
 
 def _try_loading_corpus_definition(corpus_name, stderr=sys.stderr) -> Optional[CorpusDefinition]:

backend/addcorpus/python_corpora/tests/test_corpusimport.py

@@ -36,3 +36,43 @@ def test_corpus_dir(db, settings, basic_mock_corpus):
     assert os.path.isabs(path)
     assert 'corpus.py' in os.listdir(path)
     assert 'source_data' in os.listdir(path)
+
+
+def test_corpus_settings(db, settings):
+    '''Test configuration using CORPUS_SETTINGS'''
+    settings.CORPORA = {
+        'example': 'corpora_test.basic.corpus.ExampleCorpus',
+        'example-2':  'corpora_test.basic.corpus.ExampleCorpus',
+    }
+    settings.CORPUS_SETTINGS = {
+        'example-2': {
+            'title': 'Different title'
+        }
+    }
+    corpora = load_corpus.load_all_corpus_definitions()
+    assert len(corpora) == 2
+    assert corpora['example'].title == 'Example'
+    assert corpora['example-2'].title == 'Different title'
+
+
+def test_corpus_property_override(db, settings):
+    '''
+    Test that CORPUS_SETTINGS also works when the original value is a
+    property without a setter (e.g. in CorpusDefinition itself)
+    '''
+    settings.CORPORA = {
+        'test': 'addcorpus.python_corpora.corpus.CorpusDefinition',
+    }
+    settings.CORPUS_SETTINGS = {
+        'test': {
+            'title': 'Example',
+            'description': 'Description',
+            'min_date': 1900,
+            'max_date': 2000,
+            'category': 'other',
+            'fields': [],
+        }
+    }
+    corpora = load_corpus.load_all_corpus_definitions()
+    assert corpora['test'].title == 'Example'
+

backend/addcorpus/python_corpora/tests/test_times_source.py

@@ -11,9 +11,10 @@ def times_test_settings(settings):
     settings.CORPORA = {
         'times': 'corpora.times.times.Times',
     }
-    settings.TIMES_DATA = join(settings.BASE_DIR, 'addcorpus/python_corpora/tests')
-    settings.TIMES_ES_INDEX = 'test-times'
-
+    settings.CORPUS_SETTINGS["times"] = {
+        "data_directory": join(settings.BASE_DIR, "addcorpus/python_corpora/tests"),
+        "es_index": "test-times",
+    }
 
 
 def test_times_source(times_test_settings):

backend/corpora/dbnl/dbnl.py

@@ -4,7 +4,6 @@
 from tqdm import tqdm
 from ianalyzer_readers.xml_tag import Tag, CurrentTag, TransformTag
 
-from django.conf import settings
 from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
 from ianalyzer_readers.extract import Metadata, XML, Pass, Order, Backup, Combined
 import corpora.dbnl.utils as utils
@@ -15,10 +14,11 @@
 class DBNL(XMLCorpusDefinition):
     title = 'DBNL'
     description = 'Dutch literature and publications on literary or linguistic studies, from the Middle Ages to the 19th century'
-    data_directory = settings.DBNL_DATA
+    es_index = 'dbnl'  # default setting for development
+
     min_date = datetime(year=1200, month=1, day=1)
     max_date = datetime(year=1890, month=12, day=31)
-    es_index = getattr(settings, 'DBNL_ES_INDEX', 'dbnl')
+
     image = 'dbnl.png'
     description_page = 'dbnl.md'
     citation_page = 'citation.md'
@@ -39,7 +39,7 @@ class DBNL(XMLCorpusDefinition):
     language_field = 'language_code'
 
     def sources(self, start = None, end = None):
-        metadata_corpus = DBNLMetadata()
+        metadata_corpus = DBNLMetadata(data_directory=self.data_directory)
         all_metadata = utils.index_by_id(metadata_corpus.documents())
 
         print('Extracting XML files...')
@@ -394,7 +394,7 @@ def _xml_files(self):
     has_content = FieldDefinition(
         name='has_content',
         display_name='Content available',
-        description=f'Whether the contents of this book are available on {settings.SITE_NAME}',
+        description=f'Whether the contents of this book are available on Textcavator',
         extractor=Metadata('has_xml'),
         es_mapping=bool_mapping(),
         search_filter=BooleanFilter(

backend/corpora/dbnl/dbnl_metadata.py

@@ -1,5 +1,4 @@
 import os
-from django.conf import settings
 from addcorpus.python_corpora.corpus import CSVCorpusDefinition, FieldDefinition
 from ianalyzer_readers.extract import CSV, Combined, Pass
 import corpora.dbnl.utils as utils
@@ -10,7 +9,9 @@ class DBNLMetadata(CSVCorpusDefinition):
     Used by the DBNL corpus for CSV extraction utilities -
     not intended as a standalone corpus.'''
 
-    data_directory = settings.DBNL_DATA
+    def __init__(self, data_directory=None):
+        if data_directory:
+            self.data_directory = data_directory
 
     field_entry = 'ti_id'
     delimiter = '|'

backend/corpora/dbnl/tests/test_dbnl_extraction.py

@@ -10,12 +10,20 @@
 
 @pytest.fixture
 def dbnl_corpus(settings):
-    settings.DBNL_DATA = os.path.join(here, 'data')
-    # for testing purposes, also add the metadata helper corpus
+    # for testing purposes, also add the metadata helper corpus - this would normally
+    # be instantiated from the DBNL class
     settings.CORPORA = {
         'dbnl': 'corpora.dbnl.dbnl.DBNL',
         'dbnl_metadata': 'corpora.dbnl.dbnl_metadata.DBNLMetadata',
     }
+    settings.CORPUS_SETTINGS = {
+        'dbnl': {
+            'data_directory': os.path.join(here, 'data'),
+        },
+        'dbnl_metadata': {
+            'data_directory': os.path.join(here, 'data'),
+        }
+    }
     return 'dbnl'
 
 language_name_testcases = [

backend/corpora/dbnl/tests/test_dbnl_validation.py

@@ -4,10 +4,14 @@
 here = os.path.abspath(os.path.dirname(__file__))
 
 def test_dbnl_validation(settings, load_test_corpus):
-    settings.DBNL_DATA = os.path.join(here, 'data')
     settings.CORPORA = {
         'dbnl': 'corpora.dbnl.dbnl.DBNL',
     }
+    settings.CORPUS_SETTINGS = {
+        'dbnl': {
+            'data_directory': os.path.join(here, 'data'),
+        }
+    }
 
     load_test_corpus('dbnl')
 

backend/corpora/dutchannualreports/dutchannualreports.py

@@ -25,13 +25,11 @@ class DutchAnnualReports(XMLCorpusDefinition):
     description = "Annual reports of Dutch financial and non-financial institutes"
     min_date = datetime(year=1957, month=1, day=1)
     max_date = datetime(year=2008, month=12, day=31)
-    data_directory = settings.DUTCHANNUALREPORTS_DATA
-    es_index = getattr(settings, 'DUTCHANNUALREPORTS_ES_INDEX', 'dutchannualreports')
+    es_index = 'dutchannualreports'
     image = 'dutchannualreports.jpg'
-    scan_image_type = getattr(settings, 'DUTCHANNUALREPORTS_SCAN_IMAGE_TYPE', 'application/pdf')
+    scan_image_type = 'application/pdf'
     description_page = 'dutchannualreports.md'
-    allow_image_download = getattr(settings, 'DUTCHANNUALREPORTS_ALLOW_IMAGE_DOWNLOAD', True)
-    word_model_path = getattr(settings, 'DUTCHANNUALREPORTS_WM', None)
+    allow_image_download = True
 
     languages = ['nl']
     category = 'finance'

backend/corpora/dutchnewspapers/dutchnewspapers_all.py

@@ -14,10 +14,8 @@ class DutchNewsPapersAll(DutchNewspapersPublic):
 
     title = "Dutch Newspapers (full)"
     description = "Full collection of Dutch newspapers digitised by the Koninklijke Bibliotheek."
-    data_directory = settings.DUTCHNEWSPAPERS_ALL_DATA
-    es_index = getattr(settings, 'DUTCHNEWSPAPERS_ALL_ES_INDEX', 'dutchnewspapers-all')
+    es_index = 'dutchnewspapers-all'
     max_date = datetime(year=1995, month=12, day=31)
-    word_model_path = getattr(settings, "DUTCHNEWSPAPERS_WM", None)
 
     def update_body(self, doc=None):
         if not doc:

backend/corpora/dutchnewspapers/dutchnewspapers_public.py

@@ -32,14 +32,12 @@ class DutchNewspapersPublic(XMLCorpusDefinition):
     description = "Collection of Dutch newspapers in the public domain, digitised by the Koninklijke Bibliotheek."
     min_date = datetime(year=1600, month=1, day=1)
     max_date = datetime(year=1876, month=12, day=31)
-    data_directory = getattr(settings, 'DUTCHNEWSPAPERS_DATA', None)
-    es_index = getattr(settings, 'DUTCHNEWSPAPERS_ES_INDEX', 'dutchnewspapers-public')
+    es_index = 'dutchnewspapers-public'
     image = 'dutchnewspapers.jpg'
     languages = ['nl']
     category = 'periodical'
     description_page = 'description_public.md'
     citation_page = 'citation_public.md'
-    word_model_path = getattr(settings, "DUTCHNEWSPAPERS_WM", None)
 
     @property
     def es_settings(self):

backend/corpora/ecco/ecco.py

@@ -31,10 +31,9 @@ class Ecco(XMLCorpusDefinition):
     min_date = datetime(year=1700, month=1, day=1)
     max_date = datetime(year=1800, month=12, day=31)
 
-    data_directory = settings.ECCO_DATA
-    es_index = getattr(settings, 'ECCO_ES_INDEX', 'ecco')
+    es_index = 'ecco'
     image = 'ecco.jpg'
-    scan_image_type = getattr(settings, 'ECCO_SCAN_IMAGE_TYPE', 'application/pdf')
+    scan_image_type = 'application/pdf'
     es_settings = None
     languages = ['en', 'fr', 'la', 'grc', 'de',  'it', 'cy', 'ga', 'gd']
     category = 'book'

backend/corpora/economist/economist.py

@@ -54,8 +54,7 @@ class Economist(GaleCorpus):
     description = "Archive of The Economist, a weekly news magazine."
     min_date = datetime(1843, 8, 1)
     max_date = datetime(2021, 1, 1)
-    data_directory = settings.ECONOMIST_DATA
-    es_index = getattr(settings, 'ECONOMIST_ES_INDEX', 'economist')
+    es_index = 'economist'
     image = 'A_stack_of_Economist_papers.jpg'
     description_page = 'Economist.md'
     languages = ['en']

backend/corpora/gallica/caricature.py

@@ -12,7 +12,7 @@ class Caricature(Gallica):
     max_date = datetime(year=1843, month=12, day=31)
     corpus_id = "cb344523348"
     category = "periodical"
-    es_index = getattr(settings, 'CARICATURE_INDEX', 'caricature')
+    es_index = 'caricature'
     description_page = 'caricature.md'
     image = "caricature.jpg"
 

backend/corpora/gallica/figaro.py

@@ -12,7 +12,7 @@ class Figaro(Gallica):
     max_date = datetime(year=1954, month=12, day=31)
     corpus_id = "cb34355551z"
     category = "periodical"
-    es_index = getattr(settings, 'FIGARO_INDEX', 'figaro')
+    es_index = 'figaro'
     image = "figaro.jpg"
     description_page = 'figaro.md'
 

backend/corpora/gallica/resistance.py

@@ -81,7 +81,7 @@ class JournauxResistance(Gallica):
         "cb32853253x",
     ]
     category = "periodical"
-    es_index = getattr(settings, 'RESISTANCE_INDEX', 'resistance')
+    es_index = 'resistance'
     image = "resistance.png"
     description_page = "resistance.md"
 

backend/corpora/goodreads/goodreads.py

@@ -1,11 +1,9 @@
 import re
 import os
 import os.path as op
-import glob
 import logging
 from datetime import datetime
 
-from django.conf import settings
 import openpyxl
 
 from ianalyzer_readers.extract import CSV, Metadata
@@ -22,13 +20,13 @@ class GoodReads(CSVCorpusDefinition):
     # Data overrides from .common.Corpus (fields at bottom of class)
     title = "DIOPTRA-L"
     description = "Goodreads reviews of translated literary texts"
+    es_index = 'goodreads'  # default setting for development
 
     delimiter = ';'
 
     min_date=datetime(2007, 1, 1)
     max_date=datetime(2022, 12, 31)
-    data_directory = settings.GOODREADS_DATA
-    es_index = getattr(settings, 'GOODREADS_ES_INDEX', 'goodreads')
+
     image = 'DioptraL.png'
     description_page = 'goodreads.md'
     visualize = []

backend/corpora/goodreads/tests/test_goodreads.py

@@ -7,7 +7,11 @@ def test_goodreads(settings, db, admin_client):
     settings.CORPORA = {
         'goodreads': 'corpora.goodreads.goodreads.GoodReads'
     }
-    settings.GOODREADS_DATA = ''
+    settings.CORPUS_SETTINGS = {
+        'goodreads': {
+            'data_directory': ''
+        }
+    }
 
     corpus = corpus_from_api(admin_client, 'goodreads')
     assert corpus['title'] == 'DIOPTRA-L'

backend/corpora/guardianobserver/guardianobserver.py

@@ -3,6 +3,7 @@
 separate xml-files, zipped.
 '''
 
+from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
 import logging
 logger = logging.getLogger(__name__)
 import re
@@ -12,14 +13,12 @@
 from zipfile import ZipFile
 from io import BytesIO
 
-from django.conf import settings
 from ianalyzer_readers.xml_tag import Tag
 from ianalyzer_readers import extract
 
 from api.utils import find_media_file
 from indexing.run_update_task import update_document
 from addcorpus.python_corpora import filters
-from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
 from media.image_processing import sizeof_fmt
 from media.media_url import media_url
 
@@ -37,13 +36,14 @@ class GuardianObserver(XMLCorpusDefinition):
     description_page = 'guardianobserver.md'
     min_date = datetime(year=1791, month=1, day=1)
     max_date = datetime(year=2003, month=12, day=31)
-    data_directory = settings.GO_DATA
-    es_index = getattr(settings, 'GO_ES_INDEX', 'guardianobserver')
+    scan_image_type = 'application/pdf'
+    es_index = 'guardianobserver'
+
     image = 'guardianobserver.jpg'
-    scan_image_type = getattr(settings, 'GO_SCAN_IMAGE_TYPE', 'application/pdf')
+
+
     languages = ['en']
     category = 'periodical'
-    word_model_path = getattr(settings, "GO_WM", None)
 
     @property
     def es_settings(self):

backend/corpora/guardianobserver/test_guardianobserver.py

@@ -7,7 +7,11 @@ def test_guardian_observer(settings, db, admin_client):
     settings.CORPORA = {
         'guardian-observer': 'corpora.guardianobserver.guardianobserver.GuardianObserver',
     }
-    settings.GO_DATA = ''
+    settings.CORPUS_SETTINGS = {
+        'guardian-observer': {
+            'data_directory': ''
+        }
+    }
 
     corpus = corpus_from_api(admin_client, 'guardian-observer')
     assert corpus['title'] == 'Guardian-Observer'