Merge pull request #2103 from CentreForDigitalHumanities/feature/corpus-settings

Feature/corpus settings

Author: JeltevanBoheemen

backend/addcorpus/python_corpora/corpus.py

@@ -7,7 +7,6 @@
 from datetime import datetime, date
 from os.path import isdir
 import os
-
 from django.conf import settings
 
 from ianalyzer_readers.readers.core import Reader, Field

backend/addcorpus/python_corpora/load_corpus.py

@@ -17,13 +17,42 @@ def corpus_dir(corpus_name: str) -> str:
     Arguments:
         corpus_name {str} -- Key of the corpus in CORPORA object in settings
     """
-    corpus = load_corpus_definition(corpus_name)
-    return dirname(getabsfile(corpus.__class__))
+    corpus = import_corpus_class(corpus_name)
+    return dirname(getabsfile(corpus))
 
 
-def load_corpus_definition(corpus_name) -> Type[CorpusDefinition]:
+def import_corpus_class(corpus_name: str) -> Type[CorpusDefinition]:
+    '''Imports a corpus definition class'''
     import_path = settings.CORPORA.get(corpus_name)
-    return import_string(import_path)()
+    return import_string(import_path)
+
+
+def corpus_settings(corpus_name: str) -> Dict:
+    '''Attribute overrides for a corpus as configured in settings'''
+    corpora_settings = getattr(settings, 'CORPUS_SETTINGS', {})
+    return corpora_settings.get(corpus_name, dict())
+
+
+def apply_corpus_settings(corpus_class: Type[CorpusDefinition], settings: Dict):
+    '''
+    Creates a subclass of a corpus that overrides parent attributes according to settings.
+
+    Overrides are applied on a class instead of an instance, as this allows you to
+    override `@property` attributes with static values.
+    '''
+    class ConfiguredCorpus(corpus_class):
+        pass
+
+    for attr, value in settings.items():
+        setattr(ConfiguredCorpus, attr, value)
+    return ConfiguredCorpus
+
+
+def load_corpus_definition(corpus_name: str) -> CorpusDefinition:
+    '''Imports, configures and instantiates a corpus definition'''
+    corpus_class = import_corpus_class(corpus_name)
+    configured_class = apply_corpus_settings(corpus_class, corpus_settings(corpus_name))
+    return configured_class()
 
 
 def _try_loading_corpus_definition(corpus_name, stderr=sys.stderr) -> Optional[CorpusDefinition]:

backend/addcorpus/python_corpora/tests/test_corpusimport.py

@@ -36,3 +36,43 @@ def test_corpus_dir(db, settings, basic_mock_corpus):
     assert os.path.isabs(path)
     assert 'corpus.py' in os.listdir(path)
     assert 'source_data' in os.listdir(path)
+
+
+def test_corpus_settings(db, settings):
+    '''Test configuration using CORPUS_SETTINGS'''
+    settings.CORPORA = {
+        'example': 'corpora_test.basic.corpus.ExampleCorpus',
+        'example-2':  'corpora_test.basic.corpus.ExampleCorpus',
+    }
+    settings.CORPUS_SETTINGS = {
+        'example-2': {
+            'title': 'Different title'
+        }
+    }
+    corpora = load_corpus.load_all_corpus_definitions()
+    assert len(corpora) == 2
+    assert corpora['example'].title == 'Example'
+    assert corpora['example-2'].title == 'Different title'
+
+
+def test_corpus_property_override(db, settings):
+    '''
+    Test that CORPUS_SETTINGS also works when the original value is a
+    property without a setter (e.g. in CorpusDefinition itself)
+    '''
+    settings.CORPORA = {
+        'test': 'addcorpus.python_corpora.corpus.CorpusDefinition',
+    }
+    settings.CORPUS_SETTINGS = {
+        'test': {
+            'title': 'Example',
+            'description': 'Description',
+            'min_date': 1900,
+            'max_date': 2000,
+            'category': 'other',
+            'fields': [],
+        }
+    }
+    corpora = load_corpus.load_all_corpus_definitions()
+    assert corpora['test'].title == 'Example'
+

backend/addcorpus/python_corpora/tests/test_times_source.py

@@ -11,9 +11,10 @@ def times_test_settings(settings):
     settings.CORPORA = {
         'times': 'corpora.times.times.Times',
     }
-    settings.TIMES_DATA = join(settings.BASE_DIR, 'addcorpus/python_corpora/tests')
-    settings.TIMES_ES_INDEX = 'test-times'
-
+    settings.CORPUS_SETTINGS["times"] = {
+        "data_directory": join(settings.BASE_DIR, "addcorpus/python_corpora/tests"),
+        "es_index": "test-times",
+    }
 
 
 def test_times_source(times_test_settings):

backend/corpora/dbnl/dbnl.py

@@ -4,7 +4,6 @@
 from tqdm import tqdm
 from ianalyzer_readers.xml_tag import Tag, CurrentTag, TransformTag
 
-from django.conf import settings
 from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
 from ianalyzer_readers.extract import Metadata, XML, Pass, Order, Backup, Combined
 import corpora.dbnl.utils as utils
@@ -15,10 +14,11 @@
 class DBNL(XMLCorpusDefinition):
     title = 'DBNL'
     description = 'Dutch literature and publications on literary or linguistic studies, from the Middle Ages to the 19th century'
-    data_directory = settings.DBNL_DATA
+    es_index = 'dbnl'  # default setting for development
+
     min_date = datetime(year=1200, month=1, day=1)
     max_date = datetime(year=1890, month=12, day=31)
-    es_index = getattr(settings, 'DBNL_ES_INDEX', 'dbnl')
+
     image = 'dbnl.png'
     description_page = 'dbnl.md'
     citation_page = 'citation.md'
@@ -39,7 +39,7 @@ class DBNL(XMLCorpusDefinition):
     language_field = 'language_code'
 
     def sources(self, start = None, end = None):
-        metadata_corpus = DBNLMetadata()
+        metadata_corpus = DBNLMetadata(data_directory=self.data_directory)
         all_metadata = utils.index_by_id(metadata_corpus.documents())
 
         print('Extracting XML files...')
@@ -394,7 +394,7 @@ def _xml_files(self):
     has_content = FieldDefinition(
         name='has_content',
         display_name='Content available',
-        description=f'Whether the contents of this book are available on {settings.SITE_NAME}',
+        description=f'Whether the contents of this book are available on Textcavator',
         extractor=Metadata('has_xml'),
         es_mapping=bool_mapping(),
         search_filter=BooleanFilter(

backend/corpora/dbnl/dbnl_metadata.py

@@ -1,5 +1,4 @@
 import os
-from django.conf import settings
 from addcorpus.python_corpora.corpus import CSVCorpusDefinition, FieldDefinition
 from ianalyzer_readers.extract import CSV, Combined, Pass
 import corpora.dbnl.utils as utils
@@ -10,7 +9,9 @@ class DBNLMetadata(CSVCorpusDefinition):
     Used by the DBNL corpus for CSV extraction utilities -
     not intended as a standalone corpus.'''
 
-    data_directory = settings.DBNL_DATA
+    def __init__(self, data_directory=None):
+        if data_directory:
+            self.data_directory = data_directory
 
     field_entry = 'ti_id'
     delimiter = '|'

backend/corpora/dbnl/tests/test_dbnl_extraction.py

@@ -10,12 +10,20 @@
 
 @pytest.fixture
 def dbnl_corpus(settings):
-    settings.DBNL_DATA = os.path.join(here, 'data')
-    # for testing purposes, also add the metadata helper corpus
+    # for testing purposes, also add the metadata helper corpus - this would normally
+    # be instantiated from the DBNL class
     settings.CORPORA = {
         'dbnl': 'corpora.dbnl.dbnl.DBNL',
         'dbnl_metadata': 'corpora.dbnl.dbnl_metadata.DBNLMetadata',
     }
+    settings.CORPUS_SETTINGS = {
+        'dbnl': {
+            'data_directory': os.path.join(here, 'data'),
+        },
+        'dbnl_metadata': {
+            'data_directory': os.path.join(here, 'data'),
+        }
+    }
     return 'dbnl'
 
 language_name_testcases = [

backend/corpora/dbnl/tests/test_dbnl_validation.py

@@ -4,10 +4,14 @@
 here = os.path.abspath(os.path.dirname(__file__))
 
 def test_dbnl_validation(settings, load_test_corpus):
-    settings.DBNL_DATA = os.path.join(here, 'data')
     settings.CORPORA = {
         'dbnl': 'corpora.dbnl.dbnl.DBNL',
     }
+    settings.CORPUS_SETTINGS = {
+        'dbnl': {
+            'data_directory': os.path.join(here, 'data'),
+        }
+    }
 
     load_test_corpus('dbnl')
 

backend/corpora/dutchannualreports/dutchannualreports.py

@@ -25,13 +25,11 @@ class DutchAnnualReports(XMLCorpusDefinition):
     description = "Annual reports of Dutch financial and non-financial institutes"
     min_date = datetime(year=1957, month=1, day=1)
     max_date = datetime(year=2008, month=12, day=31)
-    data_directory = settings.DUTCHANNUALREPORTS_DATA
-    es_index = getattr(settings, 'DUTCHANNUALREPORTS_ES_INDEX', 'dutchannualreports')
+    es_index = 'dutchannualreports'
     image = 'dutchannualreports.jpg'
-    scan_image_type = getattr(settings, 'DUTCHANNUALREPORTS_SCAN_IMAGE_TYPE', 'application/pdf')
+    scan_image_type = 'application/pdf'
     description_page = 'dutchannualreports.md'
-    allow_image_download = getattr(settings, 'DUTCHANNUALREPORTS_ALLOW_IMAGE_DOWNLOAD', True)
-    word_model_path = getattr(settings, 'DUTCHANNUALREPORTS_WM', None)
+    allow_image_download = True
 
     languages = ['nl']
     category = 'finance'

backend/corpora/dutchnewspapers/dutchnewspapers_all.py

@@ -14,10 +14,8 @@ class DutchNewsPapersAll(DutchNewspapersPublic):
 
     title = "Dutch Newspapers (full)"
     description = "Full collection of Dutch newspapers digitised by the Koninklijke Bibliotheek."
-    data_directory = settings.DUTCHNEWSPAPERS_ALL_DATA
-    es_index = getattr(settings, 'DUTCHNEWSPAPERS_ALL_ES_INDEX', 'dutchnewspapers-all')
+    es_index = 'dutchnewspapers-all'
     max_date = datetime(year=1995, month=12, day=31)
-    word_model_path = getattr(settings, "DUTCHNEWSPAPERS_WM", None)
 
     def update_body(self, doc=None):
         if not doc: