Merge pull request #2114 from CentreForDigitalHumanities/feature/unify-settings-formats

Feature/unify settings formats

Author: JeltevanBoheemen

backend/addcorpus/python_corpora/corpus.py

@@ -547,10 +547,3 @@ def transform_to_date_range(earliest, latest):
         'gte': earliest,
         'lte': latest
     }
-
-
-def get_deprecated_setting(setting: str):
-    '''Utility function to get a corpus-specific Django setting'''
-    if hasattr(settings, setting):
-        logger.warning('Setting %s is deprecated; use CORPUS_SETTINGS', setting)
-        return getattr(settings, setting)

backend/addcorpus/python_corpora/tests/test_times_source.py

@@ -11,9 +11,10 @@ def times_test_settings(settings):
     settings.CORPORA = {
         'times': 'corpora.times.times.Times',
     }
-    settings.TIMES_DATA = join(settings.BASE_DIR, 'addcorpus/python_corpora/tests')
-    settings.TIMES_ES_INDEX = 'test-times'
-
+    settings.CORPUS_SETTINGS["times"] = {
+        "data_directory": join(settings.BASE_DIR, "addcorpus/python_corpora/tests"),
+        "es_index": "test-times",
+    }
 
 
 def test_times_source(times_test_settings):

backend/corpora/dbnl/dbnl.py

@@ -4,7 +4,7 @@
 from tqdm import tqdm
 from ianalyzer_readers.xml_tag import Tag, CurrentTag, TransformTag
 
-from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition, get_deprecated_setting
+from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
 from ianalyzer_readers.extract import Metadata, XML, Pass, Order, Backup, Combined
 import corpora.dbnl.utils as utils
 from addcorpus.es_mappings import *
@@ -14,18 +14,11 @@
 class DBNL(XMLCorpusDefinition):
     title = 'DBNL'
     description = 'Dutch literature and publications on literary or linguistic studies, from the Middle Ages to the 19th century'
-
-    @property
-    def data_directory(self):
-        return get_deprecated_setting('DBNL_DATA') or super().data_directory
+    es_index = 'dbnl'  # default setting for development
 
     min_date = datetime(year=1200, month=1, day=1)
     max_date = datetime(year=1890, month=12, day=31)
 
-    @property
-    def es_index(self):
-        return get_deprecated_setting('DBNL_ES_INDEX') or 'dbnl'
-
     image = 'dbnl.png'
     description_page = 'dbnl.md'
     citation_page = 'citation.md'

backend/corpora/dutchannualreports/dutchannualreports.py

@@ -25,13 +25,11 @@ class DutchAnnualReports(XMLCorpusDefinition):
     description = "Annual reports of Dutch financial and non-financial institutes"
     min_date = datetime(year=1957, month=1, day=1)
     max_date = datetime(year=2008, month=12, day=31)
-    data_directory = settings.DUTCHANNUALREPORTS_DATA
-    es_index = getattr(settings, 'DUTCHANNUALREPORTS_ES_INDEX', 'dutchannualreports')
+    es_index = 'dutchannualreports'
     image = 'dutchannualreports.jpg'
-    scan_image_type = getattr(settings, 'DUTCHANNUALREPORTS_SCAN_IMAGE_TYPE', 'application/pdf')
+    scan_image_type = 'application/pdf'
     description_page = 'dutchannualreports.md'
-    allow_image_download = getattr(settings, 'DUTCHANNUALREPORTS_ALLOW_IMAGE_DOWNLOAD', True)
-    word_model_path = getattr(settings, 'DUTCHANNUALREPORTS_WM', None)
+    allow_image_download = True
 
     languages = ['nl']
     category = 'finance'

backend/corpora/dutchnewspapers/dutchnewspapers_all.py

@@ -14,10 +14,8 @@ class DutchNewsPapersAll(DutchNewspapersPublic):
 
     title = "Dutch Newspapers (full)"
     description = "Full collection of Dutch newspapers digitised by the Koninklijke Bibliotheek."
-    data_directory = settings.DUTCHNEWSPAPERS_ALL_DATA
-    es_index = getattr(settings, 'DUTCHNEWSPAPERS_ALL_ES_INDEX', 'dutchnewspapers-all')
+    es_index = 'dutchnewspapers-all'
     max_date = datetime(year=1995, month=12, day=31)
-    word_model_path = getattr(settings, "DUTCHNEWSPAPERS_WM", None)
 
     def update_body(self, doc=None):
         if not doc:

backend/corpora/dutchnewspapers/dutchnewspapers_public.py

@@ -32,14 +32,12 @@ class DutchNewspapersPublic(XMLCorpusDefinition):
     description = "Collection of Dutch newspapers in the public domain, digitised by the Koninklijke Bibliotheek."
     min_date = datetime(year=1600, month=1, day=1)
     max_date = datetime(year=1876, month=12, day=31)
-    data_directory = getattr(settings, 'DUTCHNEWSPAPERS_DATA', None)
-    es_index = getattr(settings, 'DUTCHNEWSPAPERS_ES_INDEX', 'dutchnewspapers-public')
+    es_index = 'dutchnewspapers-public'
     image = 'dutchnewspapers.jpg'
     languages = ['nl']
     category = 'periodical'
     description_page = 'description_public.md'
     citation_page = 'citation_public.md'
-    word_model_path = getattr(settings, "DUTCHNEWSPAPERS_WM", None)
 
     @property
     def es_settings(self):

backend/corpora/ecco/ecco.py

@@ -31,10 +31,9 @@ class Ecco(XMLCorpusDefinition):
     min_date = datetime(year=1700, month=1, day=1)
     max_date = datetime(year=1800, month=12, day=31)
 
-    data_directory = settings.ECCO_DATA
-    es_index = getattr(settings, 'ECCO_ES_INDEX', 'ecco')
+    es_index = 'ecco'
     image = 'ecco.jpg'
-    scan_image_type = getattr(settings, 'ECCO_SCAN_IMAGE_TYPE', 'application/pdf')
+    scan_image_type = 'application/pdf'
     es_settings = None
     languages = ['en', 'fr', 'la', 'grc', 'de',  'it', 'cy', 'ga', 'gd']
     category = 'book'

backend/corpora/economist/economist.py

@@ -54,8 +54,7 @@ class Economist(GaleCorpus):
     description = "Archive of The Economist, a weekly news magazine."
     min_date = datetime(1843, 8, 1)
     max_date = datetime(2021, 1, 1)
-    data_directory = settings.ECONOMIST_DATA
-    es_index = getattr(settings, 'ECONOMIST_ES_INDEX', 'economist')
+    es_index = 'economist'
     image = 'A_stack_of_Economist_papers.jpg'
     description_page = 'Economist.md'
     languages = ['en']

backend/corpora/gallica/caricature.py

@@ -12,7 +12,7 @@ class Caricature(Gallica):
     max_date = datetime(year=1843, month=12, day=31)
     corpus_id = "cb344523348"
     category = "periodical"
-    es_index = getattr(settings, 'CARICATURE_INDEX', 'caricature')
+    es_index = 'caricature'
     description_page = 'caricature.md'
     image = "caricature.jpg"
 

backend/corpora/gallica/figaro.py

@@ -12,7 +12,7 @@ class Figaro(Gallica):
     max_date = datetime(year=1954, month=12, day=31)
     corpus_id = "cb34355551z"
     category = "periodical"
-    es_index = getattr(settings, 'FIGARO_INDEX', 'figaro')
+    es_index = 'figaro'
     image = "figaro.jpg"
     description_page = 'figaro.md'