Change usage of config

Author: adelavega

ace/__init__.py

@@ -1,33 +1,43 @@
-# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
-# ex: set sts=4 ts=4 sw=4 et:
+# emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 sw=4 et:
 """ACE -- Automated Coordinate Extraction.
 """
-__all__ = ["config", "ingest", "database", "datatable", "set_logging_level", "scrape", "sources", "tableparser", "tests", "__version__"]
+__all__ = [
+    "config", "ingest", "database", "datatable", "set_logging_level", 
+    "scrape", "sources", "tableparser", "tests", "__version__"
+]
 
 import logging
 import sys
 import os
 
 from .version import __version__
 
+
 def set_logging_level(level=None):
     """Set package-wide logging level
 
-    Args
-        level : Logging level constant from logging module (warning, error, info, etc.)
+    Args:
+        level: Logging level constant from logging module
+               (warning, error, info, etc.)
     """
     if level is None:
         level = os.environ.get('ACE_LOGLEVEL', 'warn')
     logger.setLevel(getattr(logging, level.upper()))
     return logger.getEffectiveLevel()
 
+
 def _setup_logger(logger):
     # Basic logging setup
     console = logging.StreamHandler(sys.stdout)
-    console.setFormatter(logging.Formatter("%(levelname)-6s %(module)-7s %(message)s"))
+    formatter = logging.Formatter(
+        "%(levelname)-6s %(module)-7s %(message)s"
+    )
+    console.setFormatter(formatter)
     logger.addHandler(console)
     set_logging_level()
 
+
 # Set up logger
 logger = logging.getLogger("ace")
 _setup_logger(logger)

ace/database.py

@@ -1,8 +1,10 @@
 # Database stuff and models
 
-from sqlalchemy import (TypeDecorator, Table, Column, Integer, Float, String, Boolean,
-                        ForeignKey, DateTime, Text)
-from sqlalchemy.orm import relationship, backref, sessionmaker
+from sqlalchemy import (
+    TypeDecorator, Table, Column, Integer, Float, String, Boolean,
+    ForeignKey, DateTime, Text
+)
+from sqlalchemy.orm import relationship, sessionmaker
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.ext.associationproxy import association_proxy
@@ -13,9 +15,8 @@
 import logging
 import sys
 from os import path
-import datetime
 
-from . import config
+from .config import get_config
 from . import extract
 
 logger = logging.getLogger(__name__)
@@ -32,22 +33,32 @@ def __init__(self, adapter=None, db_name=None, user=None, password=None):
         ''' Connect to DB and initialize instance. '''
 
         # Default to settings in config file if none passed
-        if adapter is None: adapter = config.SQL_ADAPTER
+        if adapter is None:
+            adapter = get_config('SQL_ADAPTER')
 
         # Generate DB URI
         if adapter == 'sqlite':
-            db_uri = config.SQLITE_URI if db_name is None else db_name
+            db_uri = get_config('SQLITE_URI') if db_name is None else db_name
         elif adapter == 'mysql':
-            db_name = config.MYSQL_DB if db_name is None else db_name
-            if user is None: user = config.MYSQL_USER
-            if password is None: password = config.MYSQL_PASSWORD
-            db_uri = 'mysql://%s:%s@localhost/%s' % (user, password, db_name)
+            db_name = get_config('MYSQL_DB') if db_name is None else db_name
+            if user is None:
+                user = get_config('MYSQL_USER')
+            if password is None:
+                password = get_config('MYSQL_PASSWORD')
+            db_uri = f'mysql://{user}:{password}@localhost/{db_name}'
         else:
-            raise ValueError("Value of SQL_ADAPTER in settings must be either 'sqlite' or 'mysql'")
+            raise ValueError(
+                "SQL_ADAPTER must be either 'sqlite' or 'mysql'"
+            )
 
-        engine = create_engine(db_uri, echo=False, connect_args={'timeout': 15})
+        engine = create_engine(
+            db_uri, 
+            echo=False, 
+            connect_args={'timeout': 15}
+        )
 
-        if adapter == 'mysql': engine.execute("SET sql_mode=''")
+        if adapter == 'mysql':
+            engine.execute("SET sql_mode=''")
 
         Session = sessionmaker(bind=engine)
         Base.metadata.create_all(engine)
@@ -60,8 +71,6 @@ def add(self, record):
     def save(self):
         ''' Commit all stored records to file. '''
         self.session.commit()
-            # except Exception as err:
-            #     print(err)
 
     def delete_article(self, pmid):
         article = self.session.query(Article).filter_by(id=pmid).first()
@@ -71,23 +80,37 @@ def delete_article(self, pmid):
     def print_stats(self):
         ''' Summarize the current state of the DB. '''
         n_articles = self.session.query(Article).count()
-        n_articles_with_coordinates = self.session.query(Article).join(Table).filter(Table.n_activations>0).distinct('article_id').count()
+        n_articles_with_coordinates = self.session.query(Article) \
+            .join(Table) \
+            .filter(Table.n_activations > 0) \
+            .distinct('article_id') \
+            .count()
         n_tables = self.session.query(Table).count()
         n_activations = self.session.query(Activation).count()
         n_links = self.session.query(NeurovaultLink).count()
-        n_articles_with_links = self.session.query(NeurovaultLink).distinct('article_id').count()
-        print(f"The database currently contains: {n_articles} articles.\n"
-        f"{n_articles_with_coordinates} have coordinates, and {n_articles_with_links} have NeuroVault links.\n"
-        f"Total of {n_tables} tables, {n_activations} activations and {n_links} NeuroVault links.")
+        n_articles_with_links = self.session.query(NeurovaultLink) \
+            .distinct('article_id') \
+            .count()
+        
+        print(
+            f"The database currently contains: {n_articles} articles.\n"
+            f"{n_articles_with_coordinates} have coordinates, and "
+            f"{n_articles_with_links} have NeuroVault links.\n"
+            f"Total of {n_tables} tables, {n_activations} activations "
+            f"and {n_links} NeuroVault links."
+        )
 
     def article_exists(self, pmid):
         ''' Check if an article already exists in the database. '''
-        return self.session.query(exists().where(Article.id==pmid)).scalar()
+        return self.session.query(
+            exists().where(Article.id == pmid)
+        ).scalar()
 
     @property
     def articles(self):
         return self.session.query(Article).all()
 
+
 # Create a JSONString column type for convenience
 class JsonString(TypeDecorator):
     impl = Text
@@ -121,16 +144,25 @@ class Article(Base):
     abstract = Column(Text)
     citation = Column(Text)
     pubmed_metadata = Column(JsonString)
-    created_at =  Column(DateTime, default=datetime.datetime.utcnow)
-    updated_at = Column(DateTime, default=datetime.datetime.utcnow,
-                           onupdate=datetime.datetime.utcnow)
-
-    tables = relationship('Table', cascade="all, delete-orphan",
-                          backref='article')
-
-    neurovault_links = relationship('NeurovaultLink', cascade="all, delete-orphan",
-                                backref='article')
-                                
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(
+        DateTime, 
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow
+    )
+
+    tables = relationship(
+        'Table', 
+        cascade="all, delete-orphan",
+        backref='article'
+    )
+
+    neurovault_links = relationship(
+        'NeurovaultLink', 
+        cascade="all, delete-orphan",
+        backref='article'
+    )
+    
     features = association_proxy('tags', 'feature')
 
     def __init__(self, text, pmid=None, doi=None, metadata=None):
@@ -160,8 +192,11 @@ class Table(Base):
 
     id = Column(Integer, primary_key=True)
     article_id = Column(Integer, ForeignKey('articles.id'))
-    activations = relationship('Activation', cascade="all, delete-orphan",
-                                backref='table')
+    activations = relationship(
+        'Activation', 
+        cascade="all, delete-orphan",
+        backref='table'
+    )
     position = Column(Integer)   # The serial position of occurrence
     number = Column(String(10))   # The stated table ID (e.g., 1, 2b)
     label = Column(String(200))  # The full label (e.g., Table 1, Table 2b)
@@ -172,19 +207,7 @@ class Table(Base):
     input_html = Column(LongText)
 
     def finalize(self):
-        ''' Any cleanup and updating operations we need to do before saving. '''
-
-        # # Remove duplicate activations--most commonly produced by problems with
-        # # the grouping code.
-        # act_defs = set()
-        # to_keep = []
-        # for a in self.activations:
-        #     definition = json.dumps([a.x, a.y, a.z, a.groups])
-        #     if definition not in act_defs:
-        #         act_defs.add(definition)
-        #         to_keep.append(a)
-        # self.activations = to_keep
-
+        ''' Any cleanup before saving. '''
         self.n_activations = len(self.activations)
 
 
@@ -234,35 +257,35 @@ def add_col(self, key, val):
     # Validates Peak. Considers peak invalid if:
     # * At least one of X, Y, Z is nil or missing
     # * Any |coordinate| > 100
-    # * Two or more columns are zeroes (most of the time this
-    #   will indicate a problem, but occasionally a real coordinate)
-    # Depending on config, either excludes peak, or allows it through
-    # but flags potential problems for later inspection.
+    # * Two or more columns are zeroes 
     def validate(self):
-
         for c in [self.x, self.y, self.z]:
             if c == '' or c is None:
-                logger.debug('Missing x, y, or z coordinate information: [%s, %s, %s]' % tuple(
-                    [str(e) for e in [self.x, self.y, self.z]]))
+                logger.debug(
+                    'Missing x, y, or z coordinate: [%s, %s, %s]',
+                    self.x, self.y, self.z
+                )
                 return False
             try:
                 if abs(c) >= 100:
                     logger.debug(
-                        'Invalid coordinates: at least one dimension (x,y,z) >= 100.')
+                        'Invalid coordinates: dimension >= 100.'
+                    )
                     return False
-            except:
-                print(c)
-                print(sys.exc_info()[0])
+            except Exception:
+                logger.exception("Error validating coordinate")
                 raise
 
         sorted_xyz = sorted([abs(self.x), abs(self.y), abs(self.z)])
         if sorted_xyz[0] == 0 and sorted_xyz[1] == 0:
             logger.debug(
-                "At least two dimensions have value == 0; coordinate is probably not real.")
+                "Two dimensions have value 0; probably not real."
+            )
             return False
 
         return True
 
+
 class NeurovaultLink(Base):
 
     __tablename__ = 'Neurovaultlinks'

ace/export.py

@@ -107,10 +107,10 @@ def export_database(db, foldername, skip_empty=True, table_html=False):
 
         for art in articles:
             art_dir = tables_dir / str(art.id)
-            art_dir.mkdir(parents=True, exist_ok=True)
 
             for t in art.tables:
                 if t.input_html:
+                    art_dir.mkdir(parents=True, exist_ok=True)
                     table_file = art_dir / f"{t.id}.html"
                     with table_file.open('w', encoding='utf-8') as f:
                         f.write(t.input_html)