Save HTML in database

Author: adelavega

ace/config.py

@@ -58,6 +58,9 @@
 # When False, will use fallback HTML processing by default.
 USE_READABILITY = True
 
+# Whether to save the original HTML of the table in the Table object
+SAVE_ORIGINAL_HTML = False
+
 
 
 

ace/database.py

@@ -169,6 +169,7 @@ class Table(Base):
     notes = Column(Text)
     n_activations = Column(Integer)
     n_columns = Column(Integer)
+    input_html = Column(LongText)
 
     def finalize(self):
         ''' Any cleanup and updating operations we need to do before saving. '''

ace/sources.py

@@ -337,7 +337,7 @@ def n_cols_in_row(row):
         if data.data[data.n_rows- 1].count(None) == data.n_cols:
             data.data.pop()
         logger.debug("\t\tTrying to parse table...")
-        return tableparser.parse_table(data)
+        return tableparser.parse_table(data, html=str(table))
 
     def extract_doi(self, soup):
         ''' Every Source subclass must be able to extract its doi. '''

ace/tableparser.py

@@ -222,10 +222,13 @@ def create_activation(data, labels, standard_cols, group_labels=[]):
     return activation
 
 
-def parse_table(data):
+def parse_table(data, html=None):
     ''' Takes a DataTable as input and returns a Table instance. '''
 
     table = Table()
+    # Only store the original HTML if the global config allows it
+    if html is not None and config.SAVE_ORIGINAL_HTML:
+        table.input_html = html
     n_cols = data.n_cols
 
     # Identify column names: first occurrence of unique (i.e. colspan=1) label.