Merge pull request #267 from oree-xx/data

Move data_to_csv function to shared.py

Author: TimidRobot

scripts/2-process/gcs_process.py

@@ -4,7 +4,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import textwrap
@@ -82,16 +81,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_product_totals(args, count_data):
     """
     Processing count data: totals by product
@@ -131,7 +120,7 @@ def process_product_totals(args, count_data):
         data.items(), columns=["CC legal tool product", "Count"]
     )
     file_path = shared.path_join(PATHS["data_phase"], "gcs_product_totals.csv")
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_latest_prior_retired_totals(args, count_data):
@@ -212,7 +201,7 @@ def process_latest_prior_retired_totals(args, count_data):
         file_path = shared.path_join(
             PATHS["data_phase"], f"gcs_status_{key}_totals.csv"
         )
-        data_to_csv(args, dataframe, file_path)
+        shared.data_to_csv(args, dataframe, file_path)
 
 
 def process_totals_by_free_cultural(args, count_data):
@@ -245,7 +234,7 @@ def process_totals_by_free_cultural(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_free_cultural.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_restrictions(args, count_data):
@@ -279,7 +268,7 @@ def process_totals_by_restrictions(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_restrictions.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_language(args, data):
@@ -300,7 +289,7 @@ def process_totals_by_language(args, data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_language.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_country(args, data):
@@ -321,7 +310,7 @@ def process_totals_by_country(args, data):
     file_path = shared.path_join(
         PATHS["data_phase"], "gcs_totals_by_country.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def main():

scripts/2-process/github_process.py

@@ -5,7 +5,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import traceback
@@ -74,16 +73,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_totals_by_license(args, count_data):
     """
     Processing count data: totals by License
@@ -106,7 +95,7 @@ def process_totals_by_license(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_license.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def process_totals_by_restriction(args, count_data):
@@ -140,7 +129,7 @@ def process_totals_by_restriction(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "github_totals_by_restriction.csv"
     )
-    data_to_csv(args, data, file_path)
+    shared.data_to_csv(args, data, file_path)
 
 
 def main():

scripts/2-process/wikipedia_process.py

@@ -5,7 +5,6 @@
 """
 # Standard library
 import argparse
-import csv
 import os
 import sys
 import textwrap
@@ -85,16 +84,6 @@ def parse_arguments():
     return args
 
 
-def data_to_csv(args, data, file_path):
-    if not args.enable_save:
-        return
-    os.makedirs(PATHS["data_phase"], exist_ok=True)
-    # emulate csv.unix_dialect
-    data.to_csv(
-        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
-    )
-
-
 def process_highest_language_usage(args, count_data):
     """
     Processing count data: Most represented languages
@@ -113,7 +102,7 @@ def process_highest_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_highest_language_usage.csv"
     )
-    data_to_csv(args, top_10, file_path)
+    shared.data_to_csv(args, top_10, file_path)
 
 
 def process_least_language_usage(args, count_data):
@@ -136,7 +125,7 @@ def process_least_language_usage(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_least_language_usage.csv"
     )
-    data_to_csv(args, bottom_10, file_path)
+    shared.data_to_csv(args, bottom_10, file_path)
 
 
 def process_language_representation(args, count_data):
@@ -162,7 +151,7 @@ def process_language_representation(args, count_data):
     file_path = shared.path_join(
         PATHS["data_phase"], "wikipedia_language_representation.csv"
     )
-    data_to_csv(args, language_counts, file_path)
+    shared.data_to_csv(args, language_counts, file_path)
 
 
 def main():

scripts/shared.py

@@ -1,4 +1,5 @@
 # Standard library
+import csv
 import logging
 import os
 import sys
@@ -36,6 +37,16 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
+def data_to_csv(args, data, file_path):
+    if not args.enable_save:
+        return
+    os.makedirs(args.paths["data_phase"], exist_ok=True)
+    # emulate csv.unix_dialect
+    data.to_csv(
+        file_path, index=False, quoting=csv.QUOTE_ALL, lineterminator="\n"
+    )
+
+
 def check_for_data_files(args, file_paths, QUARTER):
     if args.force:
         return