Implement .env for secrets

- replace query_secrets.py with .env for secrets management
- add example.env for APIs that require client credentials
- make necessary path update

Author: Paulooh007

deviantart/deviantart_scratcher.py

@@ -12,19 +12,22 @@
 
 # Third-party
 import pandas as pd
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 today = dt.datetime.today()
-API_KEYS = query_secrets.API_KEYS
+API_KEYS = os.getenv("API_KEYS").split(",")
 API_KEYS_IND = 0
-CWD = os.path.dirname(os.path.abspath(__file__))
 DATA_WRITE_FILE = (
     f"{CWD}" f"/data_deviantart_{today.year}_{today.month}_{today.day}.csv"
 )
-PSE_KEY = query_secrets.PSE_KEY
+PSE_KEY = os.getenv("PSE_KEY")
 
 
 def get_license_list():

deviantart/query_secrets.example.py deviantart/example.envdeviantart/query_secrets.example.py renamed to deviantart/example.env

@@ -1,8 +1,9 @@
 # "Custom Search JSON API requires the use of an API key. An API key is a way
 # to identify your client to Google."
 # https://developers.google.com/custom-search/v1/introduction
-API_KEYS = []
+# API_KEYS = key1,key2
+
 # "The identifier of an engine created using the Programmable Search Engine
 # Control Panel [https://programmablesearchengine.google.com/about/]"
 # https://developers.google.com/custom-search/v1/reference/rest/v1/Search
-PSE_KEY = ""
+# PSE_KEY = 

flickr/photos.py

@@ -1,19 +1,22 @@
 # Standard library
 import json
+import os
 import os.path
 import sys
 import traceback
 
 # Third-party
 import flickrapi
-import query_secrets
+from dotenv import load_dotenv
 
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
 
 
 def main():
     flickr = flickrapi.FlickrAPI(
-        query_secrets.api_key, query_secrets.api_secret, format="json"
+        os.getenv("API_KEY"), os.getenv("API_SECRET"), format="json"
     )
 
     # use search method to pull general photo info under each cc license data

flickr/photos_detail.py

@@ -9,6 +9,7 @@
 
 # Standard library
 import json
+import os
 import os.path
 import sys
 import time
@@ -17,9 +18,12 @@
 # Third-party
 import flickrapi
 import pandas as pd
-import query_secrets
+from dotenv import load_dotenv
 
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 RETRIES = 0
 
 
@@ -149,7 +153,7 @@ def main():
     hs_csv_path = os.path.join(CWD, "hs.csv")
 
     flickr = flickrapi.FlickrAPI(
-        query_secrets.api_key, query_secrets.api_secret, format="json"
+        os.getenv("API_KEY"), os.getenv("API_SECRET"), format="json"
     )
     # below is the cc licenses list
     license_list = [1, 2, 3, 4, 5, 6, 9, 10]

…e_custom_search/query_secrets.example.py google_custom_search/example.envgoogle_custom_search/query_secrets.example.py renamed to google_custom_search/example.env google_custom_search/query_secrets.example.py renamed to google_custom_search/example.env

@@ -1,8 +1,9 @@
 # "Custom Search JSON API requires the use of an API key. An API key is a way
 # to identify your client to Google."
 # https://developers.google.com/custom-search/v1/introduction
-API_KEYS = []
+# API_KEYS = key1,key2
+
 # "The identifier of an engine created using the Programmable Search Engine
 # Control Panel [https://programmablesearchengine.google.com/about/]"
 # https://developers.google.com/custom-search/v1/reference/rest/v1/Search
-PSE_KEY = ""
+# PSE_KEY = 

google_custom_search/google_scratcher.py

@@ -12,15 +12,18 @@
 
 # Third-party
 import pandas as pd
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 today = dt.datetime.today()
-API_KEYS = query_secrets.API_KEYS
+API_KEYS = os.getenv("API_KEYS").split(",")
 API_KEYS_IND = 0
-CWD = os.path.dirname(os.path.abspath(__file__))
 DATA_WRITE_FILE = (
     f"{CWD}"
     f"/data_google_custom_search_{today.year}_{today.month}_{today.day}.csv"
@@ -36,7 +39,7 @@
     f"{today.year}_{today.month}_{today.day}.csv"
 )
 SEARCH_HALFYEAR_SPAN = 20
-PSE_KEY = query_secrets.PSE_KEY
+PSE_KEY = os.getenv("PSE_KEY")
 
 
 def get_license_list():

vimeo/query_secrets.example.py vimeo/example.envvimeo/query_secrets.example.py renamed to vimeo/example.env

@@ -3,5 +3,5 @@
 # mobile application, a dynamic web page, or a three-line script. If it's
 # making API calls, it's an app."
 # https://developer.vimeo.com/api/guides/start#register-your-app
-ACCESS_TOKEN = ""
-CLIENT_ID = ""
+# ACCESS_TOKEN = 
+# CLIENT_ID = 

vimeo/vimeo_scratcher.py

@@ -14,15 +14,18 @@
 import traceback
 
 # Third-party
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-today = dt.datetime.today()
-ACCESS_TOKEN = query_secrets.ACCESS_TOKEN
-CLIENT_ID = query_secrets.CLIENT_ID
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
+today = dt.datetime.today()
+ACCESS_TOKEN = os.getenv("ACCESS_TOKEN")
+CLIENT_ID = os.getenv("CLIENT_ID")
 DATA_WRITE_FILE = (
     f"{CWD}" f"/data_vimeo_{today.year}_{today.month}_{today.day}.csv"
 )

youtube/query_secrets.example.py youtube/example.envyoutube/query_secrets.example.py renamed to youtube/example.env

@@ -2,4 +2,4 @@
 # Your API key is available in the Developer Console's API Access pane
 # [https://console.developers.google.com/] for your project."
 # https://developers.google.com/youtube/v3/docs
-API_KEY = ""
+# API_KEY = 

youtube/youtube_scratcher.py

@@ -11,14 +11,17 @@
 import traceback
 
 # Third-party
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-today = dt.datetime.today()
-API_KEY = query_secrets.API_KEY
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
+today = dt.datetime.today()
+API_KEY = os.getenv("API_KEY")
 DATA_WRITE_FILE = (
     f"{CWD}" f"/data_youtube_{today.year}_{today.month}_{today.day}.csv"
 )