Merge branch 'main' into license-classify-modeling

Author: TimidRobot

.gitignore

@@ -142,4 +142,5 @@ ehthumbs.db
 Thumbs.db
 
 # secrets
+.env
 query_secrets.py

Pipfile

@@ -4,6 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+python-dotenv = "*"
 flickrapi = "*"
 internetarchive = "*"
 jupyterlab = "*"

Pipfile.lock

@@ -48,6 +48,23 @@ modules:
 [pipenvinstall]: https://pipenv.pypa.io/en/latest/install/#installing-pipenv
 
 
+### Running Scripts that Require Client Credentials
+
+To successfully run scripts that require client credentials, you will need to follow these steps:
+  1. Copy the contents of the `env.example` file in the script's directory to `.env`:
+        ```
+        cp env.example .env
+        ```
+  2. Uncomment the variables in the `.env` file and assign values as needed. See [`sources.md`](sources.md) on how to get credentials:
+        ```
+        GOOGLE_API_KEYS=your_api_key
+        PSE_KEY=your_pse_key
+       ```
+  3. Save the changes to the `.env` file.
+
+  4. You should now be able to run scripts that require client credentials without any issues.
+
+
 ### Tooling
 
 - **[Python Guidelines — Creative Commons Open Source][ccospyguide]**
@@ -64,124 +81,7 @@ modules:
 
 ## Data Sources
 
-
-### CC Legal Tools
-
-- [`legal-tool-paths.txt`](google_custom_search/legal-tool-paths.txt)
-  - A `.txt` provided by Timid Robot containing all legal tool paths. The data
-    from Google Custom Search will only cover 50+ general, most significant
-    categories of CC License for data collection quota constraint. As an
-    additional note, the order of precedence of license the collected data's
-    first column is sorted due to intermediate data analysis progress.
-    - [add list of all current CC legal tool paths by TimidRobot · Pull Request
-      #7 · creativecommons/quantifying][pr7]
-
-[pr7]: https://github.com/creativecommons/quantifying/pull/7
-
-
-### Flickr
-
-- The Flickr API exposes identifiers for users, photos, photosets and other
-  uniquely identifiable objects.
-- The Flickr API consists of a set of callable methods, and some API endpoints.
-- For more detailed description, visit: [API documentation - Flickr
-  Services](https://www.flickr.com/services/api/).
-- The `hs.csv` file is a sample CSV of pulled data. Ideally the script will
-  generate final data CSVs.
-- Each license will have a CSV to save the data.
-- Due to memory limit, the license CSVs are not pushed into github.
-
-
-### Google Custom Search JSON API
-
-- The Custom Search JSON API allows user-defined detailed query and access
-  towards related query data using a programmable search engine.
-  - [Custom Search JSON API Reference | Programmable Search Engine | Google
-    Developers][googlejsonapi]
-  - [Method: cse.list | Custom Search JSON API | Google Developers][cselist]
-- [`google_countries.tsv`](google_custom_search/google_countries.txt)
-  - Created by directly copy and pasting the `cr` parameter list from the
-    following link into a `.tsv` file as there were no reliable algorithmic way
-    for retrieving such data found in the process so far. The script itself
-    will take care of the formatting and country-selection process.
-    - [Country Collection Values | JSON API reference | Programmable Search
-      Engine | Google Developers][googlecountry]
-- [`google_lang.txt`](google_custom_search/google_lang.txt)
-  - Created by directly copy and pasting the `lr` parameter list from the
-    following link into a `.txt` file as there were no reliable algorithmic way
-    for retrieving such data found in the process so far. The script itself
-    will take care of the data formatting and language-selection process.
-    - [Parameter: lr | Method: cse.list | Custom Search JSON API | Google
-      Developers][googlelang]
-
-[googlejsonapi]: https://developers.google.com/custom-search/v1
-[cselist]: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list
-[googlecountry]: https://developers.google.com/custom-search/docs/json_api_reference#countryCollections
-[googlelang]: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list#body.QUERY_PARAMETERS.lr
-
-
-### Internet Archive Python Interface
-
-A python interface to archive.org to achieve API requests towards internet
-archive.
-- [`internetarchive.Search` - Internetarchive: A Python Interface to
-  archive.org][iasearch]
-
-[iasearch]: https://internetarchive.readthedocs.io/en/stable/internetarchive.html#internetarchive.Search
-
-
-### The Metropolitan Museum of Art Collection API
-
-An API endpoint for receiving Metropolitan Muesum of Art Collection's
-CC-Licensed works.
-
-[Latest Updates | The Metropolitan Museum of Art Collection API][metapi]:
-> The Metropolitan Museum of Art provides select datasets of information on
-> more than 470,000 artworks in its Collection for unrestricted commercial and
-> noncommercial use. To the extent possible under law, The Metropolitan Museum
-> of Art has waived all copyright and related or neighboring rights to this
-> dataset using the [Creative Commons Zero][cc-zero] license.
-
-[metapi]: https://metmuseum.github.io/
-[cc-zero]: https://creativecommons.org/publicdomain/zero/1.0/
-
-
-### Vimeo API
-
-The Vimeo API allows users to perform filtered, advanced search on Vimeo
-videos.
-- [Getting Started with the Vimeo API][vimeostart]
-  - [Search for videos - Vimeo API Reference: Videos][vimeoapisearch]
-
-[vimeostart]: https://developer.vimeo.com/api/guides/start
-[vimeoapisearch]: https://developer.vimeo.com/api/reference/videos#search_videos
-
-
-### MediaWiki API
-
-- The MediaWiki Action API is a web service that allows access to some wiki
-  features like authentication, page operations, and search. It can provide
-  meta information about the wiki and the logged-in user.
-  - Example query: https://commons.wikimedia.org/w/api.php?action=query&cmtitle=Category:CC-BY&list=categorymembers
-- [`language-codes_csv.csv`](wikipedia/language-codes_csv.csv)
-  - A list of language codes in ISO 639-1 Format to access statistics of each
-    wikipedia main page across different languages. In the script, this file is
-    named as `language-codes_csv` to minimize the amount of manual work
-    required for running the script provided the same language encoding file.
-    The user would have to rename the header and file name of their `.csv` ISO
-    code list according to the concurrent file on Github if they would like to
-    use some list other than the concurrent one.
-  - This file that this script uses can be downloaded from:
-    https://datahub.io/core/language-codes
-
-
-### Youtube Data API
-
-An API from YouTube for platform users to upload videos, adjust video
-parameters, and obtain search results.
-- [Search: list | YouTube Data API | Google Developers][youtubeapi]
-
-[youtubeapi]: https://developers.google.com/youtube/v3/docs/search/list
+Kindly visit the [sources.md](sources.md) file for it.
 
 
 ## History

README.md

@@ -12,19 +12,22 @@
 
 # Third-party
 import pandas as pd
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 today = dt.datetime.today()
-API_KEYS = query_secrets.API_KEYS
+API_KEYS = os.getenv("GOOGLE_API_KEYS").split(",")
 API_KEYS_IND = 0
-CWD = os.path.dirname(os.path.abspath(__file__))
 DATA_WRITE_FILE = (
     f"{CWD}" f"/data_deviantart_{today.year}_{today.month}_{today.day}.csv"
 )
-PSE_KEY = query_secrets.PSE_KEY
+PSE_KEY = os.getenv("PSE_KEY")
 
 
 def get_license_list():

deviantart/deviantart_scratcher.py

@@ -0,0 +1,41 @@
+## photos.py & photos_detail.py
+# "The flickr developer guide: https://www.flickr.com/services/developer/"
+
+# FLICKR_API_KEY = 
+# FLICKR_API_SECRET = 
+
+
+## deviantart_scratcher.py & google_scratcher.py 
+# "Custom Search JSON API requires the use of an API key. An API key is a way
+# to identify your client to Google."
+# https://developers.google.com/custom-search/v1/introduction
+
+# GOOGLE_API_KEYS = key1, key2
+
+# "The identifier of an engine created using the Programmable Search Engine
+# Control Panel [https://programmablesearchengine.google.com/about/]"
+# https://developers.google.com/custom-search/v1/reference/rest/v1/Search
+
+# PSE_KEY = 
+
+
+## vimeo_scratcher.py
+# "Before we set you loose on the API, we ask that you provide a little
+# information about your app. An app in this sense can be a full-featured
+# mobile application, a dynamic web page, or a three-line script. If it's
+# making API calls, it's an app."
+# https://developer.vimeo.com/api/guides/start#register-your-app
+
+# VIMEO_ACCESS_TOKEN = 
+# VIMEO_CLIENT_ID = 
+
+
+## youtube_scratcher.py
+# "Every request must either specify an API key (with the key parameter) [...].
+# Your API key is available in the Developer Console's API Access pane
+# [https://console.developers.google.com/] for your project."
+# https://developers.google.com/youtube/v3/docs
+
+# YOUTUBE_API_KEY = 
+
+

deviantart/query_secrets.example.py

@@ -1,19 +1,24 @@
 # Standard library
 import json
+import os
 import os.path
 import sys
 import traceback
 
 # Third-party
 import flickrapi
-import query_secrets
+from dotenv import load_dotenv
 
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
 
 
 def main():
     flickr = flickrapi.FlickrAPI(
-        query_secrets.api_key, query_secrets.api_secret, format="json"
+        os.getenv("FLICKR_API_KEY"),
+        os.getenv("FLICKR_API_SECRET"),
+        format="json",
     )
 
     # use search method to pull general photo info under each cc license data

env.example

@@ -9,6 +9,7 @@
 
 # Standard library
 import json
+import os
 import os.path
 import sys
 import time
@@ -17,9 +18,12 @@
 # Third-party
 import flickrapi
 import pandas as pd
-import query_secrets
+from dotenv import load_dotenv
 
 CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 RETRIES = 0
 
 
@@ -149,7 +153,9 @@ def main():
     hs_csv_path = os.path.join(CWD, "hs.csv")
 
     flickr = flickrapi.FlickrAPI(
-        query_secrets.api_key, query_secrets.api_secret, format="json"
+        os.getenv("FLICKR_API_KEY"),
+        os.getenv("FLICKR_API_SECRET"),
+        format="json",
     )
     # below is the cc licenses list
     license_list = [1, 2, 3, 4, 5, 6, 9, 10]

flickr/photos.py

@@ -12,15 +12,18 @@
 
 # Third-party
 import pandas as pd
-import query_secrets
 import requests
+from dotenv import load_dotenv
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+CWD = os.path.dirname(os.path.abspath(__file__))
+dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
+load_dotenv(dotenv_path)
+
 today = dt.datetime.today()
-API_KEYS = query_secrets.API_KEYS
+API_KEYS = os.getenv("GOOGLE_API_KEYS").split(",")
 API_KEYS_IND = 0
-CWD = os.path.dirname(os.path.abspath(__file__))
 DATA_WRITE_FILE = (
     f"{CWD}"
     f"/data_google_custom_search_{today.year}_{today.month}_{today.day}.csv"
@@ -36,7 +39,7 @@
     f"{today.year}_{today.month}_{today.day}.csv"
 )
 SEARCH_HALFYEAR_SPAN = 20
-PSE_KEY = query_secrets.PSE_KEY
+PSE_KEY = os.getenv("PSE_KEY")
 
 
 def get_license_list():