Merge pull request #12 from daita-technologies/develop

BEdaita · web-flow · commit 054561a67f6e · 2022-11-23T07:13:18.000+07:00
✨ Release 0.0.7 `daita-python-library`
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -31,6 +31,20 @@ jobs:
           build
           --user
 
+      - name: Make .env file
+        uses: SpicyPizza/create-envfile@v1.3
+        with:
+          envkey_DAITA_TOKEN: ${{ secrets.DAITA_TOKEN }}
+          envkey_PRESIGNED_URL: ${{ secrets.PRESIGNED_URL }}
+          envkey_CHECK_FILE_EXISTENCE: ${{ secrets.CHECK_FILE_EXISTENCE }}
+          envkey_CREATE_PRESIGNED_SINGLE_URL: ${{ secrets.CREATE_PRESIGNED_SINGLE_URL }}
+          envkey_UPLOAD_COMPRESSED_FILE: ${{ secrets.UPLOAD_COMPRESSED_FILE }}
+          envkey_UPLOAD_TASK_PROGRESS: ${{ secrets.TASK_PROGRESS }}
+          envkey_UPLOAD_UPDATE: ${{secrets.UPLOAD_UPDATE}}
+          directory: ./daita
+          file_name: .env
+          fail_on_empty: true
+
       - name: Build a binary wheel and a source tarball
         run: >-
           python -m
diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
@@ -33,6 +33,20 @@ jobs:
           build
           --user
 
+      - name: Make .env file
+        uses: SpicyPizza/create-envfile@v1.3
+        with:
+          envkey_DAITA_TOKEN: ${{ secrets.DAITA_TOKEN }}
+          envkey_PRESIGNED_URL: ${{ secrets.PRESIGNED_URL }}
+          envkey_CHECK_FILE_EXISTENCE: ${{ secrets.CHECK_FILE_EXISTENCE }}
+          envkey_CREATE_PRESIGNED_SINGLE_URL: ${{ secrets.CREATE_PRESIGNED_SINGLE_URL }}
+          envkey_UPLOAD_COMPRESSED_FILE: ${{ secrets.UPLOAD_COMPRESSED_FILE }}
+          envkey_UPLOAD_TASK_PROGRESS: ${{ secrets.TASK_PROGRESS }}
+          envkey_UPLOAD_UPDATE: ${{secrets.UPLOAD_UPDATE}}
+          directory: ./daita
+          file_name: .env
+          fail_on_empty: true
+
       - name: Build a binary wheel and a source tarball
         run: >-
           python -m
diff --git a/.github/workflows/publish-test-pypi.yml b/.github/workflows/publish-test-pypi.yml
@@ -34,6 +34,20 @@ jobs:
           build
           --user
 
+      - name: Make .env file
+        uses: SpicyPizza/create-envfile@v1.3
+        with:
+          envkey_DAITA_TOKEN: ${{ secrets.DAITA_TOKEN }}
+          envkey_PRESIGNED_URL: ${{ secrets.PRESIGNED_URL }}
+          envkey_CHECK_FILE_EXISTENCE: ${{ secrets.CHECK_FILE_EXISTENCE }}
+          envkey_CREATE_PRESIGNED_SINGLE_URL: ${{ secrets.CREATE_PRESIGNED_SINGLE_URL }}
+          envkey_UPLOAD_COMPRESSED_FILE: ${{ secrets.UPLOAD_COMPRESSED_FILE }}
+          envkey_UPLOAD_TASK_PROGRESS: ${{ secrets.TASK_PROGRESS }}
+          envkey_UPLOAD_UPDATE: ${{secrets.UPLOAD_UPDATE}}
+          directory: ./daita
+          file_name: .env
+          fail_on_empty: true
+
       - name: Build a binary wheel and a source tarball
         run: >-
           python -m
diff --git a/.gitignore b/.gitignore
@@ -28,8 +28,8 @@ share/python-wheels/
 MANIFEST
 
 # PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 pip-log.txt
@@ -83,10 +83,10 @@ ipython_config.py
 .python-version
 
 # pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
 #Pipfile.lock
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
@@ -101,6 +101,7 @@ celerybeat.pid
 
 # Environments
 .env
+.env.development
 .venv
 env/
 venv/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,12 +2,21 @@
 
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## `0.0.6` – 29-10-2022
+## `0.0.7` – 22-11-2022
 
 ### Fixed
 
-- Fixed [`Unicode decode error`](https://github.com/daita-technologies/daita-python-library/issues/3).
+- Fixed [`Remove .env.development file`](https://github.com/daita-technologies/daita-python-library/issues/5). ([#8](https://github.com/daita-technologies/daita-python-library/pull/8))
+
+### Added
+
+- Added [`Handling of unexpected interruptions during upload`](https://github.com/daita-technologies/daita-python-library/issues/6). ([#10](https://github.com/daita-technologies/daita-python-library/pull/10))
+- Added [`Show progress bar for uploads`](https://github.com/daita-technologies/daita-python-library/issues/4). ([#11](https://github.com/daita-technologies/daita-python-library/pull/11))
+
+## `0.0.6` – 29-10-2022
+
+### Fixed
 
+- Fixed [`Unicode decode error`](https://github.com/daita-technologies/daita-python-library/issues/3). ([#7](https://github.com/daita-technologies/daita-python-library/pull/7))
diff --git a/daita/.env.development b/daita/.env.development
diff --git a/daita/__init__.py b/daita/__init__.py
@@ -1 +1 @@
-__version__ = "0.0.6"
+__version__ = "0.0.7"
diff --git a/daita/daita.py b/daita/daita.py
@@ -1,17 +1,20 @@
-from daita.dashboard import dashboard
 import argparse
+from daita.utils import exception_handler
+from daita.dashboard import dashboard
+from daita.footer import footer
 
-
-parser = argparse.ArgumentParser(description="Optional app description")
-parser.add_argument("--dir", type=str, help="A required integer positional argument")
+parser = argparse.ArgumentParser(
+    description="Upload images via DAITA's official easy-to-use Python library."
+)
+parser.add_argument("--dir", type=str, help="The root directory path for the images.")
 parser.add_argument(
-    "--daita_token", type=str, help="A required integer positional argument"
+    "--daita_token", type=str, help="User access token for the DAITA platform."
 )
 args = parser.parse_args()
 dir = args.dir
 daita_token = args.daita_token
 
 
+@exception_handler
 def main():
     dashboard(daita_token=daita_token, dir=dir)
-    pass
diff --git a/daita/dashboard.py b/daita/dashboard.py
@@ -1,5 +1,4 @@
 import os
-import glob
 import requests
 import time
 from os import system, name
@@ -21,7 +20,7 @@ def validCompressfile(filename):
 
 def listImageFile(dir):
     filenames = []
-    for root, subFolder, files in os.walk(dir):
+    for root, files in os.walk(dir):
         for item in files:
             if validFileImage(item):
                 filenames.append(os.path.join(root, item))
@@ -42,8 +41,7 @@ def checkDaitaToken(daita_token):
     response = requests.get(checkDaitaTokenEndpoint, params=params)
     data = response.json()
     if data["error"] == True:
-        print(data["message"])
-        os._exit(1)
+        Exception("Failed checkDaitaToken: {}".format(data["message"]))
 
 
 def listAllFilesInDirectory(dir):
@@ -106,12 +104,16 @@ def dashboard(daita_token, dir):
 
     # check input directory
     if not os.path.isdir(dir):
+        path = dir
+        if validCompressfile(path):
+            dashboardCompressFiles([path], daita_token)
         print("Please input your directory path; try it again!")
         footer()
+
     imagefiles, compressfiles = listAllFilesInDirectory(dir)
 
     if len(imagefiles) == 0 and len(compressfiles) == 0:
-        print("Folder is empty; please select again!")
+        print("Folder is empty, please select again!")
         footer()
 
     if len(imagefiles) > 0 and len(compressfiles) == 0:
diff --git a/daita/load_env.py b/daita/load_env.py
@@ -1,8 +1,9 @@
+import os
 from dotenv import load_dotenv
 from pathlib import Path
-import os
+
 
 pwd = os.path.dirname(os.path.abspath(__file__))
-envfile = os.path.join(pwd, ".env.development")
+envfile = os.path.join(pwd, ".env")
 dotenv_path = Path(envfile)
 load_dotenv(dotenv_path=dotenv_path)
diff --git a/daita/upload_images.py b/daita/upload_images.py
@@ -1,5 +1,4 @@
 import os
-import glob
 import time
 import requests
 import hashlib
@@ -8,13 +7,13 @@
 import threading
 from tqdm import tqdm
 from daita.footer import footer
+from daita.utils import retry_handler
 from itertools import chain, islice
 
 batch_size = 10
-###### the endpoint for dev enviroment#######################################################################################
-endpointPresignURL = os.environ["PRESIGN_URL"]
-endpointCheckExistenceFile = os.environ["CHECK_EXISTENCE_FILE"]
-#############################################################################################################################
+endpointPresignURL = os.environ["CREATE_PRESIGNED_SINGLE_URL"]
+endpointCheckExistenceFile = os.environ["CHECK_FILE_EXISTENCE"]
+endpointUploadUpdate = os.environ["UPLOAD_UPDATE"]
 
 daita_token = None
 
@@ -25,8 +24,28 @@ def batcher(iterable, size):
         yield list(chain([first], islice(iterator, size - 1)))
 
 
-def getPresignUrl(filesnames):
-    filenamesDict = {os.path.basename(it): it for it in filesnames}
+@retry_handler(max_retry=10, sleep_time=1)
+def put_image_S3(filename):
+    resp = requests.post(
+        endpointPresignURL,
+        json={
+            "filename": os.path.basename(filename),
+            "daita_token": daita_token,
+            "is_zip": False,
+        },
+    )
+    preSignUrlResult = resp.json()["data"]
+    files = {"file": open(filename, "rb")}
+    requests.post(
+        preSignUrlResult["presign_url"]["url"],
+        data=preSignUrlResult["presign_url"]["fields"],
+        files=files,
+    )
+    time.sleep(0.5)
+    return
+
+
+def UploadUpdate(filesnames):
     filenamesList = []
     ls_object_info = []
     for it in filesnames:
@@ -48,22 +67,12 @@ def getPresignUrl(filesnames):
         "ls_object_info": ls_object_info,
         "daita_token": daita_token,
     }
-    preSignUrlResp = requests.post(endpointPresignURL, json=payload)
+    preSignUrlResp = requests.post(endpointUploadUpdate, json=payload)
     if preSignUrlResp.status_code != 200:
-        print("Something went wrong sorry, please check again")
+        print("Something went wrong sorry, please check again!")
         print(preSignUrlResp.text)
         footer()
-    if preSignUrlResp.json()["error"] == True:
-        print("Something went wrong sorry, please check again")
-        print(preSignUrlResp.json()["message"])
-        footer()
-
-    preSignUrlResult = preSignUrlResp.json()["data"]
-    for k, v in filenamesDict.items():
-        files = {"file": open(v, "rb")}
-        requests.post(
-            preSignUrlResult[k]["url"], data=preSignUrlResult[k]["fields"], files=files
-        )
+    return
 
 
 def generator(filenames):
@@ -79,7 +88,15 @@ def __init__(self, queue, updateQueue):
     def run(self):
         while True:
             filenames = self.queue.get()
-            getPresignUrl(filesnames=filenames)
+            try:
+                for it in filenames:
+                    put_image_S3(filename=it)
+            except Exception as e:
+                print("Debug: {}".format(e))
+                return
+
+            UploadUpdate(filesnames=filenames)
+
             self.updateQueue.put(len(filenames))
             self.queue.task_done()
 
@@ -106,7 +123,7 @@ def checkExistenceFile(filenames, daita_token):
         if it["filename"] in fullfile:
             del fullfile[it["filename"]]
 
-    for k, v in fullfile.items():
+    for v in fullfile.items():
         newFileNotExist.append(v)
     return newFileNotExist
 
@@ -130,7 +147,7 @@ def upload_images(filenames, token):
     for batch in batches:
         workerQueue.put(batch)
 
-    for _ in range(5):
+    for _ in range(10):
         t = ThreadWorker(queue=workerQueue, updateQueue=updateProcessbarQueue)
         t.setDaemon(True)
         t.start()
@@ -144,10 +161,9 @@ def listen(q):
     t1 = threading.Thread(target=listen, args=(updateProcessbarQueue,))
     t1.daemon = True
     t1.start()
-    workerQueue.join()
     numberFileTotal = len(filenames)
     print(f"Total File: {numberFileTotal}")
-    print("STARTING UPLOAD")
+    print("STARTING UPLOAD...")
     currentFileCompleted = 0
     numFileCompleted = 0
     with tqdm(total=numberFileTotal, file=sys.stdout) as pbar:
@@ -159,6 +175,7 @@ def listen(q):
             if currentFileCompleted == numberFileTotal:
                 break
     print("COMPLETED")
+    workerQueue.join()
 
 
 def dashboardImageFiles(filenames, token):
diff --git a/daita/upload_zip.py b/daita/upload_zip.py
diff --git a/daita/utils.py b/daita/utils.py
diff --git a/setup.py b/setup.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.0.6"`
	`1`	`+__version__ = "0.0.7"`