Merge branch 'main' into crag/claude

Author: PastelStorm

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 env:
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.12"
   PIPELINE_FAMILY: "general"
 
 jobs:
@@ -20,7 +20,7 @@ jobs:
       with:
         path: |
           .venv
-        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt') }}
+        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/*.txt') }}
     - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@v5
       with:
@@ -42,7 +42,7 @@ jobs:
       with:
         path: |
           .venv
-        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt') }}
+        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/*.txt') }}
     - name: Lint
       run: |
         source .venv/bin/activate
@@ -56,7 +56,7 @@ jobs:
         uses: ludeeus/action-shellcheck@master
 
   test:
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v4
@@ -65,16 +65,22 @@ jobs:
       with:
         path: |
           .venv
-        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt') }}
+        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}
+
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
     - name: Run core tests
       run: |
+        python${{ env.PYTHON_VERSION }} -m venv .venv
         source .venv/bin/activate
         sudo apt-get update && sudo apt-get install --yes poppler-utils libreoffice
+        make install-test
         make install-pandoc
         sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
         sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
         tesseract --version
-        make install-nltk-models
         make test
         make check-coverage
 
@@ -97,19 +103,26 @@ jobs:
   # TODO - figure out best practice for caching docker images
   # (Using the virtualenv to get pytest)
   test_dockerfile:
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/cache@v4
       id: virtualenv-cache
       with:
+        python-version: ${{ env.PYTHON_VERSION }}
         path: |
           .venv
-        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt') }}
+        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
     - name: Test Dockerfile
       run: |
+        python${{ env.PYTHON_VERSION }} -m venv .venv
         source .venv/bin/activate
+        make install-test
         make docker-build
         make docker-test
     # - name: Scan image

.github/workflows/docker-publish.yml

@@ -11,7 +11,7 @@ env:
   PACKAGE: "unstructured-api"
   PIPELINE_FAMILY: "general"
   PIP_VERSION: "25.1.1"
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.12"
 
 jobs:
   setup:
@@ -48,7 +48,7 @@ jobs:
         #arch: ["arm64", "amd64"]
         # NOTE(luke): temporary disable arm64 since its failing the smoke test
         arch: ["amd64"]
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest
     needs: [setup, set-short-sha]
     env:
       SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
@@ -103,7 +103,7 @@ jobs:
         # write to the build repository to cache for the publish-images job
         docker push $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA
   publish-images:
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest
     needs: [setup, set-short-sha, build-images]
     env:
       SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}

Makefile

@@ -95,7 +95,7 @@ run-web-app:
 ## test:                        runs core tests
 .PHONY: test
 test:
-	PYTHONPATH=. pytest -v test_${PIPELINE_PACKAGE} --cov=${PACKAGE_NAME} --cov-report term-missing
+	PYTHONPATH=. pytest -n auto -v test_${PIPELINE_PACKAGE} --cov=${PACKAGE_NAME} --cov-report term-missing
 
 # Setting a low bar here - need more tests!
 .PHONY: check-coverage

README.md

@@ -289,12 +289,13 @@ curl -X 'POST'
 * Using `pyenv` to manage virtualenv's is recommended
 	* Mac install instructions. See [here](https://github.com/Unstructured-IO/community#mac--homebrew) for more detailed instructions.
 		* `brew install pyenv-virtualenv`
-	  * `pyenv install 3.10.12`
+	  * `pyenv install 3.12`
   * Linux instructions are available [here](https://github.com/Unstructured-IO/community#linux).
 
   * Create a virtualenv to work in and activate it, e.g. for one named `document-processing`:
 
-	`pyenv  virtualenv 3.10.12 unstructured-api` <br />
+	`pyenv  virtualenv 3.12
+   unstructured-api` <br />
 	`pyenv activate unstructured-api`
 
 See the [Unstructured Quick Start](https://github.com/Unstructured-IO/unstructured#eight_pointed_black_star-quick-start) for the many OS dependencies that are required, if the ability to process all file types is desired.

requirements/base.in

@@ -1,9 +1,6 @@
 -c constraints.in
 unstructured[all-docs]
-# Pinning click due to a unicode issue in black
-# can remove after black drops support for Python 3.6
-# ref: https://github.com/psf/black/issues/2964
-click==8.2.1
+click
 fastapi
 uvicorn
 ratelimit