unstructured-api/.github/workflows/ci.yml at cd8a7ef3f29ba69842972ff665ac26a05d307fb9 · Unstructured-IO/unstructured-api · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  PYTHON_VERSION: "3.12"
  PIPELINE_FAMILY: "general"

jobs:
  setup:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
    - uses: actions/cache@v5
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/*.txt') }}
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Setup virtual environment (no cache hit)
      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        make install-ci

  lint:
    runs-on: ubuntu-latest
    needs: setup
    steps:
    - uses: actions/checkout@v5
    - uses: actions/cache@v5
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/*.txt') }}
    - name: Lint
      run: |
        source .venv/bin/activate
        make check

  shellcheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v5
      - name: ShellCheck
        uses: ludeeus/action-shellcheck@master

  test:
    runs-on: ubuntu-latest
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v5
    - uses: actions/cache@v5
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Run core tests
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        sudo apt-get update && sudo apt-get install --yes poppler-utils libreoffice
        make install-test
        make install-pandoc
        sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
        tesseract --version
        make test
        make check-coverage

  changelog:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
    - if: github.ref != 'refs/heads/main'
      uses: dorny/paths-filter@v3
      id: changes
      with:
        filters: |
          src:
            - 'doc_recipe/**'
            - 'recipe-notebooks/**'

    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
      uses: dangoslen/changelog-enforcer@v3

  # TODO - figure out best practice for caching docker images
  # (Using the virtualenv to get pytest)
  test_dockerfile:
    runs-on: ubuntu-latest
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v5
    - uses: actions/cache@v5
      id: virtualenv-cache
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Free up disk space
      run: |
        # Clear some space (https://github.com/actions/runner-images/issues/2840)
        echo "Disk usage before cleanup:"
        df -h

        # Remove unnecessary pre-installed software
        sudo rm -rf /usr/share/dotnet
        sudo rm -rf /opt/ghc
        sudo rm -rf /usr/local/share/boost
        sudo rm -rf /usr/local/lib/android
        sudo rm -rf /opt/hostedtoolcache/CodeQL
        sudo rm -rf /usr/local/.ghcup
        sudo rm -rf /usr/share/swift

        # Clean up docker to ensure we start fresh
        docker system prune -af --volumes

        echo "Disk usage after cleanup:"
        df -h
    - name: Test Dockerfile
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        make install-test
        make docker-build
        make docker-test
    # - name: Scan image
    #   uses: anchore/scan-action@v3
    #   with:
    #     image: "pipeline-family-${{ env.PIPELINE_FAMILY }}-dev"
    #     # NOTE(robinson) - revert this to medium when we bump libreoffice
    #     severity-cutoff: critical