unstructured-api/.github/workflows/ci.yml at 996a0addfe14a4cf5b5eb7ac467cff73d4032747 · Unstructured-IO/unstructured-api · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  PYTHON_VERSION: "3.10"
  PIPELINE_FAMILY: "general"

jobs:
  setup:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/cache@v4
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt', 'requirements/test.txt') }}
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Setup virtual environment (no cache hit)
      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        make install-ci

  lint:
    runs-on: ubuntu-latest
    needs: setup
    steps:
    - uses: actions/checkout@v4
    - uses: actions/cache@v4
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt', 'requirements/test.txt') }}
    - name: Lint
      run: |
        source .venv/bin/activate
        make check

  shellcheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: ShellCheck
        uses: ludeeus/action-shellcheck@master

  test:
    runs-on: ubuntu-latest-m
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v4
    - uses: actions/cache@v4
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt', 'requirements/test.txt') }}
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Setup virtual environment (no cache hit)
      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        make install-ci
    - name: Run core tests
      run: |
        source .venv/bin/activate
        sudo apt-get update && sudo apt-get install --yes poppler-utils libreoffice
        make install-pandoc
        sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
        tesseract --version
        make test
        make check-coverage

  changelog:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - if: github.ref != 'refs/heads/main'
      uses: dorny/paths-filter@v3
      id: changes
      with:
        filters: |
          src:
            - 'doc_recipe/**'
            - 'recipe-notebooks/**'

    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
      uses: dangoslen/changelog-enforcer@v3

  # TODO - figure out best practice for caching docker images
  # (Using the virtualenv to get pytest)
  test_dockerfile:
    runs-on: ubuntu-latest-m
    needs: [setup, lint]
    steps:
    - uses: actions/checkout@v4
    - uses: actions/cache@v4
      id: virtualenv-cache
      with:
        path: |
          .venv
        key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/base.txt', 'requirements/test.txt') }}
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Setup virtual environment (no cache hit)
      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
      run: |
        python${{ env.PYTHON_VERSION }} -m venv .venv
        source .venv/bin/activate
        make install-ci
    - name: Test Dockerfile
      run: |
        source .venv/bin/activate
        make docker-build
        make docker-test
    # - name: Scan image
    #   uses: anchore/scan-action@v3
    #   with:
    #     image: "pipeline-family-${{ env.PIPELINE_FAMILY }}-dev"
    #     # NOTE(robinson) - revert this to medium when we bump libreoffice
    #     severity-cutoff: critical