migration from pip to uv

.dockerignore

@@ -4,6 +4,7 @@
 .lgtm.yml
 __pycache__
 venv/
+.venv/
 **/build
 .env
 docker/env_file_*

.github/dependabot.yml

@@ -1,7 +1,7 @@
 version: 2
 updates:
-  - package-ecosystem: "pip"
-    directory: "/requirements"
+  - package-ecosystem: "uv"
+    directory: "/"
     schedule:
       interval: "weekly"
       day: "tuesday"

.github/workflows/codeql-analysis.yml

@@ -43,25 +43,23 @@ jobs:
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
-    - name: Set up Python
-      uses: actions/setup-python@v6.2.0
+    - name: Set up uv
+      uses: astral-sh/setup-uv@v5
       with:
-        python-version: '3.11'
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python
+      run: uv python install 3.11
 
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev libldap2-dev python-dev-is-python3 libfuzzy-dev net-tools git libcurl4-openssl-dev
-        python -m pip install --upgrade pip
-        if [ -f requirements/project-requirements.txt ];
-        then pip install -r requirements/project-requirements.txt;
-        fi
-        if [ -f requirements/certego-requirements.txt ];
-        then pip install -r requirements/certego-requirements.txt;
-        fi
+        uv sync --frozen --group certego
         # Set the `CODEQL-PYTHON` environment variable to the Python executable
         # that includes the dependencies
-        echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
+        echo "CODEQL_PYTHON=$(uv run python -c 'import sys; print(sys.executable)')" >> $GITHUB_ENV
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/pull_request_automation.yml

@@ -36,26 +36,33 @@ jobs:
       - name: Checkout IntelOwl
         uses: actions/checkout@v6.0.2
 
-      - name: Set up Python
-        uses: actions/setup-python@v6.2.0
+      - name: Perform ShellCheck Analysis
+        run: bash <(curl -s https://raw.githubusercontent.com/CICDToolbox/shellcheck/master/pipeline.sh)
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: 3.11
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
 
-      - name: Install Dependencies
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install system dependencies
         run: |
-          pip3 install --upgrade pip
-          pip3 install -r requirements/test-requirements.txt
+          sudo apt-get update
+          sudo apt-get install -y libfuzzy-dev libldap2-dev libsasl2-dev
+
+      - name: Install Dependencies
+        run: uv sync --only-group dev
 
       - name: Lint with Ruff
         run: |
-          ruff check . --output-format=github
+          uv run ruff check . --output-format=github
 
       - name: Check formatting with Ruff
         run: |
-          ruff format . --check --diff
-
-      - name: Perform ShellCheck Analysis
-        run: bash <(curl -s https://raw.githubusercontent.com/CICDToolbox/shellcheck/master/pipeline.sh)
+          uv run ruff format . --check --diff
 
   backend-tests:
     runs-on: ubuntu-latest
@@ -116,7 +123,7 @@ jobs:
 
       - name: Setup coverage
         run: |
-          docker exec intelowl_uwsgi pip3 install coverage
+          docker exec intelowl_uwsgi uv pip install coverage
 
       - name: Run test
         run: |

docker/Dockerfile

@@ -44,16 +44,23 @@ RUN apt-get update \
     vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils tshark  \
     libemail-outlook-message-perl libemail-address-perl \
     && apt-get clean && apt-get autoclean && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip3 install --no-cache-dir --upgrade pip
+    && rm -rf /var/lib/apt/lists/*
+
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# use uv to manage dependencies
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
+ENV UV_PROJECT_ENVIRONMENT=/opt/venv
 
-COPY requirements/project-requirements.txt $PYTHONPATH/project-requirements.txt
-COPY requirements/certego-requirements.txt $PYTHONPATH/certego-requirements.txt
 WORKDIR $PYTHONPATH
+ENV PATH="/opt/venv/bin:$PATH"
+COPY pyproject.toml uv.lock ./
 
-RUN pip3 install --no-cache-dir --use-pep517 --compile -r project-requirements.txt \
-    && pip3 install --no-cache-dir pycti==${PYCTI_VERSION} \
-    && pip3 install --no-cache-dir --compile -r certego-requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --group certego \
+    && uv pip install pycti==${PYCTI_VERSION}
 
 COPY api_app/analyzers_manager/repo_downloader.sh ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh
 

docker/entrypoints/celery_beat.sh

@@ -11,5 +11,5 @@ then
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
   # shellcheck disable=SC2086
-  /usr/local/bin/celery $ARGUMENTS
+  celery $ARGUMENTS
 fi

docker/entrypoints/celery_default.sh

@@ -35,5 +35,5 @@ then
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
   # shellcheck disable=SC2086
-  /usr/local/bin/celery $ARGUMENTS
+  celery $ARGUMENTS
 fi

docker/entrypoints/celery_ingestor.sh

@@ -24,5 +24,5 @@ then
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
   # shellcheck disable=SC2086
-  /usr/local/bin/celery $ARGUMENTS
+  celery $ARGUMENTS
 fi

docker/entrypoints/celery_local.sh

@@ -24,5 +24,5 @@ then
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
   # shellcheck disable=SC2086
-  /usr/local/bin/celery $ARGUMENTS
+  celery $ARGUMENTS
 fi

docker/entrypoints/celery_long.sh

@@ -24,5 +24,5 @@ then
     python3 manage.py celery_reload -c "$ARGUMENTS"
 else
   # shellcheck disable=SC2086
-  /usr/local/bin/celery $ARGUMENTS
+  celery $ARGUMENTS
 fi

docker/entrypoints/daphne.sh

@@ -7,4 +7,4 @@ done
 
 # verbosity param levels: https://github.com/django/daphne/blob/df0680c9ad699817725e18a9264df17fff2927da/daphne/cli.py#L213
 # not useful to improve logging
-/usr/local/bin/daphne --proxy-headers --access-log /var/log/intel_owl/asgi/daphne.log -p 8011 -b 0.0.0.0 --no-server-name --application-close-timeout 60 --ping-interval 30 --ping-timeout 35 intel_owl.asgi:application
+daphne --proxy-headers --access-log /var/log/intel_owl/asgi/daphne.log -p 8011 -b 0.0.0.0 --no-server-name --application-close-timeout 60 --ping-interval 30 --ping-timeout 35 intel_owl.asgi:application

docker/entrypoints/flower.sh

@@ -31,7 +31,7 @@ then
   FLOWER_PWD="flower"
 fi
 
-CMD="/usr/local/bin/celery -A intel_owl.celery --broker ${BROKER_URL} flower --broker_api=${BROKER_URL_API} --max_tasks=1000 --max_workers=500"
+CMD="celery -A intel_owl.celery --broker ${BROKER_URL} flower --broker_api=${BROKER_URL_API} --max_tasks=1000 --max_workers=500"
 htpasswd -cb /opt/deploy/shared_htpasswd/.htpasswd "${FLOWER_USR}" "${FLOWER_PWD}"
 
 if [[ ${DEBUG} == "True" ]] && [[ ${DJANGO_TEST_SERVER} == "True" ]];

docker/entrypoints/uwsgi.sh

@@ -49,5 +49,5 @@ then
 else
     $CHANGELOG_NOTIFICATION_COMMAND
     $ELASTIC_TEMPLATE_COMMAND
-    /usr/local/bin/uwsgi --ini /etc/uwsgi/sites/intel_owl.ini --stats 127.0.0.1:1717 --stats-http
+    uwsgi --ini /etc/uwsgi/sites/intel_owl.ini --stats 127.0.0.1:1717 --stats-http
 fi

integrations/bbot/Dockerfile

@@ -1,4 +1,5 @@
 FROM python:3.12-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 # Environment variables
 ENV PROJECT_PATH=/opt/deploy/bbot
@@ -17,9 +18,8 @@ WORKDIR ${PROJECT_PATH}
 # Copy application files and requirements
 COPY requirements.txt app.py ./
 
-# Upgrade pip and install Python packages
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r requirements.txt
+# Install Python packages
+RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt
 
 # Pre-install BBOT dependencies & Make script executable
 RUN bbot --install-all-deps -y --force && \

integrations/malware_tools_analyzers/Dockerfile

@@ -1,4 +1,5 @@
 FROM python:3.11-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 ARG TARGETARCH
 
@@ -38,59 +39,55 @@ RUN if [[ $TARGETARCH == "amd64" ]]; \
     && ln -s ${PROJECT_PATH}/goresym/GoReSym /usr/local/bin/goresym
 
 # Build Mandiant's Stringsifter
-WORKDIR ${PROJECT_PATH}/stringsifter
-COPY requirements/stringsifter-requirements.txt stringsifter/wrapper.py ./
+WORKDIR ${PYTHONPATH}/stringsifter
+COPY ./requirements/stringsifter-requirements.txt ./stringsifter/wrapper.py ./
 # Installed at system level because virtualenv does not work as expected
 # ModuleNotFoundError: No module named 'pybind11' while trying to build
-RUN pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r stringsifter-requirements.txt \
+RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r stringsifter-requirements.txt \
     && chmod +x wrapper.py
 
 # Build Qiling
-WORKDIR ${PROJECT_PATH}/qiling
-COPY requirements/qiling-requirements.txt qiling/analyze.py ./
+WORKDIR ${PYTHONPATH}/qiling
+COPY ./requirements/qiling-requirements.txt ./qiling/analyze.py ./
 # keystone-engine does not compile for ARM
-RUN if [[ $TARGETARCH == "amd64" ]]; then \
+RUN --mount=type=cache,target=/root/.cache/uv if [[ $TARGETARCH == "amd64" ]]; then \
     python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r qiling-requirements.txt; fi
+    && uv pip install -r qiling-requirements.txt; fi
 
 # Then Build every possible other Python application inside its virtual environment
 # Build guelfo's PEFrame
-WORKDIR ${PROJECT_PATH}/peframe
-COPY requirements/peframe-requirements.txt ./
-RUN python3 -m venv venv \
+WORKDIR ${PYTHONPATH}/peframe
+COPY ./requirements/peframe-requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r peframe-requirements.txt --no-cache-dir
+    && uv pip install -r peframe-requirements.txt --no-cache-dir
 
 # Install guelfo's artifacts
 # there is no version management on this project so we just pull the most recent changes
-WORKDIR ${PROJECT_PATH}/artifacts
-RUN python3 -m venv venv \
+WORKDIR ${PYTHONPATH}/artifacts
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
+
     && git clone https://github.com/guelfoweb/artifacts.git \
     && cd artifacts \
-    && pip install --no-cache-dir -r requirements.txt \
+    && uv pip install -r requirements.txt \
     && chmod +x artifacts.py
 
 # Build APKiD
-WORKDIR ${PROJECT_PATH}/apkid
-COPY requirements/apkid-requirements.txt ./
-RUN python3 -m venv venv \
+WORKDIR ${PYTHONPATH}/apkid
+COPY ./requirements/apkid-requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r apkid-requirements.txt
+    && uv pip install -r apkid-requirements.txt
 
 # Install DroidLysis
-WORKDIR "${PROJECT_PATH}/droidlysis"
-COPY requirements/droidlysis-requirements.txt ./
-RUN python3 -m venv venv \
+WORKDIR "${PYTHONPATH}/droidlysis"
+COPY ./requirements/droidlysis-requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
  && . venv/bin/activate \
- && pip3 install --no-cache-dir --upgrade pip \
- && pip3 install --no-cache-dir -r droidlysis-requirements.txt --no-cache-dir \
+
+ && uv pip install -r droidlysis-requirements.txt --no-cache-dir \
  && mkdir -p ~/softs \
  && cd ~/softs \
  && wget -q https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.9.3.jar \
@@ -99,21 +96,20 @@ RUN python3 -m venv venv \
  && unzip dex-tools-v2.4.zip \
  && rm -f dex-tools-v2.4.zip \
  && apt-get -y install --no-install-recommends openjdk-21-jdk \
- && mkdir -p "${PROJECT_PATH}/droidlysis/out" "${PROJECT_PATH}/droidlysis/conf" /root/.cache/droidlysis \
+ && mkdir -p "${PYTHONPATH}/droidlysis/out" "${PYTHONPATH}/droidlysis/conf" /root/.cache/droidlysis \
  && chown -R ${USER}:${USER} /root/.cache/droidlysis \
  && chmod 711 /root/ \
- && DROIDLYSIS_PATH="$(pip3 show droidlysis | grep Location | cut -d ' ' -f 2)" \
- && cp "${DROIDLYSIS_PATH}/conf/"*.conf "${PROJECT_PATH}/droidlysis/conf/" \
- && rm -f "${PROJECT_PATH}/droidlysis/conf/general.conf"
-COPY ./droidlysis/general.conf ${PROJECT_PATH}/droidlysis/conf/general.conf
+ && DROIDLYSIS_PATH="$(uv pip show droidlysis | grep Location | cut -d ' ' -f 2)" \
+ && cp "${DROIDLYSIS_PATH}/conf/"*.conf "${PYTHONPATH}/droidlysis/conf/" \
+ && rm -f "${PYTHONPATH}/droidlysis/conf/general.conf"
+COPY ./droidlysis/general.conf ${PYTHONPATH}/droidlysis/conf/general.conf
 
 # MobSF
-WORKDIR ${PROJECT_PATH}/mobsf
-COPY requirements/mobsf-requirements.txt ./
-RUN python3 -m venv venv \
+WORKDIR ${PYTHONPATH}/mobsf
+COPY ./requirements/mobsf-requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r mobsf-requirements.txt \
+    && uv pip install -r mobsf-requirements.txt \
     && mkdir -p /root/.semgrep/ \
     && chown -R ${USER}:${USER} /root/.semgrep \
     && chmod 711 /root
@@ -126,12 +122,11 @@ RUN mkdir -m 0770 -p /var/lib/fangfrisch \
     && chmod 0644 /etc/cron.d/crontab /var/log/cron.log
 
 # Build Flask REST API
-WORKDIR ${PROJECT_PATH}/flask
-COPY app.py requirements/flask-requirements.txt entrypoint.sh ./
-RUN python3 -m venv venv \
+WORKDIR ${PYTHONPATH}/flask
+COPY app.py ./requirements/flask-requirements.txt entrypoint.sh ./
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r flask-requirements.txt \
+    && uv pip install -r flask-requirements.txt \
     && chmod +x entrypoint.sh
 
 # Cleanup

integrations/nuclei_analyzer/Dockerfile

@@ -1,4 +1,5 @@
 FROM projectdiscovery/nuclei:v3.7.0
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 ENV LOG_PATH=/var/log/intel_owl/nuclei_analyzer
 ENV USER=nuclei-user
@@ -15,10 +16,9 @@ WORKDIR /app
 
 # Copy and install requirements first (better layer caching)
 COPY app.py requirements.txt entrypoint.sh ./
-RUN python3 -m venv venv \
+RUN --mount=type=cache,target=/root/.cache/uv python3 -m venv venv \
     && . venv/bin/activate \
-    && pip3 install --no-cache-dir --upgrade pip \
-    && pip3 install --no-cache-dir -r requirements.txt \
+    && uv pip install -r requirements.txt \
     && rm -rf ~/.cache/pip/*
 
 # Create log directory with proper permissions

integrations/pcap_analyzers/Dockerfile

@@ -1,5 +1,6 @@
 # https://github.com/jasonish/docker-suricata/tree/master/7.0
 FROM jasonish/suricata:7.0
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 ENV PROJECT_PATH=/opt/deploy
 ENV LOG_PATH=/var/log/intel_owl/pcap_analyzers
@@ -18,7 +19,7 @@ COPY crontab /etc/cron.d/suricata
 COPY config/suricata/etc/* /etc/suricata/
 COPY config/suricata/rules/* /var/lib/suricata/rules/
 
-RUN pip3 install -r requirements.txt --no-cache-dir \
+RUN --mount=type=cache,target=/root/.cache/uv uv pip install --system -r requirements.txt \
     && chown -R ${USER}:${USER} . /etc/suricata /var/lib/suricata \
     && mkdir -p ${LOG_PATH}/suricata/ \
     && touch /var/log/cron.log ${LOG_PATH}/suricata/suricata.log \