Switch to uv Python package installer/resolver (#4005)

[uv](https://github.com/astral-sh/uv) is an extremely fast Python
package installer and resolver, written in Rust, and designed as a
drop-in replacement for pip and pip-tools workflows (see
[post](https://astral.sh/blog/uv))

.drone.yml

@@ -69,7 +69,8 @@ steps:
     commands:
       - apt-get update && apt-get install -y netcat-traditional
       - cd engine/
-      - pip install -r requirements.txt -r requirements-dev.txt
+      - pip install uv
+      - uv pip install -r requirements.txt -r requirements-dev.txt
       - ./wait_for_test_mysql_start.sh && pytest
     depends_on:
       - rabbit_test

.github/workflows/linting-and-tests.yml

@@ -127,8 +127,8 @@ jobs:
         # makemigrations --check = Exit with a non-zero status if model changes are missing migrations
         # and don't actually write them.
         run: |
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt requirements-dev.txt
+          uv pip sync --system requirements.txt requirements-dev.txt
           python manage.py makemigrations --check
           python manage.py lintmigrations
 
@@ -185,8 +185,8 @@ jobs:
         working-directory: engine
         run: |
           apt-get update && apt-get install -y netcat-traditional
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt requirements-dev.txt
+          uv pip sync --system requirements.txt requirements-dev.txt
           ./wait_for_test_mysql_start.sh && pytest -x
 
   unit-test-backend-postgresql-rabbitmq:
@@ -235,8 +235,8 @@ jobs:
       - name: Unit Test Backend
         working-directory: engine
         run: |
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt requirements-dev.txt
+          uv pip sync --system requirements.txt requirements-dev.txt
           pytest -x
 
   unit-test-backend-sqlite-redis:
@@ -275,8 +275,8 @@ jobs:
         working-directory: engine
         run: |
           apt-get update && apt-get install -y netcat-traditional
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt requirements-dev.txt
+          uv pip sync --system requirements.txt requirements-dev.txt
           pytest -x
 
   unit-test-pd-migrator:
@@ -292,8 +292,8 @@ jobs:
       - name: Unit Test PD Migrator
         working-directory: tools/pagerduty-migrator
         run: |
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt
+          uv pip sync --system requirements.txt
           pytest -x
 
   mypy:
@@ -311,8 +311,8 @@ jobs:
       - name: mypy Static Type Checking
         working-directory: engine
         run: |
-          pip install pip-tools
+          pip install uv
-          pip-sync requirements.txt requirements-dev.txt
+          uv pip sync --system requirements.txt requirements-dev.txt
           mypy .
 
   end-to-end-tests:

Makefile

@@ -248,21 +248,21 @@ endef
 
 backend-bootstrap:
 	python3.11 -m venv $(VENV_DIR)
-	$(VENV_DIR)/bin/pip install -U pip wheel pip-tools
+	$(VENV_DIR)/bin/pip install -U pip wheel uv
-	$(VENV_DIR)/bin/pip-sync $(REQUIREMENTS_TXT) $(REQUIREMENTS_DEV_TXT)
+	$(VENV_DIR)/bin/uv pip sync $(REQUIREMENTS_TXT) $(REQUIREMENTS_DEV_TXT)
 	@if [ -f $(REQUIREMENTS_ENTERPRISE_TXT) ]; then \
-		$(VENV_DIR)/bin/pip install -r $(REQUIREMENTS_ENTERPRISE_TXT); \
+		$(VENV_DIR)/bin/uv pip install -r $(REQUIREMENTS_ENTERPRISE_TXT); \
 	fi
 
 backend-migrate:
 	$(call backend_command,python manage.py migrate)
 
 backend-compile-deps:
-	pip-compile --strip-extras $(REQUIREMENTS_IN)
+	uv pip compile --strip-extras $(REQUIREMENTS_IN)
-	pip-compile --strip-extras $(REQUIREMENTS_DEV_IN)
+	uv pip compile --strip-extras $(REQUIREMENTS_DEV_IN)
 
 backend-upgrade-deps:
-	pip-compile --strip-extras --upgrade $(REQUIREMENTS_IN)
+	uv pip compile --strip-extras --upgrade $(REQUIREMENTS_IN)
 
 run-backend-server:
 	$(call backend_command,python manage.py runserver 0.0.0.0:8080)

engine/Dockerfile

@@ -27,10 +27,12 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
     && rm grpcio-1.57.0-cp311-cp311-linux_aarch64.whl; \
     fi
 
+RUN pip install uv
+
 # TODO: figure out how to get this to work.. see comment in .github/workflows/e2e-tests.yml
 # https://stackoverflow.com/a/71846527
 # RUN --mount=type=cache,target=/root/.cache/pip,from=pip_cache pip install -r requirements.txt
-RUN pip install -r requirements.txt
+RUN uv pip install --system -r requirements.txt
 
 # we intentionally have two COPY commands, this is to have the requirements.txt in a separate build step
 # which only invalidates when the requirements.txt actually changes. This avoids having to unneccasrily reinstall deps (which is time-consuming)
@@ -63,13 +65,13 @@ RUN apk add sqlite mysql-client postgresql-client
 # TODO: figure out how to get this to work.. see comment in .github/workflows/e2e-tests.yml
 # https://stackoverflow.com/a/71846527
 # RUN --mount=type=cache,target=/root/.cache/pip,from=pip_cache pip install -r requirements-dev.txt
-RUN pip install -r requirements-dev.txt
+RUN uv pip install --system -r requirements-dev.txt
 
 FROM dev AS dev-enterprise
 # TODO: figure out how to get this to work.. see comment in .github/workflows/e2e-tests.yml
 # https://stackoverflow.com/a/71846527
 # RUN --mount=type=cache,target=/root/.cache/pip,from=pip_cache pip install -r requirements-enterprise-docker.txt
-RUN pip install -r requirements-enterprise-docker.txt
+RUN uv pip install --system -r requirements-enterprise-docker.txt
 
 FROM base AS prod
 

tools/pagerduty-migrator/requirements.in

@@ -0,0 +1,4 @@
+requests==2.31.0
+pdpyras==4.5.0
+pytest==7.1.2
+pytest-env==0.6.2

tools/pagerduty-migrator/requirements.txt

@@ -1,4 +1,40 @@
-requests==2.31.0
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+attrs==23.2.0
+    # via pytest
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+idna==3.6
+    # via requests
+iniconfig==2.0.0
+    # via pytest
+packaging==23.2
+    # via pytest
 pdpyras==4.5.0
+    # via -r requirements.in
+pluggy==1.4.0
+    # via pytest
+py==1.11.0
+    # via pytest
 pytest==7.1.2
-pytest-env==0.6.2
+    # via
+    #   -r requirements.in
+    #   pytest-env
+pytest-env==0.6.2
+    # via -r requirements.in
+requests==2.31.0
+    # via
+    #   -r requirements.in
+    #   pdpyras
+tomli==2.0.1
+    # via pytest
+urllib3==2.2.1
+    # via
+    #   pdpyras
+    #   requests