diff --git a/.github/actions/install-frontend-dependencies/action.yml b/.github/actions/install-frontend-dependencies/action.yml new file mode 100644 index 00000000..c89dc0d6 --- /dev/null +++ b/.github/actions/install-frontend-dependencies/action.yml @@ -0,0 +1,21 @@ +name: "Install frontend dependencies" +description: "Setup node + install frontend dependencies" +runs: + using: "composite" + steps: + - uses: actions/setup-node@v3 + with: + node-version: 18.16.0 + cache: "yarn" + cache-dependency-path: grafana-plugin/yarn.lock + - name: Use cached frontend dependencies + id: cache-frontend-dependencies + uses: actions/cache@v3 + with: + path: grafana-plugin/node_modules + key: ${{ runner.os }}-frontend-node-modules-${{ hashFiles('grafana-plugin/yarn.lock') }} + - name: Install frontend dependencies + if: steps.cache-frontend-dependencies.outputs.cache-hit != 'true' + shell: bash + working-directory: grafana-plugin + run: yarn install --frozen-lockfile --prefer-offline --network-timeout 500000 diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 00000000..56ace557 --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,29 @@ +name: "Setup Python" +description: "Setup Python + optionally install dependencies from a set of requirements file(s)" + +inputs: + install-dependencies: + description: "Whether to install dependencies from the Python requirements file(s)" + required: false + default: "true" + python-requirements-paths: + description: "The path(s) to the Python requirements file(s) to install" + required: false + default: "engine/requirements.txt engine/requirements-dev.txt" + +runs: + using: "composite" + steps: + - name: Setup Python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "3.11.4" + cache: "pip" + cache-dependency-path: ${{ inputs.python-requirements-paths }} + - name: Install Python dependencies + if: ${{ inputs.install-dependencies == 'true' }} + shell: bash + run: | + pip install uv + uv pip sync --system ${{ inputs.python-requirements-paths }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index adccedcf..b996a7f6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -59,11 +59,8 @@ jobs: config: ./dev/kind.yml install_only: true - - uses: actions/setup-node@v3 - with: - node-version: 18.16.0 - cache: "yarn" - cache-dependency-path: grafana-plugin/yarn.lock + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies - name: Install Tilt run: | @@ -76,18 +73,6 @@ jobs: curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/$CTLPTL_FILE_NAME | \ tar -xzv -C /usr/local/bin ctlptl - - name: Use cached frontend dependencies - id: cache-frontend-dependencies - uses: actions/cache@v3 - with: - path: grafana-plugin/node_modules - key: ${{ runner.os }}-frontend-node-modules-${{ hashFiles('grafana-plugin/yarn.lock') }} - - - name: Install frontend dependencies - if: steps.cache-frontend-dependencies.outputs.cache-hit != 'true' - working-directory: grafana-plugin - run: yarn install --frozen-lockfile --prefer-offline --network-timeout 500000 - - name: Use cached plugin frontend build id: cache-plugin-frontend uses: actions/cache@v3 diff --git a/.github/workflows/linting-and-tests.yml b/.github/workflows/linting-and-tests.yml index 188a22d3..8dedefed 100644 --- a/.github/workflows/linting-and-tests.yml +++ b/.github/workflows/linting-and-tests.yml @@ -11,6 +11,12 @@ name: Linting and Tests # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions merge_group: +env: + DJANGO_SETTINGS_MODULE: settings.ci-test + DATABASE_HOST: localhost + RABBITMQ_URI: amqp://rabbitmq:rabbitmq@localhost:5672 + SLACK_CLIENT_OAUTH_ID: 1 + concurrency: # Cancel any running workflow for the same branch when new commits are pushed. # We group both by ref_name (available when CI is triggered by a push to a branch/tag) @@ -23,52 +29,24 @@ jobs: name: "Lint entire project" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt - # following 2 steps - need to install the frontend dependencies for the eslint/prettier/stylelint steps - - uses: actions/setup-node@v3 - with: - node-version: 18.16.0 - cache: "yarn" - cache-dependency-path: grafana-plugin/yarn.lock - - name: Use cached frontend dependencies - id: cache-frontend-dependencies - uses: actions/cache@v3 - with: - path: grafana-plugin/node_modules - key: ${{ runner.os }}-frontend-node-modules-${{ hashFiles('grafana-plugin/yarn.lock') }} + install-dependencies: "false" - name: Install frontend dependencies - if: steps.cache-frontend-dependencies.outputs.cache-hit != 'true' - working-directory: grafana-plugin - run: yarn install --frozen-lockfile --prefer-offline --network-timeout 500000 + uses: ./.github/actions/install-frontend-dependencies - uses: pre-commit/action@v3.0.0 lint-test-and-build-frontend: name: "Lint, test, and build frontend" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18.16.0 - cache: "yarn" - cache-dependency-path: grafana-plugin/yarn.lock - - name: Use cached frontend dependencies - id: cache-frontend-dependencies - uses: actions/cache@v3 - with: - path: grafana-plugin/node_modules - key: ${{ runner.os }}-frontend-node-modules-${{ hashFiles('grafana-plugin/yarn.lock') }} + - name: Checkout project + uses: actions/checkout@v3 - name: Install frontend dependencies - if: steps.cache-frontend-dependencies.outputs.cache-hit != 'true' - working-directory: grafana-plugin - run: yarn install --frozen-lockfile --prefer-offline --network-timeout 500000 + uses: ./.github/actions/install-frontend-dependencies - name: Build, lint and test frontend working-directory: grafana-plugin run: yarn lint && yarn test && yarn build @@ -93,11 +71,6 @@ jobs: lint-migrations-backend-mysql-rabbitmq: name: "Lint database migrations" runs-on: ubuntu-latest - env: - DATABASE_HOST: localhost - RABBITMQ_URI: amqp://rabbitmq:rabbitmq@localhost:5672 - DJANGO_SETTINGS_MODULE: settings.ci-test - SLACK_CLIENT_OAUTH_ID: 1 services: rabbit_test: image: rabbitmq:3.12.0 @@ -114,21 +87,15 @@ jobs: ports: - 3306:3306 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Lint migrations working-directory: engine # makemigrations --check = Exit with a non-zero status if model changes are missing migrations # and don't actually write them. run: | - pip install uv - uv pip sync --system requirements.txt requirements-dev.txt python manage.py makemigrations --check python manage.py lintmigrations @@ -136,7 +103,8 @@ jobs: name: "Helm Chart Unit Tests" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout project + uses: actions/checkout@v3 - uses: azure/setup-helm@v3 with: version: v3.8.0 @@ -152,10 +120,6 @@ jobs: matrix: rbac_enabled: ["True", "False"] env: - DJANGO_SETTINGS_MODULE: settings.ci-test - DATABASE_HOST: localhost - RABBITMQ_URI: amqp://rabbitmq:rabbitmq@localhost:5672 - SLACK_CLIENT_OAUTH_ID: 1 ONCALL_TESTING_RBAC_ENABLED: ${{ matrix.rbac_enabled }} services: rabbit_test: @@ -173,20 +137,14 @@ jobs: ports: - 3306:3306 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Unit Test Backend working-directory: engine run: | apt-get update && apt-get install -y netcat-traditional - pip install uv - uv pip sync --system requirements.txt requirements-dev.txt ./wait_for_test_mysql_start.sh && pytest -x unit-test-backend-postgresql-rabbitmq: @@ -197,10 +155,6 @@ jobs: rbac_enabled: ["True", "False"] env: DATABASE_TYPE: postgresql - DATABASE_HOST: localhost - RABBITMQ_URI: amqp://rabbitmq:rabbitmq@localhost:5672 - DJANGO_SETTINGS_MODULE: settings.ci-test - SLACK_CLIENT_OAUTH_ID: 1 ONCALL_TESTING_RBAC_ENABLED: ${{ matrix.rbac_enabled }} services: rabbit_test: @@ -224,20 +178,13 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Unit Test Backend working-directory: engine - run: | - pip install uv - uv pip sync --system requirements.txt requirements-dev.txt - pytest -x + run: pytest -x unit-test-backend-sqlite-redis: name: "Backend Tests: SQLite + Redis (RBAC enabled: ${{ matrix.rbac_enabled }})" @@ -249,8 +196,6 @@ jobs: DATABASE_TYPE: sqlite3 BROKER_TYPE: redis REDIS_URI: redis://localhost:6379 - DJANGO_SETTINGS_MODULE: settings.ci-test - SLACK_CLIENT_OAUTH_ID: 1 ONCALL_TESTING_RBAC_ENABLED: ${{ matrix.rbac_enabled }} services: redis_test: @@ -263,57 +208,41 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Unit Test Backend working-directory: engine run: | apt-get update && apt-get install -y netcat-traditional - pip install uv - uv pip sync --system requirements.txt requirements-dev.txt pytest -x unit-test-migrators: name: "Unit tests - Migrators" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: tools/migrators/requirements.txt + python-requirements-paths: tools/migrators/requirements.txt - name: Unit Test Migrators working-directory: tools/migrators - run: | - pip install uv - uv pip sync --system requirements.txt - pytest -x + run: pytest -x mypy: name: "mypy" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11.4" - cache: "pip" - cache-dependency-path: | - engine/requirements.txt - engine/requirements-dev.txt + - name: Checkout project + uses: actions/checkout@v3 + - name: Setup Python + uses: ./.github/actions/setup-python - name: mypy Static Type Checking working-directory: engine - run: | - pip install uv - uv pip sync --system requirements.txt requirements-dev.txt - mypy . + run: mypy . end-to-end-tests: name: Standard e2e tests