fix flaky tests + increase runner size when building docker image (#4502)

## Which issue(s) this PR closes

Closes https://github.com/grafana/oncall-private/issues/2733

.github/workflows/build-engine-docker-image-and-publish-to-dockerhub.yml

@@ -13,7 +13,7 @@ on:
 jobs:
   build-engine-docker-image-and-publish-to-dockerhub:
     name: Build engine Docker image and publish to Dockerhub
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     # These permissions are needed to assume roles from Github's OIDC.
     # https://github.com/grafana/shared-workflows/tree/main/actions/build-push-to-dockerhub
     permissions:

.github/workflows/linting-and-tests.yml

@@ -100,7 +100,7 @@ jobs:
 
   unit-test-backend-mysql-rabbitmq:
     name: "Backend Tests: MySQL + RabbitMQ (RBAC enabled: ${{ matrix.rbac_enabled }})"
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest-16-cores
     strategy:
       matrix:
         rbac_enabled: ["True", "False"]
@@ -132,7 +132,7 @@ jobs:
 
   unit-test-backend-postgresql-rabbitmq:
     name: "Backend Tests: PostgreSQL + RabbitMQ (RBAC enabled: ${{ matrix.rbac_enabled }})"
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest-16-cores
     strategy:
       matrix:
         rbac_enabled: ["True", "False"]
@@ -171,7 +171,7 @@ jobs:
 
   unit-test-backend-sqlite-redis:
     name: "Backend Tests: SQLite + Redis (RBAC enabled: ${{ matrix.rbac_enabled }})"
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest-16-cores
     strategy:
       matrix:
         rbac_enabled: ["True", "False"]

.github/workflows/snyk-security-scan.yml

@@ -4,9 +4,17 @@ on:
   workflow_call:
 
 jobs:
-  synk-security-scan:
+  snyk-python-security-scan:
     name: Snyk security scan
     runs-on: ubuntu-latest
+    # see this PR regarding the permissions needed for this workflow
+    # https://github.com/snyk/actions/pull/79
+    permissions:
+      # required for all workflows
+      security-events: write
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python

engine/apps/alerts/tests/test_escalation_snapshot.py

@@ -183,15 +183,24 @@ def test_next_escalation_policy_snapshot(escalation_snapshot_test_setup):
 
 @pytest.mark.django_db
 @pytest.mark.parametrize(
-    "next_step_eta,expected",
+    "timedelta,time_in_past,expected",
     [
-        (None, None),
-        (timezone.now() - timezone.timedelta(weeks=50), False),
-        (timezone.now() - timezone.timedelta(minutes=4), True),
-        (timezone.now() + timezone.timedelta(minutes=4), True),
+        (None, None, None),
+        (timezone.timedelta(weeks=50), True, False),
+        (timezone.timedelta(minutes=4), True, True),
+        (timezone.timedelta(minutes=4), False, True),
     ],
 )
-def test_next_step_eta_is_valid(escalation_snapshot_test_setup, next_step_eta, expected) -> None:
+def test_next_step_eta_is_valid(escalation_snapshot_test_setup, timedelta, time_in_past, expected) -> None:
+    now = timezone.now()
+
+    if timedelta is None:
+        next_step_eta = None
+    elif time_in_past:
+        next_step_eta = now - timedelta
+    else:
+        next_step_eta = now + timedelta
+
     alert_group, _, _, _ = escalation_snapshot_test_setup
     escalation_snapshot = alert_group.escalation_snapshot
 

engine/apps/api/tests/test_user.py

@@ -1,3 +1,4 @@
+from unittest import mock
 from unittest.mock import Mock, patch
 
 import pytest
@@ -58,7 +59,7 @@ def test_current_user(
                 "user": user.username,
             }
         },
-        "cloud_connection_status": 0,
+        "cloud_connection_status": mock.ANY,
         "notification_chain_verbal": {"default": "", "important": ""},
         "slack_user_identity": None,
         "avatar": user.avatar_url,

engine/tox.ini

@@ -10,7 +10,13 @@ banned-modules =
 [pytest]
 # https://pytest-django.readthedocs.io/en/latest/configuring_django.html#order-of-choosing-settings
 # https://pytest-django.readthedocs.io/en/latest/database.html
-# dist=load = "load balance by sending any pending test to any available environment"
+# https://pytest-xdist.readthedocs.io/en/stable/distribution.html#running-tests-across-multiple-cpus
+#
+# dist=loadscope
+# Tests are grouped by module for test functions and by class for test methods.
+# Groups are distributed to available workers as whole units. This guarantees that all tests in a group run in the same
+# process. This can be useful if you have expensive module-level or class-level fixtures. Grouping by class takes
+# priority over grouping by module.
 addopts = -n auto --dist=loadscope --no-migrations --color=yes --showlocals
 # https://pytest-django.readthedocs.io/en/latest/faq.html#my-tests-are-not-being-found-why
 python_files = tests.py test_*.py *_tests.py