From 67465168b8551ebdc886013268abae550d20c564 Mon Sep 17 00:00:00 2001 From: Manu Vamadevan Date: Mon, 8 Aug 2022 20:17:31 +0200 Subject: [PATCH 01/18] 343: node selection for helm chart - engine only - Initial commit --- helm/oncall/templates/engine/deployment.yaml | 12 ++++++++++++ helm/oncall/values.yaml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/helm/oncall/templates/engine/deployment.yaml b/helm/oncall/templates/engine/deployment.yaml index 9c9d0f76..262b403a 100644 --- a/helm/oncall/templates/engine/deployment.yaml +++ b/helm/oncall/templates/engine/deployment.yaml @@ -71,3 +71,15 @@ spec: timeoutSeconds: 3 resources: {{- toYaml .Values.engine.resources | nindent 12 }} + {{- with .Values.grafana.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.grafana.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.grafana.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 95ab565e..d49fd25b 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -161,6 +161,18 @@ grafana: enabled: true plugins: - grafana-oncall-app + ## @param affinity Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: { } + ## @param nodeSelector Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: { } + ## @param tolerations Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [ ] nameOverride: "" fullnameOverride: "" From 9495ee9f140d570a08d406d0ada2aede576e23b0 Mon Sep 17 00:00:00 2001 From: Manu Vamadevan Date: Mon, 8 Aug 2022 23:25:00 +0200 Subject: [PATCH 02/18] 343: node selection for helm chart - engine only - corrected config positioning --- helm/oncall/templates/engine/deployment.yaml | 6 ++--- helm/oncall/values.yaml | 24 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/helm/oncall/templates/engine/deployment.yaml b/helm/oncall/templates/engine/deployment.yaml index 262b403a..dc836feb 100644 --- a/helm/oncall/templates/engine/deployment.yaml +++ b/helm/oncall/templates/engine/deployment.yaml @@ -71,15 +71,15 @@ spec: timeoutSeconds: 3 resources: {{- toYaml .Values.engine.resources | nindent 12 }} - {{- with .Values.grafana.nodeSelector }} + {{- with .Values.engine.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.grafana.affinity }} + {{- with .Values.engine.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.grafana.tolerations }} + {{- with .Values.engine.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index d49fd25b..9d753f83 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -28,6 +28,18 @@ engine: # requests: # cpu: 100m # memory: 128Mi + ## @param affinity Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: { } + ## @param nodeSelector Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: { } + ## @param tolerations Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [ ] # Celery workers pods configuration celery: @@ -161,18 +173,6 @@ grafana: enabled: true plugins: - grafana-oncall-app - ## @param affinity Affinity for pod assignment - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## - affinity: { } - ## @param nodeSelector Node labels for pod assignment - ## ref: https://kubernetes.io/docs/user-guide/node-selection/ - ## - nodeSelector: { } - ## @param tolerations Tolerations for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [ ] nameOverride: "" fullnameOverride: "" From b9b36b3393933f7623002f018dc321d4bca0dfb2 Mon Sep 17 00:00:00 2001 From: Manu Vamadevan Date: Tue, 9 Aug 2022 10:08:27 +0200 Subject: [PATCH 03/18] 343: node selection for helm chart - engine only - added blank line for seperation --- helm/oncall/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 9d753f83..27af56e5 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -28,6 +28,7 @@ engine: # requests: # cpu: 100m # memory: 128Mi + ## @param affinity Affinity for pod assignment ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## From b1ea2b062f6b2912c4c05c5c9138e5ecceb705e9 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 13:19:12 +0300 Subject: [PATCH 04/18] Fix shift update for web schedules --- .../schedules/models/custom_on_call_shift.py | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index c4558b12..225ec544 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -433,6 +433,32 @@ class CustomOnCallShift(models.Model): return return next_event_dt + def get_last_event_date(self, date): + """Get start date of the last event before the chosen date""" + assert date >= self.start, "Chosen date should be later or equal to initial event start date" + + event_ical = self.generate_ical(self.start) + initial_event = Event.from_ical(event_ical) + # take shift interval, not event interval. For rolling_users shift it is not the same. + interval = self.interval or 1 + initial_event["rrule"]["INTERVAL"] = interval + initial_event_start = initial_event["DTSTART"].dt + + last_event = None + # repetitions generate the next event shift according with the recurrence rules + repetitions = UnfoldableCalendar(initial_event).RepeatedEvent( + initial_event, initial_event_start.replace(microsecond=0) + ) + ical_iter = repetitions.__iter__() + for event in ical_iter: + if event.start > date: + break + last_event = event + + last_event_dt = last_event.start if last_event else initial_event_start + + return last_event_dt + @cached_property def event_ical_rules(self): # e.g. {'freq': ['WEEKLY'], 'interval': [2], 'byday': ['MO', 'WE', 'FR'], 'wkst': ['SU']} @@ -498,10 +524,9 @@ class CustomOnCallShift(models.Model): self.rolling_users = result self.save(update_fields=["rolling_users"]) - def get_rotation_user_index(self, date=None): + def get_rotation_user_index(self, date): START_ROTATION_INDEX = 0 - date = timezone.now() if not date else date result = START_ROTATION_INDEX if not self.rolling_users or self.frequency is None: @@ -544,8 +569,9 @@ class CustomOnCallShift(models.Model): return last_shift def create_or_update_last_shift(self, data): + now = timezone.now().replace(microsecond=0) # rotation start date cannot be earlier than now - data["rotation_start"] = max(data["rotation_start"], timezone.now().replace(microsecond=0)) + data["rotation_start"] = max(data["rotation_start"], now) # prepare dict with params of existing instance with last updates and remove unique and m2m fields from it shift_to_update = self.last_updated_shift or self instance_data = model_to_dict(shift_to_update) @@ -557,12 +583,10 @@ class CustomOnCallShift(models.Model): instance_data["schedule"] = self.schedule instance_data["team"] = self.team # set new event start date to keep rotation index - instance_data["start"] = timezone.datetime.combine( - instance_data["rotation_start"].date(), - instance_data["start"].time(), - ).astimezone(pytz.UTC) + if instance_data["start"] == self.start: + instance_data["start"] = self.get_last_event_date(now) # calculate rotation index to keep user rotation order - start_rotation_from_user_index = self.get_rotation_user_index() + (self.start_rotation_from_user_index or 0) + start_rotation_from_user_index = self.get_rotation_user_index(now) + (self.start_rotation_from_user_index or 0) if start_rotation_from_user_index >= len(instance_data["rolling_users"]): start_rotation_from_user_index = 0 instance_data["start_rotation_from_user_index"] = start_rotation_from_user_index From d9609dbcc251ccc25c9b2f0fd85da58d9d302be0 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 13:20:28 +0300 Subject: [PATCH 05/18] Fix priority level regex, fix getting shifts without duration --- engine/apps/schedules/constants.py | 2 +- engine/apps/schedules/ical_utils.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/engine/apps/schedules/constants.py b/engine/apps/schedules/constants.py index 719aa0b2..a2ec8adc 100644 --- a/engine/apps/schedules/constants.py +++ b/engine/apps/schedules/constants.py @@ -9,6 +9,6 @@ ICAL_ATTENDEE = "ATTENDEE" ICAL_UID = "UID" ICAL_RRULE = "RRULE" ICAL_UNTIL = "UNTIL" -RE_PRIORITY = re.compile(r"^\[L(\d)\]") +RE_PRIORITY = re.compile(r"^\[L(\d+)\]") RE_EVENT_UID_V1 = re.compile(r"amixr-([\w\d-]+)-U(\d+)-E(\d+)-S(\d+)") RE_EVENT_UID_V2 = re.compile(r"oncall-([\w\d-]+)-PK([\w\d]+)-U(\d+)-E(\d+)-S(\d+)") diff --git a/engine/apps/schedules/ical_utils.py b/engine/apps/schedules/ical_utils.py index 12fb5bd1..620e4450 100644 --- a/engine/apps/schedules/ical_utils.py +++ b/engine/apps/schedules/ical_utils.py @@ -190,18 +190,19 @@ def get_shifts_dict(calendar, calendar_type, schedule, datetime_start, datetime_ ) else: start, end = ical_events.get_start_and_end_with_respect_to_event_type(event) - result_datetime.append( - { - "start": start.astimezone(pytz.UTC), - "end": end.astimezone(pytz.UTC), - "users": users, - "missing_users": missing_users, - "priority": priority, - "source": source, - "calendar_type": calendar_type, - "shift_pk": pk, - } - ) + if start < end: + result_datetime.append( + { + "start": start.astimezone(pytz.UTC), + "end": end.astimezone(pytz.UTC), + "users": users, + "missing_users": missing_users, + "priority": priority, + "source": source, + "calendar_type": calendar_type, + "shift_pk": pk, + } + ) return result_datetime, result_date From 9b9470b358ab3a631829f1f9ff136254472c7b80 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 13:33:33 +0300 Subject: [PATCH 06/18] Fix shift update for web schedules --- engine/apps/schedules/models/custom_on_call_shift.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index 225ec544..232c824a 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -441,7 +441,9 @@ class CustomOnCallShift(models.Model): initial_event = Event.from_ical(event_ical) # take shift interval, not event interval. For rolling_users shift it is not the same. interval = self.interval or 1 - initial_event["rrule"]["INTERVAL"] = interval + if "rrule" in initial_event: + # means that shift has frequency + initial_event["rrule"]["INTERVAL"] = interval initial_event_start = initial_event["DTSTART"].dt last_event = None From 7571bfa62521ed0d419baf29a691355a59c48524 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 14:13:47 +0300 Subject: [PATCH 07/18] Fix tests for shift update --- engine/apps/api/tests/test_oncall_shift.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/engine/apps/api/tests/test_oncall_shift.py b/engine/apps/api/tests/test_oncall_shift.py index ddd21498..efa2fb96 100644 --- a/engine/apps/api/tests/test_oncall_shift.py +++ b/engine/apps/api/tests/test_oncall_shift.py @@ -412,8 +412,9 @@ def test_update_old_on_call_shift_with_future_version( token, user1, user2, organization, schedule = on_call_shift_internal_api_setup client = APIClient() - start_date = (timezone.now() - timezone.timedelta(days=3)).replace(microsecond=0) - next_rotation_start_date = start_date + timezone.timedelta(days=5) + now = timezone.now().replace(microsecond=0) + start_date = now - timezone.timedelta(days=3) + next_rotation_start_date = now + timezone.timedelta(days=1) updated_duration = timezone.timedelta(hours=4) title = "Test Shift Rotation" @@ -422,10 +423,11 @@ def test_update_old_on_call_shift_with_future_version( shift_type=CustomOnCallShift.TYPE_ROLLING_USERS_EVENT, schedule=schedule, title=title, - start=start_date, + start=next_rotation_start_date, duration=timezone.timedelta(hours=3), rotation_start=next_rotation_start_date, rolling_users=[{user1.pk: user1.public_primary_key}], + frequency=CustomOnCallShift.FREQUENCY_DAILY, ) old_on_call_shift = make_on_call_shift( schedule.organization, @@ -438,6 +440,7 @@ def test_update_old_on_call_shift_with_future_version( until=next_rotation_start_date, rolling_users=[{user1.pk: user1.public_primary_key}], updated_shift=new_on_call_shift, + frequency=CustomOnCallShift.FREQUENCY_DAILY, ) # update shift_end and priority_level data_to_update = { @@ -445,9 +448,9 @@ def test_update_old_on_call_shift_with_future_version( "priority_level": 2, "shift_start": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "shift_end": (start_date + updated_duration).strftime("%Y-%m-%dT%H:%M:%SZ"), - "rotation_start": next_rotation_start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + "rotation_start": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "until": None, - "frequency": None, + "frequency": CustomOnCallShift.FREQUENCY_DAILY, "interval": None, "by_day": None, "rolling_users": [[user1.public_primary_key]], @@ -461,27 +464,28 @@ def test_update_old_on_call_shift_with_future_version( url = reverse("api-internal:oncall_shifts-detail", kwargs={"pk": old_on_call_shift.public_primary_key}) response = client.put(url, data=data_to_update, format="json", **make_user_auth_headers(user1, token)) + response_data = response.json() - next_shift_start_date = timezone.datetime.combine(next_rotation_start_date.date(), start_date.time()).astimezone( - timezone.pytz.UTC - ) + for key in ["shift_start", "shift_end", "rotation_start"]: + data_to_update.pop(key) + response_data.pop(key) expected_payload = data_to_update | { "id": new_on_call_shift.public_primary_key, "type": CustomOnCallShift.TYPE_ROLLING_USERS_EVENT, "schedule": schedule.public_primary_key, "updated_shift": None, - "shift_start": next_shift_start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), - "shift_end": (next_shift_start_date + updated_duration).strftime("%Y-%m-%dT%H:%M:%SZ"), } assert response.status_code == status.HTTP_200_OK assert response.json() == expected_payload - new_on_call_shift.refresh_from_db() - # check if the newest version of shift was changed assert old_on_call_shift.duration != updated_duration assert old_on_call_shift.priority_level != data_to_update["priority_level"] + new_on_call_shift.refresh_from_db() + # check if the newest version of shift was changed + assert new_on_call_shift.start - now < timezone.timedelta(minutes=1) + assert new_on_call_shift.rotation_start - now < timezone.timedelta(minutes=1) assert new_on_call_shift.duration == updated_duration assert new_on_call_shift.priority_level == data_to_update["priority_level"] From edba707b42ad6a0159035a81ce4c4236df1f219a Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 14:19:32 +0300 Subject: [PATCH 08/18] Fix priority level test --- engine/apps/slack/tests/test_parse_slack_usernames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/apps/slack/tests/test_parse_slack_usernames.py b/engine/apps/slack/tests/test_parse_slack_usernames.py index df43c950..659cc27a 100644 --- a/engine/apps/slack/tests/test_parse_slack_usernames.py +++ b/engine/apps/slack/tests/test_parse_slack_usernames.py @@ -52,5 +52,5 @@ def test_remove_priority_from_username(): assert parse_username_from_string("[L1] bob") == "bob" assert parse_username_from_string(" [L1] bob ") == "bob" assert parse_username_from_string("[L2] bob[L1]") == "bob[L1]" - assert parse_username_from_string("[L27]bob") == "[L27]bob" + assert parse_username_from_string("[L27]bob") == "bob" assert parse_username_from_string("[[L2]] bob[[[L1]") == "[[L2]] bob[[[L1]" From 996e6076ab84376eb7a9911c7419c53e76ffa2b7 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 20 Sep 2022 16:42:58 +0300 Subject: [PATCH 09/18] Remove unnecessary variable --- engine/apps/schedules/models/custom_on_call_shift.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index 232c824a..3c7c41e9 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -411,8 +411,7 @@ class CustomOnCallShift(models.Model): repetitions = UnfoldableCalendar(current_event).RepeatedEvent( current_event, next_event_start.replace(microsecond=0) ) - ical_iter = repetitions.__iter__() - for event in ical_iter: + for event in repetitions.__iter__(): if end_date: # end_date exists for long events with frequency weekly and monthly if end_date >= event.start >= next_event_start: if ( @@ -451,8 +450,7 @@ class CustomOnCallShift(models.Model): repetitions = UnfoldableCalendar(initial_event).RepeatedEvent( initial_event, initial_event_start.replace(microsecond=0) ) - ical_iter = repetitions.__iter__() - for event in ical_iter: + for event in repetitions.__iter__(): if event.start > date: break last_event = event From 2355fa3a3ce37380f56a33266ae0c1841feba73b Mon Sep 17 00:00:00 2001 From: toro_ponz Date: Sun, 2 Oct 2022 00:45:01 +0900 Subject: [PATCH 10/18] fix failure of "Format Alert" button on Slack. --- engine/apps/slack/scenarios/alertgroup_appearance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/apps/slack/scenarios/alertgroup_appearance.py b/engine/apps/slack/scenarios/alertgroup_appearance.py index 7b772a0e..c4c4236b 100644 --- a/engine/apps/slack/scenarios/alertgroup_appearance.py +++ b/engine/apps/slack/scenarios/alertgroup_appearance.py @@ -57,7 +57,8 @@ class OpenAlertAppearanceDialogStep( # This is a special case for amazon sns notifications in str format CHEKED if ( - AlertReceiveChannel.INTEGRATION_AMAZON_SNS is not None + hasattr(AlertReceiveChannel, "INTEGRATION_AMAZON_SNS") + and AlertReceiveChannel.INTEGRATION_AMAZON_SNS is not None and alert_group.channel.integration == AlertReceiveChannel.INTEGRATION_AMAZON_SNS and raw_request_data == "{}" ): From a8a88f5c00b763753c6392d7b5e26494d47776ae Mon Sep 17 00:00:00 2001 From: toro_ponz Date: Mon, 3 Oct 2022 18:54:56 +0900 Subject: [PATCH 11/18] remove wasted condition check. --- engine/apps/slack/scenarios/alertgroup_appearance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/apps/slack/scenarios/alertgroup_appearance.py b/engine/apps/slack/scenarios/alertgroup_appearance.py index c4c4236b..8a335fcd 100644 --- a/engine/apps/slack/scenarios/alertgroup_appearance.py +++ b/engine/apps/slack/scenarios/alertgroup_appearance.py @@ -58,7 +58,6 @@ class OpenAlertAppearanceDialogStep( # This is a special case for amazon sns notifications in str format CHEKED if ( hasattr(AlertReceiveChannel, "INTEGRATION_AMAZON_SNS") - and AlertReceiveChannel.INTEGRATION_AMAZON_SNS is not None and alert_group.channel.integration == AlertReceiveChannel.INTEGRATION_AMAZON_SNS and raw_request_data == "{}" ): From b84b174e2087b15be880145dc1bde7b37976b7c9 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Tue, 4 Oct 2022 09:25:53 +0100 Subject: [PATCH 12/18] Allow multiple database and celery broker types (#582) * add libs for celery + redis * move redis & cache config to settings/base.py * move rmq & celery config to settings/base.py * BROKER -> BROKER_TYPE * allow multiple database types * flake8 * add sqlite db creation to dockerfile * fix ci * fix ci * debug * remove some defaults * remove prints * use local memory as cache on ci * debug * add DATABASE_DEFAULTS * add ci test for sqlite + redis * add ci test for sqlite + redis * add ci test for sqlite + redis * debug * add redis healthcheck * fix sqlite * fix dev settings * refactor dev settings * tweak ci settings * clear cache properly between tests * move db and broker types to constants * add librabbitmq deps * use amqp instead of librabbitmq --- .github/workflows/ci.yml | 32 ++++- .gitignore | 1 + DEVELOPER.md | 4 +- engine/Dockerfile | 7 +- .../apps/integrations/tests/test_ratelimit.py | 9 +- engine/requirements.txt | 4 +- engine/settings/base.py | 130 ++++++++++++++++-- engine/settings/ci-test.py | 44 +++--- engine/settings/dev.py | 59 +++----- engine/settings/helm.py | 62 +-------- engine/settings/hobby.py | 35 +---- engine/settings/prod_without_db.py | 32 ----- 12 files changed, 204 insertions(+), 215 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d122096..e4b913c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: run: | docker run -v ${PWD}/docs/sources:/hugo/content/docs/oncall/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo' - unit-test-backend: + unit-test-backend-mysql-rabbitmq: runs-on: ubuntu-latest container: python:3.9 env: @@ -66,11 +66,11 @@ jobs: pip install -r requirements.txt ./wait_for_test_mysql_start.sh && pytest --ds=settings.ci-test -x - unit-test-backend-postgresql: + unit-test-backend-postgresql-rabbitmq: runs-on: ubuntu-latest container: python:3.9 env: - DB_BACKEND: postgresql + DATABASE_TYPE: postgresql DJANGO_SETTINGS_MODULE: settings.ci-test SLACK_CLIENT_OAUTH_ID: 1 services: @@ -98,3 +98,29 @@ jobs: pip install -r requirements.txt pytest --ds=settings.ci-test -x + unit-test-backend-sqlite-redis: + runs-on: ubuntu-latest + container: python:3.9 + env: + DATABASE_TYPE: sqlite3 + BROKER_TYPE: redis + REDIS_URI: redis://redis_test:6379 + DJANGO_SETTINGS_MODULE: settings.ci-test + SLACK_CLIENT_OAUTH_ID: 1 + services: + redis_test: + image: redis:7.0.5 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - name: Unit Test Backend + run: | + apt-get update && apt-get install -y netcat + cd engine/ + pip install -r requirements.txt + pytest --ds=settings.ci-test -x diff --git a/.gitignore b/.gitignore index 308f671f..d0748610 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Backend */db.sqlite3 +engine/oncall_dev.db *.pyc venv .python-version diff --git a/DEVELOPER.md b/DEVELOPER.md index fd71c96b..347f7e95 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -59,8 +59,8 @@ pip install -U pip wheel # Copy and check .env.dev file. cp .env.dev.example .env.dev -# NOTE: if you want to use the PostgreSQL db backend add DB_BACKEND=postgresql to your .env.dev file; -# currently allowed backend values are `mysql` (default) and `postgresql` +# NOTE: if you want to use the PostgreSQL db backend add DATABASE_TYPE=postgresql to your .env.dev file; +# currently allowed backend values are `mysql` (default), `postgresql` and `sqlite3` # Apply .env.dev to current terminal. # For PyCharm it's better to use https://plugins.jetbrains.com/plugin/7861-envfile/ diff --git a/engine/Dockerfile b/engine/Dockerfile index 4a736620..8a72ef39 100644 --- a/engine/Dockerfile +++ b/engine/Dockerfile @@ -9,8 +9,11 @@ RUN pip install -r requirements.txt COPY ./ ./ -RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" TELEGRAM_TOKEN="0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" SLACK_CLIENT_OAUTH_ID=1 python manage.py collectstatic --no-input -RUN rm db.sqlite3 +# Collect static files and create an SQLite database +RUN mkdir -p /var/lib/oncall +RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db DATABASE_TYPE=sqlite3 DATABASE_NAME=/var/lib/oncall/oncall.db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" python manage.py collectstatic --no-input +RUN chown -R 1000:2000 /var/lib/oncall + # This is required for prometheus_client to sync between uwsgi workers RUN mkdir -p /tmp/prometheus_django_metrics; diff --git a/engine/apps/integrations/tests/test_ratelimit.py b/engine/apps/integrations/tests/test_ratelimit.py index 75e3d903..97b56937 100644 --- a/engine/apps/integrations/tests/test_ratelimit.py +++ b/engine/apps/integrations/tests/test_ratelimit.py @@ -8,12 +8,9 @@ from django.urls import reverse from apps.alerts.models import AlertReceiveChannel -# Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent. -def setup_module(module): - cache.clear() - - -def teardown_module(module): +@pytest.fixture(autouse=True) +def clear_cache(): + # Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent. cache.clear() diff --git a/engine/requirements.txt b/engine/requirements.txt index 1bf66e51..deb82e56 100644 --- a/engine/requirements.txt +++ b/engine/requirements.txt @@ -5,8 +5,8 @@ whitenoise==5.3.0 twilio~=6.37.0 phonenumbers==8.10.0 django-ordered-model==3.1.1 -celery==5.2.7 -redis==3.2.0 +celery[amqp,redis]==5.2.7 +redis==3.4.1 humanize==0.5.1 uwsgi==2.0.20 django-cors-headers==3.7.0 diff --git a/engine/settings/base.py b/engine/settings/base.py index 3f893246..d9ec9f36 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -64,9 +64,6 @@ TWILIO_VERIFY_SERVICE_SID = os.environ.get("TWILIO_VERIFY_SERVICE_SID") TELEGRAM_WEBHOOK_HOST = os.environ.get("TELEGRAM_WEBHOOK_HOST", BASE_URL) TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN") -os.environ.setdefault("MYSQL_PASSWORD", "empty") -os.environ.setdefault("RABBIT_URI", "empty") - # For Sending email SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY") SENDGRID_FROM_EMAIL = os.environ.get("SENDGRID_FROM_EMAIL") @@ -84,21 +81,101 @@ GRAFANA_CLOUD_ONCALL_TOKEN = os.environ.get("GRAFANA_CLOUD_ONCALL_TOKEN", None) # Outgoing webhook settings DANGEROUS_WEBHOOKS_ENABLED = getenv_boolean("DANGEROUS_WEBHOOKS_ENABLED", default=False) -# DB backend defaults -DB_BACKEND = os.environ.get("DB_BACKEND", "mysql") -DB_BACKEND_DEFAULT_VALUES = { - "mysql": { + +# Database +class DatabaseTypes: + MYSQL = "mysql" + POSTGRESQL = "postgresql" + SQLITE3 = "sqlite3" + + +DATABASE_DEFAULTS = { + DatabaseTypes.MYSQL: { "USER": "root", - "PORT": "3306", + "PORT": 3306, + }, + DatabaseTypes.POSTGRESQL: { + "USER": "postgres", + "PORT": 5432, + }, +} + +DATABASE_NAME = os.getenv("DATABASE_NAME") or os.getenv("MYSQL_DB_NAME") +DATABASE_USER = os.getenv("DATABASE_USER") or os.getenv("MYSQL_USER") +DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD") or os.getenv("MYSQL_PASSWORD") +DATABASE_HOST = os.getenv("DATABASE_HOST") or os.getenv("MYSQL_HOST") +DATABASE_PORT = os.getenv("DATABASE_PORT") or os.getenv("MYSQL_PORT") + +DATABASE_TYPE = os.getenv("DATABASE_TYPE", DatabaseTypes.MYSQL).lower() +assert DATABASE_TYPE in {DatabaseTypes.MYSQL, DatabaseTypes.POSTGRESQL, DatabaseTypes.SQLITE3} + +DATABASE_ENGINE = f"django.db.backends.{DATABASE_TYPE}" + +DATABASE_CONFIGS = { + DatabaseTypes.SQLITE3: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME or "/var/lib/oncall/oncall.db", + }, + DatabaseTypes.MYSQL: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, "OPTIONS": { "charset": "utf8mb4", "connect_timeout": 1, }, }, - "postgresql": { - "USER": "postgres", - "PORT": "5432", - "OPTIONS": {}, + DatabaseTypes.POSTGRESQL: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, + }, +} + +DATABASES = { + "default": DATABASE_CONFIGS[DATABASE_TYPE], +} +if DATABASE_TYPE == DatabaseTypes.MYSQL: + # Workaround to use pymysql instead of mysqlclient + import pymysql + + pymysql.install_as_MySQLdb() + +# Redis +REDIS_USERNAME = os.getenv("REDIS_USERNAME", "") +REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") +REDIS_HOST = os.getenv("REDIS_HOST") +REDIS_PORT = os.getenv("REDIS_PORT", 6379) +REDIS_PROTOCOL = os.getenv("REDIS_PROTOCOL", "redis") + +REDIS_URI = os.getenv("REDIS_URI") +if not REDIS_URI: + REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}" + +# Cache +CACHES = { + "default": { + "BACKEND": "redis_cache.RedisCache", + "LOCATION": [ + REDIS_URI, + ], + "OPTIONS": { + "DB": 1, + "PARSER_CLASS": "redis.connection.HiredisParser", + "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", + "CONNECTION_POOL_CLASS_KWARGS": { + "max_connections": 50, + "timeout": 20, + }, + "MAX_CONNECTIONS": 1000, + "PICKLE_VERSION": -1, + }, }, } @@ -261,7 +338,34 @@ USE_TZ = True STATIC_URL = os.environ.get("STATIC_URL", "/static/") STATIC_ROOT = "./static/" -CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@localhost:5672" +# RabbitMQ +RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME") +RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD") +RABBITMQ_HOST = os.getenv("RABBITMQ_HOST") +RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672) +RABBITMQ_PROTOCOL = os.getenv("RABBITMQ_PROTOCOL", "amqp") +RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", "") + +RABBITMQ_URI = os.getenv("RABBITMQ_URI") or os.getenv("RABBIT_URI") +if not RABBITMQ_URI: + RABBITMQ_URI = f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}" + + +# Celery +class BrokerTypes: + RABBITMQ = "rabbitmq" + REDIS = "redis" + + +BROKER_TYPE = os.getenv("BROKER_TYPE", BrokerTypes.RABBITMQ).lower() +assert BROKER_TYPE in {BrokerTypes.RABBITMQ, BrokerTypes.REDIS} + +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = RABBITMQ_URI +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = REDIS_URI +else: + raise ValueError(f"Invalid BROKER_TYPE env variable: {BROKER_TYPE}") # By default, apply_async will just hang indefinitely trying to reach to RabbitMQ even if RabbitMQ is down. # This makes apply_async retry 3 times trying to reach to RabbitMQ, with some extra info on periods between retries. diff --git a/engine/settings/ci-test.py b/engine/settings/ci-test.py index b3d39d4e..7af883d3 100644 --- a/engine/settings/ci-test.py +++ b/engine/settings/ci-test.py @@ -1,6 +1,6 @@ -# flake8: noqa: F405 +# flake8: noqa -from .base import * # noqa +from .base import * SECRET_KEY = "u5/IIbuiJR3Y9FQMBActk+btReZ5oOxu+l8MIJQWLfVzESoan5REE6UNSYYEQdjBOcty9CDak2X" @@ -9,27 +9,29 @@ MIRAGE_CIPHER_IV = "X+VFcDqtxJ5bbU+V" BASE_URL = "http://localhost" -CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672" +if DATABASE_TYPE == DatabaseTypes.SQLITE3: + DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_ci.db" +else: + DATABASES["default"] |= { + "NAME": DATABASE_NAME or "oncall_local_dev", + "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"], + "PASSWORD": DATABASE_PASSWORD or "local_dev_pwd", + "HOST": DATABASE_HOST or f"{DATABASE_TYPE}_test", + "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"], + } -if DB_BACKEND == "mysql": - # Workaround to use pymysql instead of mysqlclient - import pymysql +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672" +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = REDIS_URI - pymysql.install_as_MySQLdb() - DB_BACKEND_DEFAULT_VALUES[DB_BACKEND]["OPTIONS"] = {"charset": "utf8mb4"} - - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.{}".format(DB_BACKEND), - "NAME": os.environ.get("DB_NAME", "oncall_local_dev"), - "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")), - "PASSWORD": "local_dev_pwd", - "HOST": "{}_test".format(DB_BACKEND), - "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")), - "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}), - }, -} +# use redis as cache and celery broker on CI tests +if BROKER_TYPE != BrokerTypes.REDIS: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + } + } # Dummy Telegram token (fake one) TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" diff --git a/engine/settings/dev.py b/engine/settings/dev.py index fb7ddc3e..9c418ae4 100644 --- a/engine/settings/dev.py +++ b/engine/settings/dev.py @@ -1,27 +1,28 @@ +# flake8: noqa import os import sys -from .base import * # noqa - -if DB_BACKEND == "mysql": # noqa - # Workaround to use pymysql instead of mysqlclient - import pymysql - - pymysql.install_as_MySQLdb() +from .base import * DEBUG = True -DATABASES = { - "default": { - "ENGINE": "django.db.backends.{}".format(DB_BACKEND), # noqa - "NAME": os.environ.get("DB_NAME", "oncall_local_dev"), - "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")), # noqa - "PASSWORD": os.environ.get("DB_PASSWORD", "empty"), - "HOST": os.environ.get("DB_HOST", "127.0.0.1"), - "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")), # noqa - "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}), # noqa - }, -} +if DATABASE_TYPE == DatabaseTypes.SQLITE3: + DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_dev.db" +else: + DATABASES["default"] |= { + "NAME": DATABASE_NAME or "oncall_local_dev", + "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"], + "PASSWORD": DATABASE_PASSWORD or "empty", + "HOST": DATABASE_HOST or "127.0.0.1", + "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"], + } + +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672" +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = "redis://localhost:6379" + +CACHES["default"]["LOCATION"] = ["localhost:6379"] SECRET_KEY = os.environ.get("SECRET_KEY", "osMsNM0PqlRHBlUvqmeJ7+ldU3IUETCrY9TrmiViaSmInBHolr1WUlS0OFS4AHrnnkp1vp9S9z1") @@ -32,28 +33,6 @@ MIRAGE_CIPHER_IV = os.environ.get("MIRAGE_CIPHER_IV", "tZZa+60zTZO2NRcS") TESTING = "pytest" in sys.modules or "unittest" in sys.modules -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - "localhost:6379", - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} - -CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672" - SILKY_PYTHON_PROFILER = True # For any requests that come in with that header/value, request.is_secure() will return True. diff --git a/engine/settings/helm.py b/engine/settings/helm.py index 00fa96c3..6ae28e8a 100644 --- a/engine/settings/helm.py +++ b/engine/settings/helm.py @@ -1,64 +1,4 @@ -import os - -# Workaround to use pymysql instead of mysqlclient -import pymysql - -from .prod_without_db import * # noqa - -pymysql.install_as_MySQLdb() - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.mysql", - "NAME": os.environ.get("MYSQL_DB_NAME"), - "USER": os.environ.get("MYSQL_USER"), - "PASSWORD": os.environ["MYSQL_PASSWORD"], - "HOST": os.environ.get("MYSQL_HOST"), - "PORT": os.environ.get("MYSQL_PORT"), - "OPTIONS": { - "charset": "utf8mb4", - "connect_timeout": 1, - }, - }, -} - -RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME") -RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD") -RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST") -RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT") -RABBITMQ_PROTOCOL = os.environ.get("RABBITMQ_PROTOCOL") -RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "") - -CELERY_BROKER_URL = ( - f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}" -) - -REDIS_USERNAME = os.environ.get("REDIS_USERNAME", "") -REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") -REDIS_HOST = os.environ.get("REDIS_HOST") -REDIS_PORT = os.environ.get("REDIS_PORT", "6379") -REDIS_PROTOCOL = os.environ.get("REDIS_PROTOCOL", "redis") -REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}" - -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - REDIS_URI, - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} +from .prod_without_db import * # noqa: F401, F403 APPEND_SLASH = False SECURE_SSL_REDIRECT = False diff --git a/engine/settings/hobby.py b/engine/settings/hobby.py index 3bd73c13..ca7299b0 100644 --- a/engine/settings/hobby.py +++ b/engine/settings/hobby.py @@ -1,37 +1,6 @@ -# flake8: noqa: F405 +from .prod_without_db import * # noqa: F403 -from random import randrange - -# Workaround to use pymysql instead of mysqlclient -import pymysql - -from .prod_without_db import * # noqa - -pymysql.install_as_MySQLdb() - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.mysql", - "NAME": os.environ.get("MYSQL_DB_NAME"), - "USER": os.environ.get("MYSQL_USER"), - "PASSWORD": os.environ["MYSQL_PASSWORD"], - "HOST": os.environ.get("MYSQL_HOST"), - "PORT": os.environ.get("MYSQL_PORT"), - "OPTIONS": { - "charset": "utf8mb4", - "connect_timeout": 1, - }, - }, -} - -RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME") -RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD") -RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST") -RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT") - -CELERY_BROKER_URL = f"amqp://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}" - -MIRAGE_SECRET_KEY = SECRET_KEY +MIRAGE_SECRET_KEY = SECRET_KEY # noqa: F405 MIRAGE_CIPHER_IV = "1234567890abcdef" # use default APPEND_SLASH = False diff --git a/engine/settings/prod_without_db.py b/engine/settings/prod_without_db.py index 6b7c20d8..0c583483 100644 --- a/engine/settings/prod_without_db.py +++ b/engine/settings/prod_without_db.py @@ -15,36 +15,6 @@ except ModuleNotFoundError: from .base import * # noqa -# It's required for collectstatic to avoid connecting it to MySQL - -# Primary database must have the name "default" -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(BASE_DIR, "db.sqlite3"), # noqa - } -} - -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - os.environ.get("REDIS_URI"), - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} - SLACK_SIGNING_SECRET = os.environ.get("SLACK_SIGNING_SECRET") SLACK_SIGNING_SECRET_LIVE = os.environ.get("SLACK_SIGNING_SECRET_LIVE", "") @@ -56,8 +26,6 @@ STATIC_ROOT = "./collected_static/" DEBUG = False -CELERY_BROKER_URL = os.environ["RABBIT_URI"] - SECURE_SSL_REDIRECT = True SECURE_REDIRECT_EXEMPT = [ "^health/", From e32eecf6ff72b743042049b8120cede7b4ff0fc2 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 4 Oct 2022 16:48:33 +0800 Subject: [PATCH 13/18] Fix code styling in helm chart --- helm/oncall/values.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 7ab74f2c..f0a02773 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -29,18 +29,17 @@ engine: # cpu: 100m # memory: 128Mi - ## @param affinity Affinity for pod assignment - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## - affinity: { } - ## @param nodeSelector Node labels for pod assignment + ## Affinity for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Node labels for pod assignment ## ref: https://kubernetes.io/docs/user-guide/node-selection/ - ## - nodeSelector: { } - ## @param tolerations Tolerations for pod assignment + nodeSelector: {} + + ## Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [ ] + tolerations: [] # Celery workers pods configuration celery: From 2bc36440e61c550ddb9581642927f3b011dc4e3f Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 4 Oct 2022 16:58:35 +0800 Subject: [PATCH 14/18] Bump cryptography vertsion --- engine/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/requirements.txt b/engine/requirements.txt index deb82e56..950c1d1e 100644 --- a/engine/requirements.txt +++ b/engine/requirements.txt @@ -24,7 +24,7 @@ slack-export-viewer==1.0.0 beautifulsoup4==4.8.1 social-auth-app-django==3.1.0 sendgrid==6.1.2 -cryptography==3.2 +cryptography==3.3.2 pytest==5.4.3 pytest-django==3.9.0 pytest_factoryboy==2.0.3 From a7c37b2fba0f39e0b87912299d259e8f6aa30fce Mon Sep 17 00:00:00 2001 From: Gilberto Junior Date: Tue, 4 Oct 2022 06:19:13 -0300 Subject: [PATCH 15/18] Pagerduty migrator/add scripts (#403) * Script to import users from pagerduty to Grafana * Added README * Update tools/pagerduty-migrator/scripts/README.md Co-authored-by: Vadim Stepanov * Update tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py Co-authored-by: Vadim Stepanov * Update tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py Co-authored-by: Vadim Stepanov * Update tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py Co-authored-by: Vadim Stepanov * Update tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py Co-authored-by: Vadim Stepanov * Update tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py Co-authored-by: Vadim Stepanov * Adjusted script to follow project pattern * Changed variable name. * Guidance for running the script Co-authored-by: Vadim Stepanov --- tools/pagerduty-migrator/scripts/README.md | 12 ++++++ .../scripts/add_users_pagerduty_to_grafana.py | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tools/pagerduty-migrator/scripts/README.md create mode 100644 tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py diff --git a/tools/pagerduty-migrator/scripts/README.md b/tools/pagerduty-migrator/scripts/README.md new file mode 100644 index 00000000..c1aab511 --- /dev/null +++ b/tools/pagerduty-migrator/scripts/README.md @@ -0,0 +1,12 @@ +# PagerDuty migrator scripts + +When we run MODE="plan" we can notice that there is escalation, integration in pagerduty that needs to be linked to a user. + +To solve this problem, we can run the add_users_pagerduty_to_grafana.py script + +```bash +docker run -it --rm -e PAGERDUTY_API_TOKEN="mytoken" -e GRAFANA_URL="http://localhost:3000" -e GRAFANA_USERNAME="admin" -e GRAFANA_PASSWORD="admin" pd-oncall-migrator python /app/scripts/add_users_pagerduty_to_grafana.py +``` + +It is worth remembering that this script will create a user with a random password. +To access with the user created, it will be necessary to change the password in grafana web. \ No newline at end of file diff --git a/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py b/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py new file mode 100644 index 00000000..28aa542e --- /dev/null +++ b/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py @@ -0,0 +1,42 @@ +import os +import secrets +import sys +import requests + +from urllib.parse import urljoin +from pdpyras import APISession + +PAGERDUTY_API_TOKEN = os.environ["PAGERDUTY_API_TOKEN"] +PATH_USERS_GRAFANA = "/api/admin/users" +GRAFANA_URL = os.environ["GRAFANA_URL"] # Example: http://localhost:3000 +GRAFANA_USERNAME = os.environ["GRAFANA_USERNAME"] +GRAFANA_PASSWORD = os.environ["GRAFANA_PASSWORD"] +SUCCESS_SIGN = "✅" +ERROR_SIGN = "❌" + +def list_pagerduty_users(): + session = APISession(PAGERDUTY_API_TOKEN) + + users = session.list_all("users") + + for user in users: + password = secrets.token_urlsafe(15) + username = user["email"].split("@")[0] + json = {"name": user["name"], "email": user["email"], "login": username, "password": password} + create_grafana_user(json) + +def create_grafana_user(data): + url = urljoin(GRAFANA_URL, PATH_USERS_GRAFANA) + response = requests.request("POST", url, auth=(GRAFANA_USERNAME, GRAFANA_PASSWORD), json=data) + + if response.status_code == 200: + print(SUCCESS_SIGN + " User created: " + data["login"]) + elif response.status_code == 401: + sys.exit(ERROR_SIGN + " Invalid username or password.") + elif response.status_code == 412: + print(ERROR_SIGN + " User " + data["login"] + " already exists." ) + else: + print("{} {}".format(ERROR_SIGN, response.text)) + +if __name__ == "__main__": + list_pagerduty_users() \ No newline at end of file From a96f7315d46c4f883500e8d5bd3990115bfbe753 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Tue, 4 Oct 2022 12:55:19 +0100 Subject: [PATCH 16/18] fix constructSyncErrorMessage for empty JsonData --- .../src/containers/PluginConfigPage/PluginConfigPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx b/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx index e199bf02..ab5eb863 100644 --- a/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx +++ b/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx @@ -130,7 +130,7 @@ export const PluginConfigPage = (props: Props) => { const handleSyncException = useCallback((e) => { const buildErrMsg = (msg: string): string => - constructSyncErrorMessage(msg, plugin.meta.jsonData.onCallApiUrl); + constructSyncErrorMessage(msg, plugin.meta.jsonData?.onCallApiUrl); if (plugin.meta.jsonData?.onCallApiUrl) { const { status: statusCode } = e.response; From f378eab7b790fc04b27d1ba0a2153b706791ad74 Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Wed, 5 Oct 2022 12:42:08 -0300 Subject: [PATCH 17/18] Allow enabling schedules alpha per organization --- engine/apps/api/views/features.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/engine/apps/api/views/features.py b/engine/apps/api/views/features.py index fc56ca93..cc69514f 100644 --- a/engine/apps/api/views/features.py +++ b/engine/apps/api/views/features.py @@ -27,6 +27,7 @@ class FeaturesAPIView(APIView): return Response(self._get_enabled_features(request)) def _get_enabled_features(self, request): + DynamicSetting = apps.get_model("base", "DynamicSetting") enabled_features = [] if settings.FEATURE_SLACK_INTEGRATION_ENABLED: @@ -36,7 +37,6 @@ class FeaturesAPIView(APIView): enabled_features.append(FEATURE_TELEGRAM) if settings.MOBILE_APP_PUSH_NOTIFICATIONS_ENABLED: - DynamicSetting = apps.get_model("base", "DynamicSetting") mobile_app_settings = DynamicSetting.objects.get_or_create( name="mobile_app_settings", defaults={ @@ -59,5 +59,17 @@ class FeaturesAPIView(APIView): if settings.FEATURE_WEB_SCHEDULES_ENABLED: enabled_features.append(FEATURE_WEB_SCHEDULES) + else: + # allow enabling web schedules per org, independently of global status flag + enabled_web_schedules_orgs = DynamicSetting.objects.get_or_create( + name="enabled_web_schedules_orgs", + defaults={ + "json_value": { + "org_ids": [], + } + }, + )[0] + if request.auth.organization.pk in enabled_web_schedules_orgs.json_value["org_ids"]: + enabled_features.append(FEATURE_WEB_SCHEDULES) return enabled_features From f65430fbcaf910bf1153d658c059ad2d518eaa9f Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Wed, 5 Oct 2022 14:55:33 -0300 Subject: [PATCH 18/18] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebfa6fb2..01e27157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## v1.0.40 (2022-10-05) +- Improved database and celery backends support +- Added script to import PagerDuty users to Grafana +- Bug fixes + ## v1.0.39 (2022-10-03) - Fix issue in v1.0.38 blocking the creation of schedules and webhooks in the UI