From c4fb620328c9dd83646d7509de74ed72d0733dca Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Fri, 27 Oct 2023 15:45:00 -0300 Subject: [PATCH] Upgrade to django 4.2.6 and other deps updates (#3176) --- CHANGELOG.md | 4 + .../amixr_recurring_ical_events_adapter.py | 94 +------------------ .../schedules/models/custom_on_call_shift.py | 23 ++--- .../apps/schedules/models/on_call_schedule.py | 26 ++--- .../tests/test_custom_on_call_shift.py | 2 +- .../live_setting_django_strategy.py | 4 +- engine/apps/user_management/models/user.py | 2 +- engine/requirements.txt | 16 ++-- 8 files changed, 45 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0053ea..046d0810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix iCal imported schedules related users and next shifts per user ([#3178](https://github.com/grafana/oncall/pull/3178)) - Fix references to removed access control functions in Grafana @mderynck ([#3184](https://github.com/grafana/oncall/pull/3184)) +### Changed + +- Upgrade Django to 4.2.6 and update iCal related deps ([#3176](https://github.com/grafana/oncall/pull/3176)) + ## v1.3.45 (2023-10-19) ### Added diff --git a/engine/apps/schedules/ical_events/adapter/amixr_recurring_ical_events_adapter.py b/engine/apps/schedules/ical_events/adapter/amixr_recurring_ical_events_adapter.py index 5f574aba..155a6161 100644 --- a/engine/apps/schedules/ical_events/adapter/amixr_recurring_ical_events_adapter.py +++ b/engine/apps/schedules/ical_events/adapter/amixr_recurring_ical_events_adapter.py @@ -1,10 +1,8 @@ import datetime -import re import typing -from collections import defaultdict from icalendar import Calendar, Event -from recurring_ical_events import UnfoldableCalendar, compare_greater, is_event, time_span_contains_event +from recurring_ical_events import UnfoldableCalendar, time_span_contains_event from apps.schedules.constants import ( ICAL_DATETIME_END, @@ -21,94 +19,6 @@ from apps.schedules.ical_events.proxy.ical_proxy import IcalService EXTRA_LOOKUP_DAYS = 16 -class AmixrUnfoldableCalendar(UnfoldableCalendar): - """ - This is overridden recurring_ical_events.UnfoldableCalendar. - It is overridden because of bug when summary of recurring event stay the same after editing. - In recurring-ical-events==0.1.20b0 this problem is fixed, but all-day events without timezone lead to exception. - So i took part of code from 0.1.20b0 but leave 0.1.16b in requirements. - """ - - class RepeatedEvent(UnfoldableCalendar.RepeatedEvent): - RE_DATETIME_VALUE = re.compile(r"\d+T\d+") - - class Repetition(UnfoldableCalendar.RepeatedEvent.Repetition): - """ - A repetition of an event. Overridden version of - recurring_ical_events.UnfoldableCalendar.RepeatedEvent.Repetition. This is overridden to remove the 'RRULE' - param from ATTRIBUTES_TO_DELETE_ON_COPY, because the 'UNTIL' param must be stored in repetition events to - calculate its end date. - """ - - ATTRIBUTES_TO_DELETE_ON_COPY = ["RDATE", "EXDATE"] - - def create_rule_with_start(self, rule_string, start): - """Override to handle issue with non-UTC UNTIL value including time information.""" - try: - return super().create_rule_with_start(rule_string, start) - except ValueError: - # string: FREQ=WEEKLY;UNTIL=20191023T100000;BYDAY=TH;WKST=SU - # ValueError: RRULE UNTIL values must be specified in UTC when DTSTART is timezone-aware - # https://stackoverflow.com/a/49991809 - rule_list = rule_string.split(";UNTIL=") - assert len(rule_list) == 2 - date_end_index = rule_list[1].find(";") - if date_end_index == -1: - date_end_index = len(rule_list[1]) - until_string = rule_list[1][:date_end_index] - if self.RE_DATETIME_VALUE.match(until_string): - rule_string = rule_list[0] + rule_list[1][date_end_index:] + ";UNTIL=" + until_string + "Z" - return super().create_rule_with_start(rule_string, self.start) - # otherwise, keep raising - raise - - def between(self, start, stop): - """Return events at a time between start (inclusive) and end (inclusive)""" - span_start = self.to_datetime(start) - span_stop = self.to_datetime(stop) - events = [] - events_by_id = defaultdict(dict) # UID (str) : RECURRENCE-ID(datetime) : event (Event) - default_uid = object() - - def add_event(event): - """Add an event and check if it was edited.""" - same_events = events_by_id[event.get("UID", default_uid)] - recurrence_id = event.get("RECURRENCE-ID", event["DTSTART"]).dt - # Start of code from 0.1.20b0 - if isinstance(recurrence_id, datetime.datetime): - recurrence_id = recurrence_id.date() - other = same_events.get(recurrence_id, None) - if other: - event_recurrence_id = event.get("RECURRENCE-ID", None) - other_recurrence_id = other.get("RECURRENCE-ID", None) - if event_recurrence_id is not None and other_recurrence_id is None: - events.remove(other) - elif event_recurrence_id is None and other_recurrence_id is not None: - return - else: - event_sequence = event.get("SEQUENCE", None) - other_sequence = other.get("SEQUENCE", None) - if event_sequence is not None and other_sequence is not None: - if event["SEQUENCE"] < other["SEQUENCE"]: - return - events.remove(other) - # End of code from 0.1.20b0 - same_events[recurrence_id] = event - events.append(event) - - for event in self.calendar.walk(): - if not is_event(event): - continue - repetitions = self.RepeatedEvent(event, span_start) - for repetition in repetitions: - if compare_greater(repetition.start, span_stop) or compare_greater(repetition.start, repetition.stop): - # future repetitions could produce invalid events (because of the until rrule) - break - if repetition.is_in_span(span_start, span_stop): - add_event(repetition.as_vevent()) - return events - - class AmixrRecurringIcalEventsAdapter(IcalService): def get_events_from_ical_between( self, calendar: Calendar, start_date: datetime.datetime, end_date: datetime.datetime @@ -123,7 +33,7 @@ class AmixrRecurringIcalEventsAdapter(IcalService): make one more pass for events array to filter out events which are between start_date and end_date. EXTRA_LOOKUP_DAYS is empirical. """ - events = AmixrUnfoldableCalendar(calendar).between( + events = UnfoldableCalendar(calendar).between( start_date - datetime.timedelta(days=EXTRA_LOOKUP_DAYS), end_date + datetime.timedelta(days=EXTRA_LOOKUP_DAYS), ) diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index 3423de96..a68e8d4e 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -7,6 +7,7 @@ from calendar import monthrange from uuid import uuid4 import pytz +import recurring_ical_events from dateutil import relativedelta from django.conf import settings from django.core.validators import MinLengthValidator @@ -16,7 +17,6 @@ from django.forms.models import model_to_dict from django.utils import timezone from django.utils.functional import cached_property from icalendar.cal import Event -from recurring_ical_events import UnfoldableCalendar from apps.schedules.tasks import ( drop_cached_ical_task, @@ -488,13 +488,14 @@ class CustomOnCallShift(models.Model): next_event_start = current_event_start # Calculate the minimum start date for the next event based on rotation frequency. We don't need to do this # for the first rotation, because in this case the min start date will be the same as the current event date. + DAYS_IN_A_WEEK = 7 + DAYS_IN_A_MONTH = monthrange(current_event_start.year, current_event_start.month)[1] if get_next_date: if self.frequency == CustomOnCallShift.FREQUENCY_HOURLY: next_event_start = current_event_start + datetime.timedelta(hours=ONE_HOUR) elif self.frequency == CustomOnCallShift.FREQUENCY_DAILY: next_event_start = current_event_start + datetime.timedelta(days=ONE_DAY) elif self.frequency == CustomOnCallShift.FREQUENCY_WEEKLY: - DAYS_IN_A_WEEK = 7 # count days before the next week starts days_for_next_event = DAYS_IN_A_WEEK - current_event_start.weekday() + self.week_start if days_for_next_event > DAYS_IN_A_WEEK: @@ -504,7 +505,6 @@ class CustomOnCallShift(models.Model): days=days_for_next_event + DAYS_IN_A_WEEK * (interval - 1) ) elif self.frequency == CustomOnCallShift.FREQUENCY_MONTHLY: - DAYS_IN_A_MONTH = monthrange(current_event_start.year, current_event_start.month)[1] # count days before the next month starts days_for_next_event = DAYS_IN_A_MONTH - current_event_start.day + ONE_DAY # count next event start date with respect to event interval @@ -533,10 +533,12 @@ class CustomOnCallShift(models.Model): next_event = None # repetitions generate the next event shift according with the recurrence rules - repetitions = UnfoldableCalendar(current_event).RepeatedEvent( - current_event, next_event_start.replace(microsecond=0) - ) - for event in repetitions.__iter__(): + repeated_event = recurring_ical_events.RepeatedEvent(current_event) + max_date_range = next_event_start + datetime.timedelta(days=DAYS_IN_A_MONTH) + if end_date: + max_date_range = max(end_date, max_date_range) + repetitions = repeated_event.within_days(next_event_start.replace(microsecond=0), max_date_range) + for event in repetitions: if end_date: # end_date exists for long events with frequency weekly and monthly if end_date >= event.start >= next_event_start: if ( @@ -572,10 +574,9 @@ class CustomOnCallShift(models.Model): last_event = None # repetitions generate the next event shift according with the recurrence rules - repetitions = UnfoldableCalendar(initial_event).RepeatedEvent( - initial_event, initial_event_start.replace(microsecond=0) - ) - for event in repetitions.__iter__(): + repeated_event = recurring_ical_events.RepeatedEvent(initial_event) + repetitions = repeated_event.within_days(initial_event_start, date) + for event in repetitions: if event.start > date: break last_event = event diff --git a/engine/apps/schedules/models/on_call_schedule.py b/engine/apps/schedules/models/on_call_schedule.py index 26761bc5..f3bb8342 100644 --- a/engine/apps/schedules/models/on_call_schedule.py +++ b/engine/apps/schedules/models/on_call_schedule.py @@ -349,18 +349,22 @@ class OnCallSchedule(PolymorphicModel): include_shift_info: bool = False, ) -> ScheduleEvents: """Return filtered events from schedule.""" - shifts = ( - list_of_oncall_shifts_from_ical( - self, - datetime_start, - datetime_end, - with_empty, - with_gap, - filter_by=filter_by, - from_cached_final=from_cached_final, + try: + shifts = ( + list_of_oncall_shifts_from_ical( + self, + datetime_start, + datetime_end, + with_empty, + with_gap, + filter_by=filter_by, + from_cached_final=from_cached_final, + ) + or [] ) - or [] - ) + except ValueError: + # raised when filtering events on a non-saved/deleted schedule + return [] shifts_data = {} if include_shift_info: pks = set(shift["shift_pk"] for shift in shifts) diff --git a/engine/apps/schedules/tests/test_custom_on_call_shift.py b/engine/apps/schedules/tests/test_custom_on_call_shift.py index 1f904d0e..01c490f6 100644 --- a/engine/apps/schedules/tests/test_custom_on_call_shift.py +++ b/engine/apps/schedules/tests/test_custom_on_call_shift.py @@ -1749,7 +1749,7 @@ def test_week_start_changed_daily_shift( on_call_shift.add_rolling_users(rolling_users) ical_data = on_call_shift.convert_to_ical() - expected_start = "DTSTART;VALUE=DATE-TIME:{}T000000Z".format(last_sunday.strftime("%Y%m%d")) + expected_start = "DTSTART:{}T000000Z".format(last_sunday.strftime("%Y%m%d")) assert expected_start in ical_data diff --git a/engine/apps/social_auth/live_setting_django_strategy.py b/engine/apps/social_auth/live_setting_django_strategy.py index 62f14d6e..6e103bfb 100644 --- a/engine/apps/social_auth/live_setting_django_strategy.py +++ b/engine/apps/social_auth/live_setting_django_strategy.py @@ -2,7 +2,7 @@ import logging from django.conf import settings from django.shortcuts import resolve_url -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.functional import Promise from social_django.strategy import DjangoStrategy @@ -29,7 +29,7 @@ class LiveSettingDjangoStrategy(DjangoStrategy): # Force text on URL named settings that are instance of Promise if name.endswith("_URL"): if isinstance(value, Promise): - value = force_text(value) + value = force_str(value) value = resolve_url(value) return value diff --git a/engine/apps/user_management/models/user.py b/engine/apps/user_management/models/user.py index e75490bb..92829216 100644 --- a/engine/apps/user_management/models/user.py +++ b/engine/apps/user_management/models/user.py @@ -308,7 +308,7 @@ class User(models.Model): self._timezone = value def is_in_working_hours(self, dt: datetime.datetime, tz: typing.Optional[str] = None) -> bool: - assert dt.tzinfo == pytz.utc, "dt must be in UTC" + assert dt.tzinfo == datetime.timezone.utc, "dt must be in UTC" # Default to user's timezone if not tz: diff --git a/engine/requirements.txt b/engine/requirements.txt index f39f828e..e6ac5e73 100644 --- a/engine/requirements.txt +++ b/engine/requirements.txt @@ -1,5 +1,5 @@ -django==3.2.20 -djangorestframework==3.12.4 +django==4.2.6 +djangorestframework==3.14.0 slack_sdk==3.21.3 whitenoise==5.3.0 twilio~=6.37.0 @@ -22,20 +22,20 @@ git+https://github.com/grafana/django-redis-cache.git@bump-redis-version-to-v4.6 hiredis==1.0.0 django-ratelimit==2.0.0 django-filter==2.4.0 -icalendar==4.0.7 -recurring-ical-events==0.1.16b0 +icalendar==5.0.10 +recurring-ical-events==2.1.0 slack-export-viewer==1.1.4 beautifulsoup4==4.12.2 -social-auth-app-django==5.0.0 +social-auth-app-django==5.3.0 cryptography==38.0.4 # version 39.0.0 introduced an issue - https://stackoverflow.com/a/75053968/3902555 factory-boy<3.0 django-log-request-id==1.6.0 -django-polymorphic==3.0.0 -django-rest-polymorphic==0.1.9 +django-polymorphic==3.1.0 +django-rest-polymorphic==0.1.10 https://github.com/grafana/fcm-django/archive/refs/tags/v1.0.12r1.tar.gz django-mirage-field==1.3.0 django-mysql==4.6.0 -PyMySQL==1.0.2 +PyMySQL==1.1.0 psycopg2==2.9.3 emoji==2.4.0 regex==2021.11.2