Support excluding time intervals during storage invoicing and fixed typo

src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py

@@ -95,6 +95,9 @@ def add_arguments(self, parser):
                             default='nerc-invoicing')
         parser.add_argument('--upload-to-s3', default=False, action='store_true',
                           help='Upload generated CSV invoice to S3 storage.')
+        parser.add_argument('--excluded-date-ranges', type=str, 
+                            default=None, nargs='+',
+                            help='List of date ranges excluded from billing')
 
     @staticmethod
     def default_start_argument():
@@ -147,7 +150,8 @@ def process_invoice_row(allocation, attrs, su_name, rate):
             time = 0
             for attribute in attrs:
                 time += utils.calculate_quota_unit_hours(
-                    allocation, attribute, options['start'], options['end']
+                    allocation, attribute, options['start'], options['end'],
+                    excluded_intervals_list
                 )
             if time > 0:
                 row = InvoiceRow(
@@ -170,6 +174,13 @@ def process_invoice_row(allocation, attrs, su_name, rate):
         logger.info(f'Processing invoices for {options["invoice_month"]}.')
         logger.info(f'Interval {options["start"] - options["end"]}.')
 
+        if options["excluded_date_ranges"]:
+            excluded_intervals_list = utils.load_excluded_intervals(
+                options["excluded_date_ranges"]
+            )
+        else:
+            excluded_intervals_list = None
+
         openstack_resources = Resource.objects.filter(
             resource_type=ResourceType.objects.get(
                 name='OpenStack'
@@ -197,7 +208,7 @@ def process_invoice_row(allocation, attrs, su_name, rate):
 
             for allocation in openstack_allocations:
                 allocation_str = f'{allocation.pk} of project "{allocation.project.title}"'
-                msg = f'Starting billing for for allocation {allocation_str}.'
+                msg = f'Starting billing for allocation {allocation_str}.'
                 logger.debug(msg)
 
                 process_invoice_row(
@@ -208,7 +219,7 @@ def process_invoice_row(allocation, attrs, su_name, rate):
 
             for allocation in openshift_allocations:
                 allocation_str = f'{allocation.pk} of project "{allocation.project.title}"'
-                msg = f'Starting billing for for allocation {allocation_str}.'
+                msg = f'Starting billing for allocation {allocation_str}.'
                 logger.debug(msg)
 
                 process_invoice_row(

src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py

@@ -11,6 +11,9 @@
 from coldfront.core.allocation import models as allocation_models
 
 
+SECONDS_IN_DAY = 3600 * 24
+
+
 class TestCalculateAllocationQuotaHours(base.TestBase):
     def test_new_allocation_quota(self):
         self.resource = self.new_openshift_resource(
@@ -377,3 +380,135 @@ def test_new_allocation_quota_change_request(self):
             pytz.utc.localize(datetime.datetime(2020, 3, 31, 23, 59, 59))
         )
         self.assertEqual(value, 144)
+
+    def test_calculate_time_excluded_intervals(self):
+        """Test get_included_duration for correctness"""
+        def get_excluded_interval_datetime_list(excluded_interval_list):
+            return [
+                [datetime.datetime(t1[0], t1[1], t1[2], 0, 0, 0), 
+                datetime.datetime(t2[0], t2[1], t2[2], 0, 0, 0)] 
+                for t1, t2 in excluded_interval_list
+            ]
+
+        # Single interval within active period
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 15), (2020, 3, 16)),)
+        )
+
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 15, 0, 0, 0),
+            datetime.datetime(2020, 3, 17, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, SECONDS_IN_DAY * 1)
+
+        # Interval starts before active period
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 13), (2020, 3, 16)),)
+        )
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 15, 0, 0, 0),
+            datetime.datetime(2020, 3, 18, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, SECONDS_IN_DAY * 2)
+
+        # Interval ending after active period
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 16), (2020, 3, 18)),)
+        )
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 15, 0, 0, 0),
+            datetime.datetime(2020, 3, 17, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, SECONDS_IN_DAY)
+
+        # Intervals outside active period
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 1), (2020, 3, 5)),
+             ((2020, 3, 10), (2020, 3, 11)),
+             ((2020, 3, 20), (2020, 3, 25)),)
+        )
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 12, 0, 0, 0),
+            datetime.datetime(2020, 3, 19, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, SECONDS_IN_DAY * 7)
+
+        # Multiple intervals in and out of active period
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 13), (2020, 3, 15)),
+             ((2020, 3, 16), (2020, 3, 17)),
+             ((2020, 3, 18), (2020, 3, 20)),)
+        )
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 14, 0, 0, 0),
+            datetime.datetime(2020, 3, 19, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, SECONDS_IN_DAY * 2)
+
+        # Interval completely excluded
+        excluded_intervals = get_excluded_interval_datetime_list(
+            (((2020, 3, 1), (2020, 3, 30)),)
+        )
+        value = utils.get_included_duration(
+            datetime.datetime(2020, 3, 14, 0, 0, 0),
+            datetime.datetime(2020, 3, 18, 0, 0, 0),
+            excluded_intervals
+        )
+        self.assertEqual(value, 0)
+
+    def test_load_excluded_intervals(self):
+        """Test load_excluded_intervals returns valid output"""
+
+        # Single interval
+        interval_list = [
+            "2023-01-01,2023-01-02"
+        ]
+        output = utils.load_excluded_intervals(interval_list)
+        self.assertEqual(output, [
+            [datetime.datetime(2023, 1, 1, 0, 0, 0),
+            datetime.datetime(2023, 1, 2, 0, 0, 0)]
+        ])
+
+        # More than 1 interval
+        interval_list = [
+            "2023-01-01,2023-01-02",
+            "2023-01-04,2023-01-15",
+        ]
+        output = utils.load_excluded_intervals(interval_list)
+        self.assertEqual(output, [
+            [datetime.datetime(2023, 1, 1, 0, 0, 0),
+            datetime.datetime(2023, 1, 2, 0, 0, 0)],
+            [datetime.datetime(2023, 1, 4, 0, 0, 0),
+            datetime.datetime(2023, 1, 15, 0, 0, 0)]
+        ])
+
+    def test_load_excluded_intervals_invalid(self):
+        """Test when given invalid time intervals"""
+
+        # First interval is invalid
+        invalid_interval = ["foo"]
+        with self.assertRaises(ValueError):
+            utils.load_excluded_intervals(invalid_interval)
+
+        # First interval is valid, but not second
+        invalid_interval = ["2001-01-01,2002-01-01", "foo,foo"]
+        with self.assertRaises(ValueError):
+            utils.load_excluded_intervals(invalid_interval)
+
+        # End date is before start date
+        invalid_interval = ["2000-10-01,2000-01-01"]
+        with self.assertRaises(AssertionError):
+            utils.load_excluded_intervals(invalid_interval)
+
+        # Overlapping intervals
+        invalid_interval = [
+            "2000-01-01,2000-01-04",
+            "2000-01-02,2000-01-06",                
+        ]
+        with self.assertRaises(AssertionError):
+            utils.load_excluded_intervals(invalid_interval)

src/coldfront_plugin_cloud/utils.py

@@ -58,7 +58,8 @@ def get_sanitized_project_name(project_name):
 def calculate_quota_unit_hours(allocation: Allocation,
                                attribute: str,
                                start: datetime,
-                               end: datetime):
+                               end: datetime,
+                               exclude_interval_list=None):
     """Returns unit*hours of quota allocated in a given period.
 
     Calculation is rounded up by the hour and tracks the history of change
@@ -149,21 +150,73 @@ def calculate_quota_unit_hours(allocation: Allocation,
             print(f"Matching request: Last event at {last_event_time}, cr at"
                   f" {cr.history.first().created}, change at {event_time}")
 
-            before = math.ceil((created - last_event_time).total_seconds())
-            after = math.ceil((event_time - created).total_seconds())
+            before = get_included_duration(last_event_time, created, exclude_interval_list)
+            after = get_included_duration(created, event_time, exclude_interval_list)
 
             value_times_seconds += (before * last_event_value) + (after * int(event.value))
             print(f"Last event at {last_event_time}, cr created at {created}, approved at {event_time}")
         else:
-            seconds_since_last_event = math.ceil((event_time - last_event_time).total_seconds())
+            seconds_since_last_event = get_included_duration(
+                last_event_time, event_time, exclude_interval_list)
             value_times_seconds += seconds_since_last_event * last_event_value
 
         last_event_time = event_time
         unbounded_last_event_time = event.modified
         last_event_value = int(event.value)
 
     # The value remains the same from the last event until the end.
-    since_last_event = math.ceil((end - last_event_time).total_seconds())
+    since_last_event = get_included_duration(last_event_time, end, exclude_interval_list)
     value_times_seconds += since_last_event * last_event_value
 
     return math.ceil(value_times_seconds / 3600)
+
+def load_excluded_intervals(excluded_interval_arglist):
+    def interval_sort_key(e):
+        return e[0]
+
+    def check_overlapping_intervals(excluded_intervals_list):
+        prev_interval = excluded_intervals_list[0]
+        for i in range(1, len(excluded_intervals_list)): 
+            cur_interval = excluded_intervals_list[i]
+            assert cur_interval[0] >= prev_interval[1], \
+            f"Interval start date {cur_interval[0]} overlaps with another interval's end date {prev_interval[1]}"
+            prev_interval = cur_interval
+
+    excluded_intervals_list = list()
+    for interval in excluded_interval_arglist:
+        start, end = interval.strip().split(",")
+        start_dt, end_dt = [datetime.datetime.strptime(i, "%Y-%m-%d") for i in [start, end]]
+        assert end_dt > start_dt, f"Interval end date ({end}) is before start date ({start})!"
+        excluded_intervals_list.append(
+            [
+                datetime.datetime.strptime(start, "%Y-%m-%d"),
+                datetime.datetime.strptime(end, "%Y-%m-%d")
+            ] 
+        )
+
+    excluded_intervals_list.sort(key=interval_sort_key)
+    check_overlapping_intervals(excluded_intervals_list)
+
+    return excluded_intervals_list
+
+def _clamp_time(time, min_time, max_time):
+    if time < min_time:
+        time = min_time
+    if time > max_time:
+        time = max_time
+    return time
+
+
+def get_included_duration(start: datetime.datetime, 
+                          end: datetime.datetime, 
+                          excluded_intervals):
+    total_interval_duration = (end - start).total_seconds()
+    if not excluded_intervals: 
+        return total_interval_duration
+
+    for e_interval_start, e_interval_end in excluded_intervals:
+        e_interval_start = _clamp_time(e_interval_start, start, end)
+        e_interval_end = _clamp_time(e_interval_end, start, end)
+        total_interval_duration -= (e_interval_end - e_interval_start).total_seconds()
+
+    return math.ceil(total_interval_duration)