I think #223 is finished. But test_datamodule is failing, although that might be a separate issue, see #276

Author: JackKelly

nowcasting_dataset/dataset/datamodule.py

@@ -78,6 +78,7 @@ class NowcastingDataModule(pl.LightningDataModule):
     pv_load_azimuth_and_elevation: bool = False
     split_method: SplitMethod = SplitMethod.DAY  # which split method should be used
     seed: Optional[int] = None  # seed used to make quasi random split data
+    t0_datetime_freq: str = "5T"  # Frequency of the t0 datetimes.
 
     skip_n_train_batches: int = 0  # number of train batches to skip
     skip_n_validation_batches: int = 0  # number of validation batches to skip
@@ -379,10 +380,14 @@ def _get_t0_datetimes(self) -> pd.DatetimeIndex:
 
         # TODO: Allow user to configure the frequency using the config yaml file.
         # https://github.com/openclimatefix/nowcasting_dataset/issues/277
-        return nd_time.time_periods_to_datetimes(
-            time_periods=intersection_of_t0_time_periods, freq="30T"
+        t0_datetimes = nd_time.time_periods_to_datetimes(
+            time_periods=intersection_of_t0_time_periods, freq=self.t0_datetime_freq
         )
 
+        # Align to the nearest t0_datetime_freq.  For example, if t0_datetime_freq is '5T'
+        # then ensure the minutes past the hour are exactly divisible by 5.
+        return t0_datetimes.round(self.t0_datetime_freq)
+
     def _check_has_prepared_data(self):
         if not self.has_prepared_data:
             raise RuntimeError("Must run prepare_data() first!")

nowcasting_dataset/time.py

@@ -55,7 +55,7 @@ def select_daylight_datetimes(
     return datetimes[mask]
 
 
-def period_to_datetime_index(period: pd.DataFrame, freq: str) -> pd.DatetimeIndex:
+def period_to_datetime_index(period: pd.Series, freq: str) -> pd.DatetimeIndex:
     """Return a DatetimeIndex from period['start_dt'] to period['end_dt'] at frequency freq."""
     return pd.date_range(period["start_dt"], period["end_dt"], freq=freq)
 

tests/test_dataset.py

@@ -7,12 +7,16 @@
 from nowcasting_dataset.dataset.batch import Batch
 
 
+def _get_t0_datetimes(data_source, freq) -> pd.DatetimeIndex:
+    t0_periods = data_source.get_contiguous_t0_time_periods()
+    t0_datetimes = nd_time.time_periods_to_datetimes(t0_periods, freq=freq)
+    return t0_datetimes
+
+
 @pytest.fixture
 def dataset(sat_data_source, general_data_source):
-    all_datetimes = sat_data_source.datetime_index()
-    t0_datetimes = nd_time.get_t0_datetimes(
-        datetimes=all_datetimes, total_seq_length=2, history_duration=pd.Timedelta(0)
-    )
+    t0_datetimes = _get_t0_datetimes(sat_data_source, freq="5T")
+
     return NowcastingDataset(
         batch_size=8,
         n_batches_per_epoch_per_worker=64,
@@ -24,13 +28,7 @@ def dataset(sat_data_source, general_data_source):
 
 @pytest.fixture
 def dataset_gsp(gsp_data_source, general_data_source):
-    all_datetimes = gsp_data_source.datetime_index()
-    t0_datetimes = nd_time.get_t0_datetimes(
-        datetimes=all_datetimes,
-        total_seq_length=2,
-        history_duration=pd.Timedelta(0),
-        max_gap=nd_time.THIRTY_MINUTES,
-    )
+    t0_datetimes = _get_t0_datetimes(gsp_data_source, freq="30T")
 
     return NowcastingDataset(
         batch_size=8,