Clarify the time zone in the chart (fixes DEV-133)

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party=pandas, model_bakery, iso8601, htimeseries, simpletail, rest_captcha, dj_rest_auth, allauth, celery, parler, parler_rest, geowidgets, freezegun
+known_third_party=pandas, model_bakery, htimeseries, simpletail, rest_captcha, dj_rest_auth, allauth, celery, parler, parler_rest, geowidgets, freezegun
 known_first_party=enhydris
 known_django=django,rest_framework
 sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER

doc/dev/webservice-api.rst

@@ -673,7 +673,11 @@ You can provide time limits using the following query parameters
 ``start_date=<TIME>&end_date=<TIME>``.  For instance, to request data prior to
 2015 only, we can make the following request::
 
-    curl 'https://openmeteo.org/api/stations/1334/timeseries/232/chart/?end_date=2015-01-01T00:00`
+    curl 'https://openmeteo.org/api/stations/1334/timeseries/232/chart/?end_date=2015-01-01T00:00Z`
+
+The timestamp in the response is a Unix timestamp (i.e. seconds since
+1970-01-01T00:00:00Z). In the query parameters, the timestamp is in ISO8601
+format and must always contain a time zone.
 
 The purpose of this endpoint is to be used when creating a chart for the
 time series. When the user pans or zooms the chart, a new request with

doc/general/install.rst

@@ -17,7 +17,7 @@ PostgreSQL + PostGIS + TimescaleDB_
 GDAL                                                  1.9 [2]
 ===================================================== ============
 
-[1] Enhydris runs on Python 3.9 or later.  It does not run on Python 2.
+[1] Enhydris runs on Python 3.11 or later.  It does not run on Python 2.
 setuptools and pip are needed in order to install the rest of the Python
 modules.
 

enhydris/api/tests/test_views/test_timeseries.py

@@ -9,7 +9,6 @@
 from django.test.utils import override_settings
 from rest_framework.test import APITestCase
 
-import iso8601
 import numpy as np
 import pandas as pd
 from htimeseries import HTimeseries
@@ -121,7 +120,7 @@ def test_response_content(self):
         )
 
     def test_request_with_start_date(self):
-        response = self._get_response(urlsuffix="?start_date=2017-11-23T17:24")
+        response = self._get_response(urlsuffix="?start_date=2017-11-23T17:24Z")
         self.assertEqual(response.content.decode(), "2018-11-25 01:00,2.00,\r\n")
 
     def test_response_content_in_other_timezone(self):
@@ -321,10 +320,7 @@ def test_authorized_user_can_posttsdata(self, m: MagicMock):
 
 @override_settings(ENHYDRIS_AUTHENTICATION_REQUIRED=False)
 class TsdataPostGarbageTestCase(APITestCase):
-    @patch(
-        "enhydris.models.Timeseries.insert_or_append_data",
-        side_effect=iso8601.ParseError,
-    )
+    @patch("enhydris.models.Timeseries.insert_or_append_data", side_effect=ValueError)
     def setUp(self, m: MagicMock):
         self.mock_append_data = m
         user = baker.make(User, username="admin", is_superuser=True)
@@ -400,7 +396,7 @@ def _make_request(self, query_string: str):
 
     @patch("enhydris.models.Timeseries.get_data")
     def test_called_get_data_with_proper_start_date(self, m: MagicMock):
-        self._make_request("start_date=2005-08-23T19:54")
+        self._make_request("start_date=2005-08-23T19:54Z")
         m.assert_called_once_with(
             start_date=dt.datetime(2005, 8, 23, 19, 54, tzinfo=ZoneInfo("Etc/GMT")),
             end_date=None,
@@ -409,7 +405,7 @@ def test_called_get_data_with_proper_start_date(self, m: MagicMock):
 
     @patch("enhydris.models.Timeseries.get_data")
     def test_called_get_data_with_proper_end_date(self, m: MagicMock):
-        self._make_request("end_date=2005-08-23T19:54")
+        self._make_request("end_date=2005-08-23T19:54Z")
         m.assert_called_once_with(
             start_date=None,
             end_date=dt.datetime(2005, 8, 23, 19, 54, tzinfo=ZoneInfo("Etc/GMT")),
@@ -418,7 +414,7 @@ def test_called_get_data_with_proper_end_date(self, m: MagicMock):
 
     @patch("enhydris.models.Timeseries.get_data")
     def test_called_get_data_with_very_early_start_date(self, m: MagicMock):
-        self._make_request("start_date=0001-01-01T00:01")
+        self._make_request("start_date=0001-01-01T00:01Z")
         m.assert_called_once_with(
             start_date=dt.datetime(1680, 1, 1, 0, 0, tzinfo=ZoneInfo("Etc/GMT")),
             end_date=None,
@@ -427,7 +423,7 @@ def test_called_get_data_with_very_early_start_date(self, m: MagicMock):
 
     @patch("enhydris.models.Timeseries.get_data")
     def test_called_get_data_with_very_late_start_date(self, m: MagicMock):
-        self._make_request("end_date=3999-01-01T00:01")
+        self._make_request("end_date=3999-01-01T00:01Z")
         m.assert_called_once_with(
             start_date=None,
             end_date=dt.datetime(2260, 1, 1, 0, 0, tzinfo=ZoneInfo("Etc/GMT")),
@@ -853,29 +849,33 @@ def test_no_bounds_supplied(self, mock: MagicMock, _):
         mock.assert_called_once_with(start_date=None, end_date=None)
 
     def test_start_date_filter(self, mock: MagicMock, _):
-        response = self.client.get(self.url + "?start_date=2012-03-01T00:00")
+        response = self.client.get(self.url + "?start_date=2012-03-01T00:00Z")
         self.assertEqual(response.status_code, 200)
         mock.assert_called_once_with(
-            start_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=ZoneInfo(self.timezone)),
+            start_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=dt.timezone.utc),
             end_date=None,
         )
 
+    def test_start_date_filter_without_timezone(self, mock: MagicMock, _):
+        response = self.client.get(self.url + "?start_date=2012-03-01T00:00")
+        self.assertEqual(response.status_code, 400)
+
     def test_end_date_filter(self, mock: MagicMock, _):
-        response = self.client.get(self.url + "?end_date=2012-03-01T00:00")
+        response = self.client.get(self.url + "?end_date=2012-03-01T00:00Z")
         self.assertEqual(response.status_code, 200)
         mock.assert_called_once_with(
             start_date=None,
-            end_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=ZoneInfo(self.timezone)),
+            end_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=dt.timezone.utc),
         )
 
     def test_start_and_end_date_filters(self, mock: MagicMock, _):
         response = self.client.get(
-            self.url + "?start_date=2012-03-01T00:00&end_date=2017-03-01T00:00"
+            self.url + "?start_date=2012-03-01T00:00Z&end_date=2017-03-01T00:00Z"
         )
         self.assertEqual(response.status_code, 200)
         mock.assert_called_once_with(
-            start_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=ZoneInfo(self.timezone)),
-            end_date=dt.datetime(2017, 3, 1, 0, 0, tzinfo=ZoneInfo(self.timezone)),
+            start_date=dt.datetime(2012, 3, 1, 0, 0, tzinfo=dt.timezone.utc),
+            end_date=dt.datetime(2017, 3, 1, 0, 0, tzinfo=dt.timezone.utc),
         )
 
 

enhydris/api/views.py

@@ -6,19 +6,18 @@
 from io import StringIO
 from typing import Any
 from wsgiref.util import FileWrapper
-from zoneinfo import ZoneInfo
 
 from django.db import IntegrityError
 from django.db.models import Prefetch, QuerySet
 from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
+from django.utils.timezone import is_aware
 from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet
 
-import iso8601
 import numpy as np
 import pandas as pd
 from htimeseries import HTimeseries
@@ -282,9 +281,16 @@ def chart(
     ):
         timeseries = get_object_or_404(models.Timeseries, pk=int(pk))
         self.check_object_permissions(request, timeseries)
-        serializer = serializers.TimeseriesRecordChartSerializer(
-            self._get_chart_data(request, timeseries), many=True
-        )
+        try:
+            serializer = serializers.TimeseriesRecordChartSerializer(
+                self._get_chart_data(request, timeseries), many=True
+            )
+        except ValueError as e:
+            return HttpResponse(
+                status=status.HTTP_400_BAD_REQUEST,
+                content=str(e),
+                content_type="text/plain",
+            )
         return Response(serializer.data)  # type: ignore
 
     def _get_chart_data(self, request: Request, timeseries: models.Timeseries):
@@ -336,17 +342,25 @@ def _get_stats_for_interval(
         }
 
     def _get_date_bounds(self, request: Request, timeseries: models.Timeseries):
-        tz = ZoneInfo(timeseries.timeseries_group.gentity.display_timezone)
-        start_date = request.GET.get("start_date") or ""
-        end_date = request.GET.get("end_date") or ""
-        start_date = self._get_date_from_string(start_date, tz)  # type: ignore
-        end_date = self._get_date_from_string(end_date, tz)  # type: ignore
+        start_date = request.GET.get("start_date") or None
+        end_date = request.GET.get("end_date") or None
+        if start_date is not None:
+            start_date = self._get_date_from_string(start_date)  # type: ignore
+        if end_date is not None:
+            end_date = self._get_date_from_string(end_date)  # type: ignore
         return start_date, end_date
 
     def _get_data(self, request: Request, pk: int, format: str | None = None):
         timeseries = self.get_object()
         self.check_object_permissions(request, timeseries)
-        start_date, end_date = self._get_date_bounds(request, timeseries)
+        try:
+            start_date, end_date = self._get_date_bounds(request, timeseries)
+        except ValueError as e:
+            return HttpResponse(
+                status=status.HTTP_400_BAD_REQUEST,
+                content=str(e),
+                content_type="text/plain",
+            )
         fmt_param = request.GET.get("fmt", "csv").lower()
         timezone_param = request.GET.get("timezone", None)
 
@@ -393,28 +407,25 @@ def _post_data(self, request: Request, pk: int, format: str | None = None):
                 append_only=(mode == "append"),
             )
             return HttpResponse(status=status.HTTP_204_NO_CONTENT)
-        except (IntegrityError, iso8601.ParseError, ValueError, KeyError) as e:
+        except (IntegrityError, ValueError, KeyError) as e:
             return HttpResponse(
                 status=status.HTTP_400_BAD_REQUEST,
                 content=str(e),
                 content_type="text/plain",
             )
 
-    def _get_date_from_string(self, adate: str, tz: dt.timezone):
-        date = self._parse_date(adate, tz)
-        if not date:
-            return None
+    def _get_date_from_string(self, adate: str):
+        date = self._parse_date(adate)
+        if not date or not is_aware(date):
+            raise ValueError(f"Invalid date: '{adate}' (must contain timezone)")
         return self._bring_date_within_system_limits(date)
 
-    def _parse_date(self, adate: str, tz: dt.timezone):
-        try:
-            return iso8601.parse_date(adate, default_timezone=tz)
-        except iso8601.ParseError:
-            return None
+    def _parse_date(self, adate: str):
+        return dt.datetime.fromisoformat(adate)
 
     def _bring_date_within_system_limits(self, date: dt.datetime):
-        if date.isoformat() < "1680-01-01T00:00":
-            date = dt.datetime(1680, 1, 1, 0, 0, tzinfo=date.tzinfo)
-        if date.isoformat() > "2260-01-01T00:00":
-            date = dt.datetime(2260, 1, 1, 0, 0, tzinfo=date.tzinfo)
+        if date < dt.datetime(1680, 1, 1, 0, 0, tzinfo=dt.timezone.utc):
+            date = dt.datetime(1680, 1, 1, 0, 0, tzinfo=dt.timezone.utc)
+        if date > dt.datetime(2260, 1, 1, 0, 0, tzinfo=dt.timezone.utc):
+            date = dt.datetime(2260, 1, 1, 0, 0, tzinfo=dt.timezone.utc)
         return date

enhydris/static/js/enhydris.js

@@ -164,8 +164,8 @@ enhydris.chart = {
     /* Make API call to fetch & populate chart data between two selected
     data points (i.e. start datetime and end datetime).
     */
-    const startDate = new Date(min).toISOString().substring(0, 16);
-    const endDate = new Date(max).toISOString().substring(0, 16);
+    const startDate = new Date(min).toISOString();
+    const endDate = new Date(max).toISOString();
 
     this.mainChart.updateSeries([
       {
@@ -333,19 +333,25 @@ enhydris.chart = {
         tooltip: {
           enabled: false,
         },
+        title: {
+          text: 't (UTC)',
+        },
       },
       yaxis: {
         labels: {
           formatter(val) {
             return val.toFixed(enhydris.precision >= 0 ? enhydris.precision : 0);
           },
         },
+        title: {
+          text: enhydris.unit,
+        },
       },
       tooltip: {
         enabled: true,
         x: {
           show: true,
-          format: 'dd MMM yyyy HH:mm',
+          format: 'dd MMM yyyy HH:mm U\\TC',
         },
       },
       legend: {

enhydris/synoptic/models.py

@@ -7,7 +7,6 @@
 from django.db import DataError, IntegrityError, models
 from django.utils.translation import gettext as _
 
-import iso8601
 from rocc import Threshold, rocc
 
 from enhydris.models import DISPLAY_TIMEZONE_CHOICES, Station, TimeseriesGroup
@@ -226,7 +225,7 @@ def _check_rate_of_change(self, asyntsg):
         if not messages:
             return None
         message = messages[-1]
-        date = iso8601.parse_date(message.split()[0]).replace(tzinfo=None)
+        date = dt.datetime.fromisoformat(message.split()[0]).replace(tzinfo=None)
         if date == self.last_common_date.replace(tzinfo=None):
             return message
 

enhydris/telemetry/models.py

@@ -13,8 +13,6 @@
 from django.db import IntegrityError, models
 from django.utils.translation import gettext_lazy as _
 
-import iso8601
-
 import enhydris
 from enhydris.models import Station, Timeseries, TimeseriesGroup
 
@@ -118,7 +116,7 @@ def _cleanup_measurements(self, measurements):
         prev_timestamp = dt.datetime(1, 1, 1, 0, 0)
         measurements.seek(0)
         for line in measurements:
-            cur_timestamp = iso8601.parse_date(line.split(",")[0])
+            cur_timestamp = dt.datetime.fromisoformat(line.split(",")[0])
             cur_timestamp = cur_timestamp.replace(second=0, tzinfo=None)
             if cur_timestamp == prev_timestamp:
                 continue

enhydris/templates/enhydris/timeseries_group_detail/main-default.html

@@ -61,6 +61,7 @@ <h1>{{ object.gentity.name }}</h1>
     enhydris.strLoading = "{% trans 'Loading...'  %}";
     enhydris.strNoData = "{% trans 'No data' %}";
     enhydris.precision = parseInt("{{ object.precision }}");
+    enhydris.unit = "{{ object.unit_of_measurement.symbol }}";
     enhydris.strInfo = "{% trans 'Info' %}";
 
     {% if object.default_timeseries %}

requirements.txt

@@ -1,6 +1,5 @@
 Django>=5.2,<6
 djangorestframework>=3.9,<4
-iso8601
 pytz
 rules>=2.0
 dj-rest-auth==2.1.7