diff --git a/enhydris/admin/__init__.py b/enhydris/admin/__init__.py index 505f177e..864e56dd 100644 --- a/enhydris/admin/__init__.py +++ b/enhydris/admin/__init__.py @@ -1,9 +1,6 @@ -from django.conf import settings from django.contrib import admin from django.utils.translation import gettext_lazy as _ -from parler.admin import TranslatableAdmin - from enhydris import models from .garea import GareaAdmin # NOQA @@ -32,14 +29,15 @@ class EventTypeAdmin(admin.ModelAdmin): list_display = ("id", "descr") +class VariableTranslationInline(admin.TabularInline): + model = models.VariableTranslation + extra = 1 + + @admin.register(models.Variable) -class VariableAdmin(TranslatableAdmin): +class VariableAdmin(admin.ModelAdmin): list_display = ("id", "descr", "last_modified") - - def get_queryset(self, request): - return models.Variable.objects.translated(settings.LANGUAGE_CODE).order_by( - "translations__descr" - ) + inlines = [VariableTranslationInline] @admin.register(models.UnitOfMeasurement) diff --git a/enhydris/api/serializers.py b/enhydris/api/serializers.py index 4ff7fd8f..5a7c32e8 100644 --- a/enhydris/api/serializers.py +++ b/enhydris/api/serializers.py @@ -4,9 +4,6 @@ from django.utils import translation from rest_framework import serializers -from parler_rest.fields import TranslatedFieldsField -from parler_rest.serializers import TranslatableModelSerializer - from enhydris import models @@ -104,13 +101,16 @@ class Meta: fields = "__all__" -class VariableSerializer(TranslatableModelSerializer): - translations = TranslatedFieldsField(shared_model=models.Variable, required=False) +class VariableSerializer(serializers.ModelSerializer): + translations = serializers.SerializerMethodField() class Meta: model = models.Variable fields = "__all__" + def get_translations(self, obj: models.Variable) -> dict[str, dict[str, str]]: + return {t.language_code: {"descr": t.descr} for t in obj.translations.all()} + class UnitOfMeasurementSerializer(serializers.ModelSerializer): class Meta: diff --git a/enhydris/api/tests/test_serializers.py b/enhydris/api/tests/test_serializers.py index 1014e46a..268a43bb 100644 --- a/enhydris/api/tests/test_serializers.py +++ b/enhydris/api/tests/test_serializers.py @@ -105,9 +105,9 @@ def test_type_deserialization_for_checked(self): class TimeseriesSerializerUniqueTypeTestCase(APITestCase): @classmethod def setUpTestData(cls): - cls.timeseries_group = baker.make( - models.TimeseriesGroup, - variable__descr="Temperature", + cls.timeseries_group = baker.make(models.TimeseriesGroup) + cls.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" ) def test_only_one_initial_timeseries_per_group(self): diff --git a/enhydris/api/tests/test_views/test_misc.py b/enhydris/api/tests/test_views/test_misc.py index 3478ba07..53788ce1 100644 --- a/enhydris/api/tests/test_views/test_misc.py +++ b/enhydris/api/tests/test_views/test_misc.py @@ -49,11 +49,25 @@ def test_get_event_type(self): @override_settings(ENHYDRIS_AUTHENTICATION_REQUIRED=False) class VariableTestCase(APITestCase): def setUp(self): - self.variable = baker.make(models.Variable, descr="Temperature") + self.variable = models.Variable.objects.create() + self.variable.translations.create(language_code="en", descr="Temperature") + self.variable.translations.create(language_code="el", descr="Θερμοκρασία") def test_get_variable(self): r = self.client.get("/api/variables/{}/".format(self.variable.id)) self.assertEqual(r.status_code, 200) + self.assertEqual( + r.json()["translations"], + {"en": {"descr": "Temperature"}, "el": {"descr": "Θερμοκρασία"}}, + ) + + def test_list_variable(self): + r = self.client.get("/api/variables/") + self.assertEqual(r.status_code, 200) + self.assertEqual( + r.json()["results"][0]["translations"], + {"en": {"descr": "Temperature"}, "el": {"descr": "Θερμοκρασία"}}, + ) @override_settings(ENHYDRIS_AUTHENTICATION_REQUIRED=False) diff --git a/enhydris/api/tests/test_views/test_search.py b/enhydris/api/tests/test_views/test_search.py index 6e4cbd04..18a369c3 100644 --- a/enhydris/api/tests/test_views/test_search.py +++ b/enhydris/api/tests/test_views/test_search.py @@ -6,7 +6,6 @@ from rest_framework.test import APITestCase from model_bakery import baker -from parler.utils.context import switch_language from enhydris import models @@ -101,10 +100,6 @@ class SearchByRemarksWithAccentsTestCase(SearchByRemarksTestCase): language_settings = { "LANGUAGE_CODE": "en", - "PARLER_LANGUAGES": { - settings.SITE_ID: ({"code": "en"}, {"code": "fr"}), - "default": {"fallbacks": ["en"], "hide_untranslated": False}, - }, } @@ -119,13 +114,10 @@ def _create_models(self): self._create_timeseries(station1, "Rain", "Pluie") self._create_timeseries(station2, "Humidity", "Humidité") - def _create_timeseries(self, station, var_en, var_fr): - variable = models.Variable() - with switch_language(variable, "en"): - variable.descr = var_en - with switch_language(variable, "fr"): - variable.descr = var_fr - variable.save() + def _create_timeseries(self, station: models.Station, var_en: str, var_fr: str): + variable = models.Variable.objects.create() + variable.translations.create(language_code="en", descr=var_en) + variable.translations.create(language_code="fr", descr=var_fr) baker.make( models.Timeseries, timeseries_group__gentity=station, diff --git a/enhydris/api/tests/test_views/test_search_by_ts_has_years.py b/enhydris/api/tests/test_views/test_search_by_ts_has_years.py index c408cd06..bae5dbd9 100644 --- a/enhydris/api/tests/test_views/test_search_by_ts_has_years.py +++ b/enhydris/api/tests/test_views/test_search_by_ts_has_years.py @@ -36,7 +36,9 @@ def _make_timeseries(self, station, variable_descr, datastr): result = baker.make( models.Timeseries, timeseries_group__gentity=station, - timeseries_group__variable__descr=variable_descr, + ) + result.timeseries_group.variable.translations.create( + language_code="en", descr=variable_descr ) result.set_data(StringIO(datastr), default_timezone="Etc/GMT-2") return result diff --git a/enhydris/api/tests/test_views/test_station.py b/enhydris/api/tests/test_views/test_station.py index ca8cfcd7..89984dfa 100644 --- a/enhydris/api/tests/test_views/test_station.py +++ b/enhydris/api/tests/test_views/test_station.py @@ -225,8 +225,9 @@ def _create_timeseries_groups(self): self._create_timeseries_group(self.station_agios_athanasios, "Humidity") def _create_timeseries_group(self, station, variable_descr): - baker.make( - models.TimeseriesGroup, gentity=station, variable__descr=variable_descr + timeseries_group = baker.make(models.TimeseriesGroup, gentity=station) + timeseries_group.variable.translations.create( + language_code="en", descr=variable_descr ) def test_station_csv(self): @@ -247,8 +248,8 @@ def test_station_with_geometry_with_no_srid_is_included(self): def test_num_queries(self): self._create_timeseries_groups() - # There should be seven queries: one for stations, one for timeseries_groups, - # one for timeseries. The other four are two for django_session and two - # for a savepoint. - with self.assertNumQueries(7): + # There should be eight queries: one for stations, one for timeseries_groups, + # one for timeseries, one for prefetching variable translations. The + # other four are two for django_session and two for a savepoint. + with self.assertNumQueries(8): self.client.get("/api/stations/csv/") diff --git a/enhydris/api/tests/test_views/test_timeseries.py b/enhydris/api/tests/test_views/test_timeseries.py index 066ad858..a480e082 100644 --- a/enhydris/api/tests/test_views/test_timeseries.py +++ b/enhydris/api/tests/test_views/test_timeseries.py @@ -469,9 +469,11 @@ def setUp(self): self.timeseries_group = baker.make( models.TimeseriesGroup, gentity=self.station, - variable__descr="irrelevant", precision=2, ) + self.timeseries_group.variable.translations.create( + language_code="en", descr="irrelevant" + ) self.timeseries = baker.make( models.Timeseries, timeseries_group=self.timeseries_group, @@ -583,7 +585,8 @@ class TimeseriesPostTestCase(APITestCase): def setUp(self): self.user1 = baker.make(User, is_active=True, is_superuser=False) self.user2 = baker.make(User, is_active=True, is_superuser=False) - self.variable = baker.make(models.Variable, descr="Temperature") + self.variable = models.Variable.objects.create() + self.variable.translations.create(language_code="en", descr="Temperature") self.unit_of_measurement = baker.make(models.UnitOfMeasurement) self.station = baker.make(models.Station, creator=self.user1) self.timeseries_group = baker.make(models.TimeseriesGroup, gentity=self.station) @@ -629,7 +632,8 @@ def test_returns_proper_error_when_creating_second_initial_timeseries(self): class TimeseriesPostWithWrongStationOrTimeseriesGroupTestCase(APITestCase): def setUp(self): self.user = baker.make(User, is_active=True, is_superuser=False) - self.variable = baker.make(models.Variable, descr="Temperature") + self.variable = models.Variable.objects.create() + self.variable.translations.create(language_code="en", descr="Temperature") self.unit_of_measurement = baker.make(models.UnitOfMeasurement) self.station1 = baker.make(models.Station, creator=self.user) self.timeseries_group_1_1 = baker.make( @@ -690,7 +694,8 @@ def test_create_timeseries_with_everything_correct(self): class TimeseriesPostWithWrongTimeseriesTypeTestCase(APITestCase): def setUp(self): self.user = baker.make(User, is_active=True, is_superuser=False) - self.variable = baker.make(models.Variable, descr="Temperature") + self.variable = models.Variable.objects.create() + self.variable.translations.create(language_code="en", descr="Temperature") self.unit_of_measurement = baker.make(models.UnitOfMeasurement) self.station = baker.make(models.Station, creator=self.user) self.timeseries_group = baker.make(models.TimeseriesGroup, gentity=self.station) diff --git a/enhydris/api/tests/test_views/test_timeseries_group.py b/enhydris/api/tests/test_views/test_timeseries_group.py index 511e3c37..9bf0586a 100644 --- a/enhydris/api/tests/test_views/test_timeseries_group.py +++ b/enhydris/api/tests/test_views/test_timeseries_group.py @@ -34,7 +34,8 @@ class TimeseriesGroupPostTestCase(APITestCase): def setUp(self): self.user1 = baker.make(User, is_active=True, is_superuser=False) self.user2 = baker.make(User, is_active=True, is_superuser=False) - self.variable = baker.make(models.Variable, descr="Temperature") + self.variable = models.Variable.objects.create() + self.variable.translations.create(language_code="en", descr="Temperature") self.unit_of_measurement = baker.make(models.UnitOfMeasurement) self.station = baker.make(models.Station, creator=self.user1) self.station2 = baker.make(models.Station, creator=self.user1) diff --git a/enhydris/api/views.py b/enhydris/api/views.py index 1bb12d41..ac6e5694 100644 --- a/enhydris/api/views.py +++ b/enhydris/api/views.py @@ -56,7 +56,13 @@ def csv(self, request: Request): queryset=models.TimeseriesGroup.objects.select_related( "variable", "unit_of_measurement" ) - .prefetch_related("timeseries_set") + .prefetch_related( + "timeseries_set", + Prefetch( + "variable__translations", + to_attr="prefetched_translations", + ), + ) .order_by("variable__id"), ) ) @@ -90,7 +96,7 @@ class EventTypeViewSet(ReadOnlyModelViewSet[models.EventType]): class VariableViewSet(ReadOnlyModelViewSet[models.Variable]): serializer_class = serializers.VariableSerializer - queryset = models.Variable.objects.all() # type: ignore + queryset = models.Variable.objects.prefetch_related("translations") # type: ignore class UnitOfMeasurementViewSet(ReadOnlyModelViewSet[models.UnitOfMeasurement]): diff --git a/enhydris/autoprocess/tests/test_admin.py b/enhydris/autoprocess/tests/test_admin.py index 5de99b05..8f2a247d 100644 --- a/enhydris/autoprocess/tests/test_admin.py +++ b/enhydris/autoprocess/tests/test_admin.py @@ -33,7 +33,8 @@ def _create_data(cls): cls.organization = enhydris.models.Organization.objects.create( name="Serial killers SA" ) - cls.variable = baker.make(enhydris.models.Variable, descr="myvar") + cls.variable = enhydris.models.Variable.objects.create() + cls.variable.translations.create(language_code="en", descr="myvar") cls.unit = baker.make(enhydris.models.UnitOfMeasurement) cls.station = baker.make( enhydris.models.Station, creator=cls.user, owner=cls.organization diff --git a/enhydris/autoprocess/tests/test_models/test_aggregation.py b/enhydris/autoprocess/tests/test_models/test_aggregation.py index 0fa675da..6276fb2d 100644 --- a/enhydris/autoprocess/tests/test_models/test_aggregation.py +++ b/enhydris/autoprocess/tests/test_models/test_aggregation.py @@ -18,7 +18,8 @@ class AggregationTestCase(TestCase): def setUp(self): self.station = baker.make(Station) - variable = baker.make(Variable, descr="Irrelevant") + variable = Variable.objects.create() + variable.translations.create(language_code="en", descr="irrelevant") self.timeseries_group = baker.make( TimeseriesGroup, gentity=self.station, variable=variable ) @@ -59,7 +60,8 @@ def test_str(self): def test_no_extra_queries_for_str(self): self._baker_make_aggregation() - with self.assertNumQueries(1): + # One query for the aggregation, one for prefetching of variable translations + with self.assertNumQueries(2): str(Aggregation.objects.first()) def test_wrong_resulting_timestamp_offset_1(self): @@ -225,12 +227,14 @@ def _execute(self, max_missing): self.aggregation = baker.make( Aggregation, timeseries_group__gentity=station, - timeseries_group__variable__descr="Hello", target_time_step="1h", method="sum", max_missing=max_missing, resulting_timestamp_offset="1min", ) + self.aggregation.timeseries_group.variable.translations.create( + language_code="en", descr="Hello" + ) self.aggregation._htimeseries = HTimeseries(self.source_timeseries) self.aggregation._htimeseries.time_step = "10min" return self.aggregation.process_timeseries().data @@ -283,9 +287,8 @@ class AggregationProcessTimeseriesWhenNoTimeStepTestCase(TestCase): @classmethod def setUpTestData(cls): station = baker.make(Station) - timeseries_group = baker.make( - TimeseriesGroup, gentity=station, variable__descr="hello" - ) + timeseries_group = baker.make(TimeseriesGroup, gentity=station) + timeseries_group.variable.translations.create(language_code="en", descr="hello") cls.aggregation = baker.make( Aggregation, timeseries_group=timeseries_group, @@ -319,9 +322,8 @@ class AggregationRegularizationModeTestCase(TestCase): @classmethod def setUpTestData(cls): station = baker.make(Station) - timeseries_group = baker.make( - TimeseriesGroup, gentity=station, variable__descr="hello" - ) + timeseries_group = baker.make(TimeseriesGroup, gentity=station) + timeseries_group.variable.translations.create(language_code="en", descr="hello") cls.aggregation = baker.make( Aggregation, timeseries_group=timeseries_group, @@ -389,12 +391,8 @@ class AggregationRecalculatesLastValueIfNeededTestCase(TestCase): def setUp(self): station = baker.make(Station, name="Hobbiton", display_timezone="Etc/GMT-2") - timeseries_group = baker.make( - TimeseriesGroup, - gentity=station, - variable__descr="h", - precision=0, - ) + timeseries_group = baker.make(TimeseriesGroup, gentity=station, precision=0) + timeseries_group.variable.translations.create(language_code="en", descr="h") source_timeseries = baker.make( Timeseries, timeseries_group=timeseries_group, @@ -482,12 +480,8 @@ class AggregationTooFewValuesTestCase(TestCase): def setUp(self): station = baker.make(Station, name="Hobbiton", display_timezone="Etc/GMT-2") - timeseries_group = baker.make( - TimeseriesGroup, - gentity=station, - variable__descr="h", - precision=0, - ) + timeseries_group = baker.make(TimeseriesGroup, gentity=station, precision=0) + timeseries_group.variable.translations.create(language_code="en", descr="h") source_timeseries = baker.make( Timeseries, timeseries_group=timeseries_group, diff --git a/enhydris/autoprocess/tests/test_models/test_autoprocess.py b/enhydris/autoprocess/tests/test_models/test_autoprocess.py index 15987bf2..ca41beb2 100644 --- a/enhydris/autoprocess/tests/test_models/test_autoprocess.py +++ b/enhydris/autoprocess/tests/test_models/test_autoprocess.py @@ -81,10 +81,9 @@ def test_auto_process_is_not_triggered_before_commit(self): class AutoProcessExecuteTestCase(ClearCacheMixin, TestCase): def setUp(self): station = baker.make(Station, display_timezone="Etc/GMT-2") - self.timeseries_group = baker.make( - TimeseriesGroup, - gentity=station, - variable__descr="irrelevant", + self.timeseries_group = baker.make(TimeseriesGroup, gentity=station) + self.timeseries_group.variable.translations.create( + language_code="en", descr="irrelevant" ) self.checks = baker.make(Checks, timeseries_group=self.timeseries_group) self.range_check = baker.make(RangeCheck, checks=self.checks) @@ -115,10 +114,9 @@ class AutoProcessRecalculateTestCaseBase(TestCase): def setUp(self, m: mock.MagicMock): self.mock_process_timeseries = m station = baker.make(Station, display_timezone="Etc/GMT-2") - self.timeseries_group = baker.make( - TimeseriesGroup, - gentity=station, - variable__descr="h", + self.timeseries_group = baker.make(TimeseriesGroup, gentity=station) + self.timeseries_group.variable.translations.create( + language_code="en", descr="h" ) self.source_timeseries = baker.make( Timeseries, timeseries_group=self.timeseries_group, type=Timeseries.INITIAL diff --git a/enhydris/autoprocess/tests/test_models/test_checks.py b/enhydris/autoprocess/tests/test_models/test_checks.py index 60e332ab..18c453ab 100644 --- a/enhydris/autoprocess/tests/test_models/test_checks.py +++ b/enhydris/autoprocess/tests/test_models/test_checks.py @@ -92,23 +92,29 @@ def test_target_timeseries_has_same_time_step_as_source(self): @mock.patch("enhydris.models.Timeseries.insert_or_append_data") def test_runs_range_check(self, m1, m2): station = baker.make(Station, display_timezone="Etc/GMT") - range_check = baker.make( - RangeCheck, - checks__timeseries_group__gentity=station, - checks__timeseries_group__variable__descr="Temperature", + range_check = baker.make(RangeCheck, checks__timeseries_group__gentity=station) + range_check.checks.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" ) range_check.checks.execute(recalculate=False) m2.assert_called_once() def test_no_extra_queries_for_str(self): - baker.make(Checks, timeseries_group__variable__descr="Temperature") - with self.assertNumQueries(1): + checks = baker.make(Checks) + checks.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" + ) + # One query for the checks, one for prefetching of variable translations + with self.assertNumQueries(2): str(Checks.objects.first()) class ChecksAutoDeletionTestCase(TestCase): def setUp(self): - self.checks = baker.make(Checks, timeseries_group__variable__descr="pH") + self.checks = baker.make(Checks) + self.checks.timeseries_group.variable.translations.create( + language_code="en", descr="pH" + ) self.range_check = baker.make(RangeCheck, checks=self.checks) self.roc_check = baker.make(RateOfChangeCheck, checks=self.checks) diff --git a/enhydris/migrations/0121_remove_parler_part1.py b/enhydris/migrations/0121_remove_parler_part1.py new file mode 100644 index 00000000..170cf02a --- /dev/null +++ b/enhydris/migrations/0121_remove_parler_part1.py @@ -0,0 +1,43 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("enhydris", "0120_remove_gpoint_original_srid"), + ] + + operations = [ + migrations.CreateModel( + name="NewVariableTranslation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "language_code", + models.CharField(max_length=15, verbose_name="Language"), + ), + ("descr", models.CharField(max_length=200, verbose_name="Description")), + ( + "variable", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="translations", + to="enhydris.variable", + verbose_name="Variable", + ), + ), + ], + options={ + "unique_together": {("variable", "language_code")}, + }, + ), + ] diff --git a/enhydris/migrations/0122_remove_parler_part2.py b/enhydris/migrations/0122_remove_parler_part2.py new file mode 100644 index 00000000..adc71954 --- /dev/null +++ b/enhydris/migrations/0122_remove_parler_part2.py @@ -0,0 +1,36 @@ +from django.apps.registry import Apps +from django.db import migrations + + +def forward(apps: Apps, _) -> None: + VariableTranslation = apps.get_model("enhydris", "VariableTranslation") + NewVariableTranslation = apps.get_model("enhydris", "NewVariableTranslation") + + for vt in VariableTranslation.objects.all(): + NewVariableTranslation.objects.create( + variable_id=vt.master_id, + language_code=vt.language_code, + descr=vt.descr, + ) + + +def backward(apps: Apps, _) -> None: + VariableTranslation = apps.get_model("enhydris", "VariableTranslation") + NewVariableTranslation = apps.get_model("enhydris", "NewVariableTranslation") + for nvt in NewVariableTranslation.objects.all(): + VariableTranslation.objects.create( + master_id=nvt.variable_id, + language_code=nvt.language_code, + descr=nvt.descr, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("enhydris", "0121_remove_parler_part1"), + ] + + operations = [ + migrations.RunPython(forward, backward), + ] diff --git a/enhydris/migrations/0123_remove_parler_part3.py b/enhydris/migrations/0123_remove_parler_part3.py new file mode 100644 index 00000000..0194771f --- /dev/null +++ b/enhydris/migrations/0123_remove_parler_part3.py @@ -0,0 +1,18 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("enhydris", "0122_remove_parler_part2"), + ] + + operations = [ + migrations.DeleteModel( + name="VariableTranslation", + ), + migrations.RenameModel( + old_name="NewVariableTranslation", + new_name="VariableTranslation", + ), + ] diff --git a/enhydris/models/__init__.py b/enhydris/models/__init__.py index d16b3ca7..d3a28328 100644 --- a/enhydris/models/__init__.py +++ b/enhydris/models/__init__.py @@ -13,7 +13,12 @@ ) from .lentity import Lentity, Organization, Person from .timeseries import Timeseries, TimeseriesRecord, TimeseriesStorage, check_time_step -from .timeseries_group import TimeseriesGroup, UnitOfMeasurement, Variable +from .timeseries_group import ( + TimeseriesGroup, + UnitOfMeasurement, + Variable, + VariableTranslation, +) __all__ = ( "DISPLAY_TIMEZONE_CHOICES", @@ -35,5 +40,6 @@ "check_time_step", "TimeseriesGroup", "Variable", + "VariableTranslation", "UnitOfMeasurement", ) diff --git a/enhydris/models/timeseries_group.py b/enhydris/models/timeseries_group.py index 56ba0671..95805799 100644 --- a/enhydris/models/timeseries_group.py +++ b/enhydris/models/timeseries_group.py @@ -6,17 +6,14 @@ from django.conf import settings from django.contrib.gis.db import models from django.core.cache import cache -from django.db.models import FilteredRelation, Q +from django.db.models import FilteredRelation, Q, Value from django.db.models.functions import Coalesce from django.db.models.manager import Manager from django.utils.functional import cached_property from django.utils.timezone import now +from django.utils.translation import get_language from django.utils.translation import gettext_lazy as _ -from parler.managers import TranslatableManager -from parler.models import TranslatableModel, TranslatedFields -from parler.utils import get_active_language_choices - from .base import Lookup from .gentity import Gentity @@ -24,43 +21,29 @@ from enhydris.models import Timeseries -class VariableManager(TranslatableManager): +class VariableManager(models.Manager["Variable"]): def get_queryset(self): - try: - langs = get_active_language_choices() - lang1 = langs[0] - lang2 = langs[1] if len(langs) > 1 else "nonexistent" - except ValueError: - lang1 = settings.LANGUAGE_CODE - try: - lang2 = settings.LANGUAGES[1][0] - except IndexError: - lang2 = "nonexistent" + lang1 = get_language() + lang2 = settings.LANGUAGE_CODE return ( super() .get_queryset() - .annotate( - translation1=FilteredRelation( - "translations", condition=Q(translations__language_code=lang1) - ) - ) - .annotate( - translation2=FilteredRelation( - "translations", condition=Q(translations__language_code=lang2) - ) - ) - .annotate(descr=Coalesce("translation1__descr", "translation2__descr")) - .order_by("descr") + .annotate(t1=self._filtered_relation(lang1)) + .annotate(t2=self._filtered_relation(lang2)) + .annotate(sort_key=Coalesce("t1__descr", "t2__descr", Value(""))) + .order_by("sort_key") ) + def _filtered_relation(self, language_code: str | None): + return FilteredRelation( + "translations", condition=Q(translations__language_code=language_code) + ) + + +class Variable(models.Model): + translations: Manager["VariableTranslation"] -class Variable(TranslatableModel): last_modified = models.DateTimeField(default=now, null=True, editable=False) - translations = TranslatedFields( - descr=models.CharField( - max_length=200, blank=True, verbose_name=_("Description") - ) - ) objects = VariableManager() @@ -68,13 +51,63 @@ class Meta: verbose_name = _("Variable") verbose_name_plural = _("Variables") + @property + def descr(self) -> str: + # Return the VariableManager's annotation if it exists. + if "sort_key" in vars(self): + return vars(self)["sort_key"] + + language_code = get_language() + descr_cache = vars(self).setdefault("_descr_cache", {}) + if language_code in descr_cache: + return descr_cache[language_code] + prefetched_translations = getattr(self, "prefetched_translations", None) + if prefetched_translations is not None: + for preferred_language in (language_code, settings.LANGUAGE_CODE): + for t in prefetched_translations: + if t.language_code == preferred_language: + descr_cache[language_code] = t.descr + return t.descr + descr_cache[language_code] = "" + return "" + + try: + descr = self.translations.get(language_code=language_code).descr + except VariableTranslation.DoesNotExist: + try: + descr = self.translations.get( + language_code=settings.LANGUAGE_CODE + ).descr + except VariableTranslation.DoesNotExist: + descr = "" + descr_cache[language_code] = descr + return descr + def __str__(self): - # For an explanation of this, see - # enhydris.tests.test_models.VariableTestCase.test_translation_bug() - result = self.descr - if result is None: - return self.translations.first().descr - return result + return self.descr + + +class VariableTranslation(models.Model): + variable: models.ForeignKey[Variable, Variable] = models.ForeignKey( + Variable, + on_delete=models.CASCADE, + related_name="translations", + verbose_name=_("Variable"), + ) + language_code: models.CharField[str, str] = models.CharField( + max_length=15, verbose_name=_("Language") + ) + descr: models.CharField[str, str] = models.CharField( + max_length=200, verbose_name=_("Description") + ) + + class Meta: + unique_together = ( + ( + "variable", + "language_code", + ), + ) class UnitOfMeasurement(Lookup): @@ -146,16 +179,7 @@ class TimeseriesGroup(models.Model): def get_name(self): if self.name: return self.name - try: - return self.variable.descr - except ValueError: - # Sometimes the current language is set to null; this happens particularly - # when the Django admin is recording what changes happened to an object (see - # django.contrib.admin.utils.construct_change_message). In that case, - # django-parler raises a ValueError exception when we attempt to access - # self.variable.descr. Not sure whether this is a Django problem or a - # django-parler problem. Working around by returning the group id. - return f"Timeseries group {self.id}" + return self.variable.descr class Meta: verbose_name = _("Time series group") diff --git a/enhydris/synoptic/tests/data.py b/enhydris/synoptic/tests/data.py index ba90fbf6..6f9097d4 100644 --- a/enhydris/synoptic/tests/data.py +++ b/enhydris/synoptic/tests/data.py @@ -81,10 +81,15 @@ def _create_synoptic_group_stations(self): ) def _create_variables(self): - self.var_rain = baker.make(Variable, descr="Rain") - self.var_temperature = baker.make(Variable, descr="Temperature") - self.var_wind_speed = baker.make(Variable, descr="Wind speed") - self.var_wind_gust = baker.make(Variable, descr="Wind gust") + self.var_rain = self._create_variable("Rain") + self.var_temperature = self._create_variable("Temperature") + self.var_wind_speed = self._create_variable("Wind speed") + self.var_wind_gust = self._create_variable("Wind gust") + + def _create_variable(self, name: str): + variable = Variable.objects.create() + variable.translations.create(language_code="en", descr=name) + return variable def _create_timeseries_groups(self): self._create_timeseries_groups_for_komboti() diff --git a/enhydris/telemetry/tests/test_models.py b/enhydris/telemetry/tests/test_models.py index 6182f670..71366a40 100644 --- a/enhydris/telemetry/tests/test_models.py +++ b/enhydris/telemetry/tests/test_models.py @@ -134,9 +134,11 @@ def setUpTestData(cls): TimeseriesGroup, id=42, gentity=cls.station, - variable__descr="Temperature", precision=1, ) + cls.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" + ) cls.telemetry = baker.make( Telemetry, station=cls.station, diff --git a/enhydris/tests/__init__.py b/enhydris/tests/__init__.py index 01513a2b..30842eb2 100644 --- a/enhydris/tests/__init__.py +++ b/enhydris/tests/__init__.py @@ -18,7 +18,6 @@ import pandas as pd from htimeseries import HTimeseries from model_bakery import baker -from parler.utils.context import switch_language from enhydris import models @@ -58,10 +57,12 @@ def _create_test_timeseries( name="Daily temperature", gentity=cls.station, unit_of_measurement__symbol="mm", - variable__descr="Temperature", precision=1, remarks="This timeseries group rocks", ) + cls.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" + ) more_kwargs: dict[str, Any] = {} if publicly_available is not None: more_kwargs["publicly_available"] = publicly_available @@ -100,10 +101,8 @@ def create_timeseries(cls, publicly_available: bool | None = None): geom=Point(x=21.00000, y=39.00000, srid=4326), display_timezone=cls.timezone, ) - cls.variable = models.Variable() - with switch_language(cls.variable, "en"): - cls.variable.descr = "Beauty" - cls.variable.save() + cls.variable = models.Variable.objects.create() + cls.variable.translations.create(language_code="en", descr="Beauty") cls.timeseries_group = baker.make( models.TimeseriesGroup, gentity=cls.station, diff --git a/enhydris/tests/admin/test_station.py b/enhydris/tests/admin/test_station.py index 993bc147..85955330 100644 --- a/enhydris/tests/admin/test_station.py +++ b/enhydris/tests/admin/test_station.py @@ -1,6 +1,7 @@ import os from io import StringIO from locale import LC_CTYPE, getlocale, setlocale +from typing import Any from unittest import mock from django.contrib.auth.models import Permission, User @@ -522,7 +523,9 @@ def test_form_is_valid(self): class TimeseriesUploadFileMixin: - def _get_basic_form_contents(self): + def _get_basic_form_contents(self) -> dict[str, Any]: + variable = models.Variable.objects.create() + variable.translations.create(language_code="en", descr="myvar") return { "name": "Hobbiton", "owner": models.Organization.objects.create(name="Serial killers SA").id, @@ -532,9 +535,7 @@ def _get_basic_form_contents(self): **get_formset_parameters(self.client, "/admin/enhydris/station/add/"), "timeseriesgroup_set-TOTAL_FORMS": "1", "timeseriesgroup_set-INITIAL_FORMS": "0", - "timeseriesgroup_set-0-variable": baker.make( - models.Variable, descr="myvar" - ).id, + "timeseriesgroup_set-0-variable": variable.pk, "timeseriesgroup_set-0-unit_of_measurement": baker.make( models.UnitOfMeasurement ).id, @@ -699,7 +700,9 @@ def setUp(self): ) self.client.login(username="alice", password="topsecret") - def _get_basic_form_contents(self): + def _get_basic_form_contents(self) -> dict[str, Any]: + variable = models.Variable.objects.create() + variable.translations.create(language_code="en", descr="myvar") return { "name": "Hobbiton", "owner": models.Organization.objects.create(name="Serial killers SA").id, @@ -709,9 +712,7 @@ def _get_basic_form_contents(self): **get_formset_parameters(self.client, "/admin/enhydris/station/add/"), "timeseriesgroup_set-TOTAL_FORMS": "1", "timeseriesgroup_set-INITIAL_FORMS": "0", - "timeseriesgroup_set-0-variable": baker.make( - models.Variable, descr="myvar" - ).id, + "timeseriesgroup_set-0-variable": variable.pk, "timeseriesgroup_set-0-unit_of_measurement": baker.make( models.UnitOfMeasurement ).id, diff --git a/enhydris/tests/test_models/test_gentity.py b/enhydris/tests/test_models/test_gentity.py index f344c13d..4f2a8da6 100644 --- a/enhydris/tests/test_models/test_gentity.py +++ b/enhydris/tests/test_models/test_gentity.py @@ -169,9 +169,11 @@ def setUp(self): self.timeseries_group = baker.make( models.TimeseriesGroup, gentity=self.station, - variable__descr="irrelevant", precision=2, ) + self.timeseries_group.variable.translations.create( + language_code="en", descr="irrelevant" + ) def _create_timeseries( self, diff --git a/enhydris/tests/test_models/test_timeseries.py b/enhydris/tests/test_models/test_timeseries.py index 06b4d9f6..2714a66b 100644 --- a/enhydris/tests/test_models/test_timeseries.py +++ b/enhydris/tests/test_models/test_timeseries.py @@ -438,6 +438,7 @@ def setUpClass(cls): def setUp(self): cache.clear() + self.timeseries.timeseries_group.variable.__dict__.pop("_descr_cache", None) # Make sure we've accessed gpoint already, otherwise it screws up the number of # queries later @@ -450,20 +451,26 @@ def setUp(self): def test_cache(self): self.expected_result = self.original_expected_result - self._get_data_and_check_num_queries(1, start_date=None, end_date=None) + # Two queries: getting data and prefetching variable translations. + self._get_data_and_check_num_queries(2, start_date=None, end_date=None) self._get_data_and_check_num_queries(0, start_date=None, end_date=None) # Check cache invalidation self.timeseries.save() + # This time there's one query; the prefetched variable translations are + # still cached. self._get_data_and_check_num_queries(1, start_date=None, end_date=None) def test_refetches_if_does_not_include_everything_from_start_date(self): start_date_1 = dt.datetime(2017, 12, 1, 1, 0, 0, tzinfo=dt.timezone.utc) self.expected_result = self.original_expected_result.iloc[1:] - self._get_data_and_check_num_queries(1, start_date=start_date_1, end_date=None) + # Two queries: getting data and prefetching variable translations. + self._get_data_and_check_num_queries(2, start_date=start_date_1, end_date=None) start_date_2 = dt.datetime(2017, 11, 1, 1, 0, 0, tzinfo=dt.timezone.utc) self.expected_result = self.original_expected_result + # This time there's one query; the prefetched variable translations are + # still cached. self._get_data_and_check_num_queries(1, start_date=start_date_2, end_date=None) # Try again with start_date_1; this time we should have everything in cache @@ -473,10 +480,13 @@ def test_refetches_if_does_not_include_everything_from_start_date(self): def test_refetches_if_does_not_include_everything_to_end_date(self): end_date_1 = dt.datetime(2018, 10, 1, 1, 0, 0, tzinfo=dt.timezone.utc) self.expected_result = self.original_expected_result.iloc[:-1] - self._get_data_and_check_num_queries(1, start_date=None, end_date=end_date_1) + # Two queries: getting data and prefetching variable translations. + self._get_data_and_check_num_queries(2, start_date=None, end_date=end_date_1) end_date_2 = dt.datetime(2018, 12, 1, 1, 0, 0, tzinfo=dt.timezone.utc) self.expected_result = self.original_expected_result + # This time there's one query; the prefetched variable translations are + # still cached. self._get_data_and_check_num_queries(1, start_date=None, end_date=end_date_2) # Try again with end_date_1; this time we should have everything in cache @@ -526,8 +536,9 @@ def test_cached_empty_dataframe(self): @mock.patch("enhydris.models.timeseries.Timeseries._invalidate_cached_data") def test_race_condition(self, m: mock.MagicMock): - # Populate cache - self._get_data_and_check_num_queries(1, start_date=None, end_date=None) + # Populate cache. There are two queries: one for getting the data and + # one for prefetching variable translations. + self._get_data_and_check_num_queries(2, start_date=None, end_date=None) # Pretend there's a race condition. We delete the time series data, but # because we've mocked _invalidate_cached_data, the cache will not be deleted. diff --git a/enhydris/tests/test_models/test_timeseries_group.py b/enhydris/tests/test_models/test_timeseries_group.py index 6afb19b3..be0d1bd1 100644 --- a/enhydris/tests/test_models/test_timeseries_group.py +++ b/enhydris/tests/test_models/test_timeseries_group.py @@ -7,42 +7,24 @@ from django.utils import translation from model_bakery import baker -from parler.utils.context import switch_language from enhydris import models from enhydris.tests import TimeseriesDataMixin class VariableTestCase(TestCase): - def test_create(self): - gact = models.Variable(descr="Temperature") - gact.save() - self.assertEqual(models.Variable.objects.first().descr, "Temperature") - - def test_update(self): - baker.make(models.Variable, descr="Irrelevant") - gact = models.Variable.objects.first() - gact.descr = "Temperature" - gact.save() - self.assertEqual(models.Variable.objects.first().descr, "Temperature") - - def test_delete(self): - baker.make(models.Variable, descr="Temperature") - gact = models.Variable.objects.first() - gact.delete() - self.assertEqual(models.Variable.objects.count(), 0) - def test_str(self): gact = self._create_variable("Temperature", "Θερμοκρασία") self.assertEqual(str(gact), "Temperature") - with switch_language(gact, "el"): + with translation.override("el"): self.assertEqual(str(gact), "Θερμοκρασία") def test_manager_includes_objects_with_missing_translations(self): - variable = baker.make(models.Variable, descr="hello") + variable = models.Variable.objects.create() + variable.translations.create(language_code="en", descr="hello") self.assertEqual(str(variable), "hello") - with switch_language(variable, "el"): - models.Variable.objects.get(id=variable.id) # Shouldn't raise anything + with translation.override("el"): + models.Variable.objects.get(id=variable.pk) # Shouldn't raise anything def test_sort(self): self._create_variable("Temperature", "Θερμοκρασία") @@ -57,10 +39,14 @@ def test_sort(self): ["Θερμοκρασία", "Υγρασία"], ) - def _create_variable(self, english_name, greek_name): - baker.make(models.Variable, descr=english_name) - variable = models.Variable.objects.get(translations__descr=english_name) - variable.translations.create(language_code="el", descr=greek_name) + def _create_variable(self, english_name: str, greek_name: str): + variable = models.Variable.objects.create() + models.VariableTranslation.objects.create( + variable=variable, language_code="el", descr=greek_name + ) + models.VariableTranslation.objects.create( + variable=variable, language_code="en", descr=english_name + ) return variable @override_settings( @@ -69,28 +55,13 @@ def _create_variable(self, english_name, greek_name): LANGUAGES={("en", "English"), ("el", "Ελληνικά")}, ) def test_translation_bug(self): - # Normally Variable.__str__() should return a simple "return self.descr". - # However, there's a tricky bug somewhere, most probably in django-parler, but I - # can't nail it. Sometimes, when there's no registered translation in the - # active language, self.descr is None (when it should fall back to the fallback - # language). It occurs when trying to visit /admin/enhydris/station/add/, the - # active language is Greek, and one of the variables has an English translation - # but not a Greek translation. We work around it by changing Variable.__str__() - # to return whatever translation exists. - # - # Unfortunately, PARLER_LANGUAGES seems to not be overridable in tests; - # therefore, if you change Variable.__str__() to a simple "return self.descr", - # the only way to make this test fail is by manually specifying this in the - # settings: - # PARLER_LANGUAGES={ - # SITE_ID: [{"code": "en"}, {"code": "el"}], - # "default": {"fallbacks": ["en"], "hide_untranslated": True}, - # } + # The admin must render even when active language has no translation. User.objects.create_user( username="alice", password="topsecret", is_active=True, is_staff=True ) self.client.login(username="alice", password="topsecret") - baker.make(models.Variable, descr="pH") + variable = models.Variable.objects.create() + variable.translations.create(language_code="en", descr="pH") response = self.client.get( "/admin/enhydris/station/add/", HTTP_ACCEPT_LANGUAGE="el" ) @@ -109,8 +80,9 @@ def test_str_when_symbol_is_empty(self): class TimeseriesGroupGetNameTestCase(TestCase): def setUp(self): - self.timeseries_group = baker.make( - models.TimeseriesGroup, variable__descr="Temperature", name="" + self.timeseries_group = baker.make(models.TimeseriesGroup, name="") + self.timeseries_group.variable.translations.create( + language_code="en", descr="Temperature" ) def test_get_name_when_name_is_blank(self): @@ -122,18 +94,12 @@ def test_get_name_when_name_is_not_blank(self): def test_get_name_when_translations_are_inactive(self): with translation.override(None): - self.timeseries_group.variable._current_language = None - self.assertEqual( - self.timeseries_group.get_name(), - f"Timeseries group {self.timeseries_group.id}", - ) + self.assertEqual(self.timeseries_group.get_name(), "Temperature") class TimeseriesGroupDefaultTimeseriesTestCase(TestCase): def setUp(self): - self.timeseries_group = baker.make( - models.TimeseriesGroup, variable__descr="Temperature", name="" - ) + self.timeseries_group = baker.make(models.TimeseriesGroup, name="") self.initial_timeseries = self._make_timeseries(models.Timeseries.INITIAL) self.checked_timeseries = self._make_timeseries(models.Timeseries.CHECKED) self.regularized_timeseries = self._make_timeseries( diff --git a/enhydris_project/settings/__init__.py b/enhydris_project/settings/__init__.py index d916f64e..83650661 100644 --- a/enhydris_project/settings/__init__.py +++ b/enhydris_project/settings/__init__.py @@ -43,7 +43,6 @@ "grappelli", "django.contrib.admin", "rules.apps.AutodiscoverRulesConfig", - "parler", "nested_admin", "crequest", "bootstrap4", diff --git a/enhydris_project/settings/ci.py b/enhydris_project/settings/ci.py index e91cf75e..3686ad15 100644 --- a/enhydris_project/settings/ci.py +++ b/enhydris_project/settings/ci.py @@ -18,10 +18,6 @@ ("en", "English"), ("el", "Ελληνικά"), } -PARLER_LANGUAGES = { - SITE_ID: [{"code": LANGUAGE_CODE}, {"code": "el"}], # NOQA - "default": {"fallbacks": ["en"], "hide_untranslated": True}, -} headless = ChromeOptions() headless.add_argument("--headless")