From 433f232d417335f378b5f8e875b74d85e95f6d8c Mon Sep 17 00:00:00 2001 From: Kalev Takkis Date: Mon, 12 Feb 2024 14:42:43 +0000 Subject: [PATCH 1/4] stashing --- viewer/target_loader.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/viewer/target_loader.py b/viewer/target_loader.py index b9665633..0a61455c 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -1437,12 +1437,33 @@ def process_bundle(self): self.report.log(logging.ERROR, msg) raise KeyError(msg) from exc - # moved this bit from init + try: + config_inputs = config["inputs"] + except KeyError as exc: + msg = "'inputs' key missing in config file" + self.report.log(logging.ERROR, msg) + raise KeyError(msg) from exc + + try: + code_prefix = config_inputs[0]["code_prefix"] + except KeyError as exc: + msg = "'code_prefix' key missing in config file" + self.report.log(logging.ERROR, msg) + raise KeyError(msg) from exc + try: + code_prefix_tooltip = config_inputs[0]["code_prefix_tooltip"] + except KeyError as exc: + msg = "'code_prefix_tooltip' key missing in config file" + self.report.log(logging.ERROR, msg) + raise KeyError(msg) from exc + self.target, target_created = Target.objects.get_or_create( title=self.target_name, display_name=self.target_name, ) + logger.debug("tooltip: %s", code_prefix_tooltip) + # TODO: original target loader's function get_create_projects # seems to handle more cases. adopt or copy visit = self.proposal_ref.split()[0] @@ -1681,20 +1702,21 @@ def process_bundle(self): # technically it should be validated in previous try-catch block logger.error("Non-standard SiteObservation code 2: %s", last) - logger.debug("iter_pos: %s", iter_pos) - # ... and create new one starting from next item suffix = alphanumerator(start_from=iter_pos) for so in so_group.filter(code__isnull=True): - code = f"{so.experiment.code.split('-')[1]}{next(suffix)}" + code = f"{code_prefix}{so.experiment.code.split('-')[1]}{next(suffix)}" # test uniqueness for target # TODO: this should ideally be solved by db engine, before # rushing to write the trigger, have think about the # loader concurrency situations - prefix = alphanumerator() - while code in current_list: - code = f"{next(prefix)}{code}" + if code in current_list: + msg = ( + f"short code {code} already exists for this target; " + + "specify a code_prefix to resolve this conflict" + ) + self.report.log(logging.ERROR, msg) so.code = code so.save() From 7fd97c9569be90769ddce44d1ec5ee011fd14f5e Mon Sep 17 00:00:00 2001 From: Kalev Takkis Date: Tue, 13 Feb 2024 15:25:32 +0000 Subject: [PATCH 2/4] Short code prefix and tooltip to backend Target loader now reads short code prefix and tooltip from meta_aligner.yaml. Tooltip is saved to Experiment model. TODO: make tooltip available via API --- .../0043_experiment_prefix_tooltip.py | 17 +++++++++ viewer/models.py | 1 + viewer/target_loader.py | 37 +++++++------------ 3 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 viewer/migrations/0043_experiment_prefix_tooltip.py diff --git a/viewer/migrations/0043_experiment_prefix_tooltip.py b/viewer/migrations/0043_experiment_prefix_tooltip.py new file mode 100644 index 00000000..93477ed4 --- /dev/null +++ b/viewer/migrations/0043_experiment_prefix_tooltip.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.23 on 2024-02-13 15:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('viewer', '0042_alter_xtalformsite_xtalform_site_num'), + ] + + operations = [ + migrations.AddField( + model_name='experiment', + name='prefix_tooltip', + field=models.TextField(null=True), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index e3839475..c2b8af72 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -196,6 +196,7 @@ class Experiment(models.Model): map_info = ArrayField(models.FileField(max_length=255), null=True) type = models.PositiveSmallIntegerField(null=True) pdb_sha256 = models.TextField(null=True) + prefix_tooltip = models.TextField(null=True) compounds = models.ManyToManyField( "Compound", through="ExperimentCompound", diff --git a/viewer/target_loader.py b/viewer/target_loader.py index 0a61455c..d4845174 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -700,6 +700,7 @@ def _enumerate_objects(self, objects: dict, attr: str) -> None: def process_experiment( self, item_data: tuple[str, dict] | None = None, + prefix_tooltips: dict[str, str] | None = None, validate_files: bool = True, **kwargs, ) -> ProcessedObject | None: @@ -734,6 +735,7 @@ def process_experiment( """ del kwargs assert item_data + assert prefix_tooltips logger.debug("incoming data: %s", item_data) experiment_name, data = item_data @@ -813,6 +815,9 @@ def process_experiment( # version int old versions are kept target loader version = 1 + code_prefix = extract(key="code_prefix") + prefix_tooltip = prefix_tooltips.get(code_prefix, "") + fields = { "code": experiment_name, } @@ -830,6 +835,7 @@ def process_experiment( "mtz_info": str(self._get_final_path(mtz_info)), "cif_info": str(self._get_final_path(cif_info)), "map_info": map_info_paths, + "prefix_tooltip": prefix_tooltip, # this doesn't seem to be present # pdb_sha256: } @@ -839,6 +845,7 @@ def process_experiment( index_fields = { "xtalform": assigned_xtalform, "smiles": smiles, + "code_prefix": code_prefix, } return ProcessedObject( @@ -1437,33 +1444,11 @@ def process_bundle(self): self.report.log(logging.ERROR, msg) raise KeyError(msg) from exc - try: - config_inputs = config["inputs"] - except KeyError as exc: - msg = "'inputs' key missing in config file" - self.report.log(logging.ERROR, msg) - raise KeyError(msg) from exc - - try: - code_prefix = config_inputs[0]["code_prefix"] - except KeyError as exc: - msg = "'code_prefix' key missing in config file" - self.report.log(logging.ERROR, msg) - raise KeyError(msg) from exc - try: - code_prefix_tooltip = config_inputs[0]["code_prefix_tooltip"] - except KeyError as exc: - msg = "'code_prefix_tooltip' key missing in config file" - self.report.log(logging.ERROR, msg) - raise KeyError(msg) from exc - self.target, target_created = Target.objects.get_or_create( title=self.target_name, display_name=self.target_name, ) - logger.debug("tooltip: %s", code_prefix_tooltip) - # TODO: original target loader's function get_create_projects # seems to handle more cases. adopt or copy visit = self.proposal_ref.split()[0] @@ -1496,6 +1481,7 @@ def process_bundle(self): self.version_number = meta["version_number"] self.version_dir = meta["version_dir"] self.previous_version_dirs = meta["previous_version_dirs"] + prefix_tooltips = meta["code_prefix_tooltips"] # check transformation matrix files ( # pylint: disable=unbalanced-tuple-unpacking @@ -1554,7 +1540,9 @@ def process_bundle(self): ), ) - experiment_objects = self.process_experiment(yaml_data=crystals) + experiment_objects = self.process_experiment( + yaml_data=crystals, prefix_tooltips=prefix_tooltips + ) compound_objects = self.process_compound( yaml_data=crystals, experiments=experiment_objects ) @@ -1705,6 +1693,9 @@ def process_bundle(self): # ... and create new one starting from next item suffix = alphanumerator(start_from=iter_pos) for so in so_group.filter(code__isnull=True): + code_prefix = experiment_objects[so.experiment.code].index_data[ + "code_prefix" + ] code = f"{code_prefix}{so.experiment.code.split('-')[1]}{next(suffix)}" # test uniqueness for target From f43eabf186a52518693a1f26c8b2338df2913bd5 Mon Sep 17 00:00:00 2001 From: Kalev Takkis Date: Wed, 14 Feb 2024 12:18:23 +0000 Subject: [PATCH 3/4] Prefix tooltip now serverd by api/site_observation --- viewer/managers.py | 1 + viewer/serializers.py | 1 + viewer/target_loader.py | 10 +++++----- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/viewer/managers.py b/viewer/managers.py index 836ff422..7a1a4826 100644 --- a/viewer/managers.py +++ b/viewer/managers.py @@ -17,6 +17,7 @@ def filter_qs(self): ).annotate( target=F("experiment__experiment_upload__target"), compound_code=F("cmpd__compound_code"), + prefix_tooltip=F("experiment__prefix_tooltip"), ) return qs diff --git a/viewer/serializers.py b/viewer/serializers.py index c969d3da..15093985 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -951,6 +951,7 @@ class Meta: class SiteObservationReadSerializer(serializers.ModelSerializer): compound_code = serializers.StringRelatedField() + prefix_tooltip = serializers.StringRelatedField() class Meta: model = models.SiteObservation diff --git a/viewer/target_loader.py b/viewer/target_loader.py index d4845174..968a6305 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -1652,16 +1652,13 @@ def process_bundle(self): canon_site_confs=canon_site_conf_objects, ) - values = ["xtalform_site__xtalform", "canon_site_conf__canon_site", "cmpd"] + values = ["canon_site_conf__canon_site", "cmpd"] qs = ( SiteObservation.objects.values(*values) .order_by(*values) .annotate(obvs=ArrayAgg("id")) .values_list("obvs", flat=True) ) - current_list = SiteObservation.objects.filter( - experiment__experiment_upload__target=self.target - ).values_list('code', flat=True) for elem in qs: # objects in this group should be named with same scheme so_group = SiteObservation.objects.filter(pk__in=elem) @@ -1702,7 +1699,10 @@ def process_bundle(self): # TODO: this should ideally be solved by db engine, before # rushing to write the trigger, have think about the # loader concurrency situations - if code in current_list: + if SiteObservation.objects.filter( + experiment__experiment_upload__target=self.target, + code=code, + ).exists(): msg = ( f"short code {code} already exists for this target; " + "specify a code_prefix to resolve this conflict" From 632719ac1365522c27ce7faeebac874740b36b14 Mon Sep 17 00:00:00 2001 From: Kalev Takkis Date: Wed, 14 Feb 2024 14:15:01 +0000 Subject: [PATCH 4/4] Site observation groups for shortcodes now by experiment --- viewer/target_loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/viewer/target_loader.py b/viewer/target_loader.py index 968a6305..8878bda9 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -1652,7 +1652,8 @@ def process_bundle(self): canon_site_confs=canon_site_conf_objects, ) - values = ["canon_site_conf__canon_site", "cmpd"] + # values = ["canon_site_conf__canon_site", "cmpd"] + values = ["experiment"] qs = ( SiteObservation.objects.values(*values) .order_by(*values)