Merge pull request #1261 from ISISScientificComputing/1260_reduction_host

Author: Dimitar Tasev

WebApp/autoreduce_webapp/autoreduce_webapp/fixtures/run_with_one_variable.json

@@ -1,5 +1,4 @@
-[
-    {
+[{
         "model": "reduction_viewer.reductionrun",
         "pk": 1,
         "fields": {
@@ -14,7 +13,8 @@
             "hidden_in_failviewer": 0,
             "experiment_id": 1,
             "instrument_id": 1,
-            "started_by": -1
+            "started_by": -1,
+            "reduction_host": "test-host-123"
         }
     },
     {

WebApp/autoreduce_webapp/reduction_viewer/migrations/0006_reductionrun_reduction_host.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.2 on 2021-03-29 09:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reduction_viewer', '0005_auto_20210326_1628'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='reductionrun',
+            name='reduction_host',
+            field=models.TextField(blank=True, default='', verbose_name='Reduction hostname'),
+        ),
+    ]

WebApp/autoreduce_webapp/reduction_viewer/models.py

@@ -107,6 +107,7 @@ class ReductionRun(models.Model):
     graph = models.TextField(null=True, blank=True)
     message = models.TextField(null=True, blank=True)
     reduction_log = models.TextField(blank=True)
+    reduction_host = models.TextField(default="", blank=True, verbose_name="Reduction hostname")
     # Scripts should be 100,000 chars or less. The DB supports up to 4GB strings here
     script = models.TextField(blank=False, validators=[MaxLengthValidator(100000)])
 

WebApp/autoreduce_webapp/selenium_tests/pages/run_summary_page.py

@@ -117,6 +117,12 @@ def last_updated_text(self) -> str:
         """
         return self.driver.find_element_by_id("last_updated").text
 
+    def reduction_host_text(self) -> WebElement:
+        """
+        Returns the reduction host text
+        """
+        return self.driver.find_element_by_id("reduction_host").text
+
     def images(self) -> List[WebElement]:
         """
         Returns all image elements on the page.

WebApp/autoreduce_webapp/selenium_tests/tests/test_run_summary.py

@@ -101,6 +101,7 @@ def test_reduction_job_panel_displayed(self):
         assert self.page.instrument_text() == f"Instrument: {run.instrument.name}"
         assert self.page.rb_number_text() == f"RB Number: {run.experiment.reference_number}"
         assert self.page.last_updated_text() == "Last Updated: 19 Oct 2020, 6:35 p.m."
+        assert self.page.reduction_host_text() == "Host:test-host-123"
 
     def test_reduction_job_panel_reset_to_values_first_used_for_run(self):
         """Test that the button to reset the variables to the values first used for the run works"""

WebApp/autoreduce_webapp/templates/run_summary.html

@@ -142,6 +142,7 @@ <h2>Reduction Job #{{ run.title }}</h2>
                                         <em>No software data found</em>
                                     {% endif %}
                                 </div>
+                                <div id="reduction_host"><strong>Host:</strong>{{ run.reduction_host }}</div>
                             </div>
                         </div>
                         <div class="row">

model/database/records.py

@@ -9,18 +9,22 @@
 Contains various helper methods for managing or creating ORM records
 """
 
-import datetime
-import model.database.access
+import socket
+from django.utils import timezone
 
 
-def create_reduction_run_record(experiment, instrument, message, run_version, script_text, status):
+def create_reduction_run_record(experiment, instrument, message, run_version, script_text, status, db_handle=None):
     """
     Creates an ORM record for the given reduction run and returns
     this record without saving it to the DB
     """
-    db_handle = model.database.access.start_database()
+    if not db_handle:
+        # pylint:disable=import-outside-toplevel
+        import model.database.access
+        db_handle = model.database.access.start_database()
+
     data_model = db_handle.data_model
-    time_now = datetime.datetime.utcnow()
+    time_now = timezone.now()
     reduction_run = data_model.ReductionRun(run_number=message.run_number,
                                             run_version=run_version,
                                             run_description=message.description,
@@ -33,5 +37,6 @@ def create_reduction_run_record(experiment, instrument, message, run_version, sc
                                             instrument=instrument,
                                             status_id=status.id,
                                             script=script_text,
-                                            started_by=message.started_by)
+                                            started_by=message.started_by,
+                                            reduction_host=socket.getfqdn())
     return reduction_run

model/database/tests/test_records.py

@@ -9,6 +9,7 @@
 Unit tests for the record helper module
 """
 
+import socket
 from unittest import TestCase, mock
 import model.database.records as records
 
@@ -32,9 +33,8 @@ def test_create_reduction_record_starts_db(db_layer):
         db_layer.start_database.return_value.data_model\
             .ReductionRun.assert_called_once()
 
-    @mock.patch("model.database.access")
-    @mock.patch("datetime.datetime")
-    def test_create_reduction_record_forwards_correctly(self, datetime_patch, db_layer):
+    @mock.patch("model.database.records.timezone")
+    def test_create_reduction_record_forwards_correctly(self, timezone_mock):
         """
         Test: Reduction Record uses args correctly.
         Any fields which are hard-coded are mocked with ANY to prevent
@@ -50,22 +50,23 @@ def test_create_reduction_record_forwards_correctly(self, datetime_patch, db_lay
         mock_script_text = mock.NonCallableMock()
         mock_status = mock.NonCallableMock()
 
-        returned = records.create_reduction_run_record(experiment=mock_experiment,
-                                                       instrument=mock_inst,
-                                                       message=mock_msg,
-                                                       run_version=mock_run_version,
-                                                       script_text=mock_script_text,
-                                                       status=mock_status)
+        with mock.patch("model.database") as db_layer:
+            returned = records.create_reduction_run_record(experiment=mock_experiment,
+                                                           instrument=mock_inst,
+                                                           message=mock_msg,
+                                                           run_version=mock_run_version,
+                                                           script_text=mock_script_text,
+                                                           status=mock_status)
 
-        mock_record_orm = db_layer.start_database.return_value.data_model
+            mock_record_orm = db_layer.access.start_database.return_value.data_model
 
         self.assertEqual(mock_record_orm.ReductionRun.return_value, returned)
 
         mock_record_orm.ReductionRun.assert_called_once_with(
             run_number=mock_msg.run_number,
             run_version=mock_run_version,
-            created=datetime_patch.utcnow.return_value,
-            last_updated=datetime_patch.utcnow.return_value,
+            created=timezone_mock.now.return_value,
+            last_updated=timezone_mock.now.return_value,
             experiment=mock_experiment,
             instrument=mock_inst,
             status_id=mock_status.id,
@@ -75,4 +76,5 @@ def test_create_reduction_record_forwards_correctly(self, datetime_patch, db_lay
             run_description=mock.ANY,
             hidden_in_failviewer=mock.ANY,
             admin_log=mock.ANY,
-            reduction_log=mock.ANY)
+            reduction_log=mock.ANY,
+            reduction_host=socket.getfqdn())

queue_processors/queue_processor/handle_message.py

@@ -13,7 +13,6 @@
 update relevant DB fields or logging out the status.
 """
 import logging
-import socket
 from contextlib import contextmanager
 from typing import Optional
 from django.db import transaction
@@ -133,7 +132,8 @@ def do_create_reduction_record(self, message: Message, experiment, instrument):
                                                                message=message,
                                                                run_version=message.run_version,
                                                                script_text=script_text,
-                                                               status=self.status.get_queued())
+                                                               status=self.status.get_queued(),
+                                                               db_handle=self.database)
         reduction_run.save()
 
         # Create a new data location entry which has a foreign key linking it to the current
@@ -277,7 +277,6 @@ def _common_reduction_run_update(reduction_run, status, message):
         reduction_run.finished = timezone.now()
         reduction_run.message = message.message
         reduction_run.reduction_log = message.reduction_log
-        message.admin_log += "Running on host: %s" % socket.getfqdn()
         reduction_run.admin_log = message.admin_log
 
     @staticmethod

queue_processors/queue_processor/reduction/runner.py

@@ -9,7 +9,6 @@
 
 import io
 import logging
-import socket
 import sys
 import traceback
 
@@ -61,7 +60,6 @@ def reduce(self):
     def _do_reduce(self):
         """Actually do the reduction job."""
         self.message.software = self._get_mantid_version()
-        logger.info("Running on host: %s", socket.gethostname())
         if self.message.description is not None:
             logger.info("DESCRIPTION: %s", self.message.description)