Merge branch 'dev' of github.com:biocore/qiita into dev

Author: antgonza

.gitignore

@@ -72,3 +72,6 @@ qiita_pet/*.conf
 
 # jupyter notebooks input data
 notebooks/*/*.tsv.gz
+
+# jupyter notebooks input data
+notebooks/resource-allocation/data

qiita_core/configuration_manager.py

@@ -109,6 +109,12 @@ class ConfigurationManager(object):
         The portal subdirectory used in the URL
     portal_fp : str
         The filepath to the portal styling config file
+    stats_map_center_latitude : float
+        The center latitude of the world map, shown on the Stats map.
+        Defaults to 40.01027 (Boulder, CO, USA)
+    stats_map_center_longitude : float
+        The center longitude of the world map, shown on the Stats map.
+        Defaults to -105.24827 (Boulder, CO, USA)
     qiita_env : str
         The script used to start the qiita environment
     private_launcher : str
@@ -117,6 +123,10 @@ class ConfigurationManager(object):
         The script used to start the plugins
     plugin_dir : str
         The path to the directory containing the plugin configuration files
+    help_email : str
+        The email address a user should write to when asking for help
+    sysadmin_email : str
+        The email address, Qiita sends internal notifications to a sys admin
 
     Raises
     ------
@@ -234,6 +244,32 @@ def _get_main(self, config):
             self.key_file = join(install_dir, 'qiita_core', 'support_files',
                                  'ci_server.key')
 
+        self.help_email = config.get('main', 'HELP_EMAIL')
+        if not self.help_email:
+            raise ValueError(
+                "You did not specify the HELP_EMAIL address in the main "
+                "section of Qiita's config file. This address is essential "
+                "for users to ask for help as it is displayed at various "
+                "location throughout Qiita's web pages.")
+        if (self.help_email == '[email protected]') and \
+           (self.test_environment is False):
+            warnings.warn(
+                "Using the github fake email for HELP_EMAIL, "
+                "are you sure this is OK?")
+
+        self.sysadmin_email = config.get('main', 'SYSADMIN_EMAIL')
+        if not self.sysadmin_email:
+            raise ValueError(
+                "You did not specify the SYSADMIN_EMAIL address in the main "
+                "section of Qiita's config file. Serious issues will "
+                "automatically be reported to a sys admin, an according "
+                "address is therefore required!")
+        if (self.sysadmin_email == '[email protected]') and \
+           (self.test_environment is False):
+            warnings.warn(
+                "Using the github fake email for SYSADMIN_EMAIL, "
+                "are you sure this is OK?")
+
     def _get_job_scheduler(self, config):
         """Get the configuration of the job_scheduler section"""
         self.job_scheduler_owner = config.get(
@@ -317,5 +353,40 @@ def _get_portal(self, config):
         else:
             self.portal_dir = ""
 
+        msg = ("The value %s for %s you set in Qiita's configuration file "
+               "(section 'portal') for the Stats world map cannot be "
+               "intepreted as a float! %s")
+        lat_default = 40.01027  # Boulder CO, USA
+        try:
+            self.stats_map_center_latitude = config.get(
+                'portal', 'STATS_MAP_CENTER_LATITUDE', fallback=lat_default)
+            if self.stats_map_center_latitude == '':
+                self.stats_map_center_latitude = lat_default
+            self.stats_map_center_latitude = float(
+                self.stats_map_center_latitude)
+        except ValueError as e:
+            raise ValueError(msg % (self.stats_map_center_latitude,
+                                    'STATS_MAP_CENTER_LATITUDE', e))
+
+        lon_default = -105.24827  # Boulder CO, USA
+        try:
+            self.stats_map_center_longitude = config.get(
+                'portal', 'STATS_MAP_CENTER_LONGITUDE', fallback=lon_default)
+            if self.stats_map_center_longitude == '':
+                self.stats_map_center_longitude = lon_default
+            self.stats_map_center_longitude = float(
+                self.stats_map_center_longitude)
+        except ValueError as e:
+            raise ValueError(msg % (self.stats_map_center_longitude,
+                                    'STATS_MAP_CENTER_LONGITUDE', e))
+        for (name, val) in [('latitude', self.stats_map_center_latitude),
+                            ('longitude', self.stats_map_center_longitude)]:
+            msg = ("The %s of %s you set in Qiita's configuration file "
+                   "(section 'portal') for the Stats world map cannot be %s!")
+            if val < -180:
+                raise ValueError(msg % (name, val, 'smaller than -180°'))
+            if val > 180:
+                raise ValueError(msg % (name, val, 'larger than 180°'))
+
     def _iframe(self, config):
         self.iframe_qiimp = config.get('iframe', 'QIIMP')

qiita_core/support_files/config_test.cfg

@@ -68,6 +68,12 @@ COOKIE_SECRET = SECRET
 # The value used to secure JWTs for delegated permission artifact download.
 JWT_SECRET = SUPER_SECRET
 
+# Address a user should write to when asking for help
+HELP_EMAIL = [email protected]
+
+# The email address, Qiita sends internal notifications to a sys admin
+SYSADMIN_EMAIL = [email protected]
+
 # ----------------------------- SMTP settings -----------------------------
 [smtp]
 # The hostname to connect to
@@ -177,6 +183,13 @@ PORTAL_DIR =
 # Full path to portal styling config file
 PORTAL_FP =
 
+# The center latitude of the world map, shown on the Stats map.
+# Defaults to 40.01027 (Boulder, CO, USA)
+STATS_MAP_CENTER_LATITUDE =
+
+# The center longitude of the world map, shown on the Stats map.
+# Defaults to -105.24827 (Boulder, CO, USA)
+STATS_MAP_CENTER_LONGITUDE =
 
 # ----------------------------- iframes settings ---------------------------
 [iframe]

qiita_core/tests/test_configuration_manager.py

@@ -29,7 +29,7 @@ def setUp(self):
 
         self.conf = ConfigParser()
         with open(self.conf_fp, newline=None) as f:
-            self.conf.readfp(f)
+            self.conf.read_file(f)
 
     def tearDown(self):
         if self.old_conf_fp is not None:
@@ -132,6 +132,8 @@ def test_get_main(self):
 
         # Warning raised if No files will be allowed to be uploaded
         # Warning raised if no cookie_secret
+        self.conf.set('main', 'HELP_EMAIL', 'ignore@me')
+        self.conf.set('main', 'SYSADMIN_EMAIL', 'ignore@me')
         with warnings.catch_warnings(record=True) as warns:
             obs._get_main(self.conf)
 
@@ -180,6 +182,35 @@ def test_get_main(self):
 
         self.assertEqual(obs.qiita_env, "")
 
+    def test_help_email(self):
+        obs = ConfigurationManager()
+
+        with warnings.catch_warnings(record=True) as warns:
+            # warning get only issued when in non test environment
+            self.conf.set('main', 'TEST_ENVIRONMENT', 'FALSE')
+
+            obs._get_main(self.conf)
+            self.assertEqual(obs.help_email, '[email protected]')
+            self.assertEqual(obs.sysadmin_email, '[email protected]')
+
+            obs_warns = [str(w.message) for w in warns]
+            exp_warns = [
+                'Using the github fake email for HELP_EMAIL, '
+                'are you sure this is OK?',
+                'Using the github fake email for SYSADMIN_EMAIL, '
+                'are you sure this is OK?']
+            self.assertCountEqual(obs_warns, exp_warns)
+
+        # test if it falls back to [email protected]
+        self.conf.set('main', 'HELP_EMAIL', '')
+        with self.assertRaises(ValueError):
+            obs._get_main(self.conf)
+
+        # test if it falls back to [email protected]
+        self.conf.set('main', 'SYSADMIN_EMAIL', '')
+        with self.assertRaises(ValueError):
+            obs._get_main(self.conf)
+
     def test_get_job_scheduler(self):
         obs = ConfigurationManager()
 
@@ -214,6 +245,50 @@ def test_get_portal(self):
         obs._get_portal(self.conf)
         self.assertEqual(obs.portal_dir, "/gold_portal")
 
+    def test_get_portal_latlong(self):
+        obs = ConfigurationManager()
+
+        # if parameters are given, but not set, they should default to Boulder
+        self.assertEqual(obs.stats_map_center_latitude, 40.01027)
+        self.assertEqual(obs.stats_map_center_longitude, -105.24827)
+
+        # a string cannot be parsed as a float
+        self.conf.set('portal', 'STATS_MAP_CENTER_LATITUDE', 'kurt')
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+
+        # check for illegal float values
+        self.conf.set('portal', 'STATS_MAP_CENTER_LATITUDE', "-200")
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+        self.conf.set('portal', 'STATS_MAP_CENTER_LATITUDE', "200")
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+
+        # check if value defaults if option is missing altogether
+        self.conf.remove_option('portal', 'STATS_MAP_CENTER_LATITUDE')
+        obs._get_portal(self.conf)
+        self.assertEqual(obs.stats_map_center_latitude, 40.01027)
+
+        # same as above, but for longitude
+        # a string cannot be parsed as a float
+        self.conf.set('portal', 'STATS_MAP_CENTER_LONGITUDE', 'kurt')
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+
+        # check for illegal float values
+        self.conf.set('portal', 'STATS_MAP_CENTER_LONGITUDE', "-200")
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+        self.conf.set('portal', 'STATS_MAP_CENTER_LONGITUDE', "200")
+        with self.assertRaises(ValueError):
+            obs._get_portal(self.conf)
+
+        # check if value defaults if option is missing altogether
+        self.conf.remove_option('portal', 'STATS_MAP_CENTER_LONGITUDE')
+        obs._get_portal(self.conf)
+        self.assertEqual(obs.stats_map_center_longitude, -105.24827)
+
 
 CONF = """
 # ------------------------------ Main settings --------------------------------
@@ -274,6 +349,12 @@ def test_get_portal(self):
 # The value used to secure JWTs for delegated permission artifact download.
 JWT_SECRET = SUPER_SECRET
 
+# Address a user should write to when asking for help
+HELP_EMAIL = [email protected]
+
+# The email address, Qiita sends internal notifications to a sys admin
+SYSADMIN_EMAIL = [email protected]
+
 # ----------------------------- SMTP settings -----------------------------
 [smtp]
 # The hostname to connect to
@@ -380,6 +461,14 @@ def test_get_portal(self):
 # Full path to portal styling config file
 PORTAL_FP = /tmp/portal.cfg
 
+# The center latitude of the world map, shown on the Stats map.
+# Defaults to 40.01027 (Boulder, CO, USA)
+STATS_MAP_CENTER_LATITUDE =
+
+# The center longitude of the world map, shown on the Stats map.
+# Defaults to -105.24827 (Boulder, CO, USA)
+STATS_MAP_CENTER_LONGITUDE =
+
 # ----------------------------- iframes settings ---------------------------
 [iframe]
 QIIMP = https://localhost:8898/

qiita_db/processing_job.py

@@ -493,7 +493,7 @@ def resource_allocation_info(self):
                 samples, columns, input_size = self.shape
                 parts = []
                 error_msg = ('Obvious incorrect allocation. Please '
-                             'contact [email protected]')
+                             'contact %s' % qiita_config.help_email)
                 for part in allocation.split('--'):
                     param = ''
                     if part.startswith('time '):
@@ -902,7 +902,7 @@ def _set_status(self, value, error_msg=None):
                 if self.user.level in {'admin', 'wet-lab admin'}:
                     if value == 'error':
                         qdb.util.send_email(
-                            '[email protected]', msg['subject'],
+                            qiita_config.sysadmin_email, msg['subject'],
                             msg['message'])
 
             sql = """UPDATE qiita.processing_job

qiita_db/study.py

@@ -1092,7 +1092,7 @@ def artifacts(self, dtype=None, artifact_type=None):
 
             if artifact_type:
                 sql_args.append(artifact_type)
-                sql_where += "AND artifact_type = %s"
+                sql_where += " AND artifact_type = %s"
 
             sql = """SELECT artifact_id
                      FROM qiita.artifact

qiita_db/test/test_data/jobs_2024-02-21.tsv.gz

@@ -21,6 +21,10 @@
 from qiita_core.util import qiita_test_checker
 import qiita_db as qdb
 
+from matplotlib.figure import Figure
+from matplotlib.axes import Axes
+import matplotlib.pyplot as plt
+
 
 @qiita_test_checker()
 class DBUtilTestsBase(TestCase):
@@ -1303,6 +1307,64 @@ def test_quick_mounts_purge(self):
             qdb.util.quick_mounts_purge()
 
 
+class ResourceAllocationPlotTests(TestCase):
+    def setUp(self):
+
+        self.PATH_TO_DATA = ('./qiita_db/test/test_data/'
+                             'jobs_2024-02-21.tsv.gz')
+        self.CNAME = "Validate"
+        self.SNAME = "Diversity types - alpha_vector"
+        self.col_name = 'samples * columns'
+        self.df = pd.read_csv(self.PATH_TO_DATA, sep='\t',
+                              dtype={'extra_info': str})
+
+    def test_plot_return(self):
+        # check the plot returns correct objects
+        fig1, axs1 = qdb.util.resource_allocation_plot(
+            self.PATH_TO_DATA, self.CNAME, self.SNAME, self.col_name)
+        self.assertIsInstance(
+            fig1, Figure,
+            "Returned object fig1 is not a Matplotlib Figure")
+        for ax in axs1:
+            self.assertIsInstance(
+                ax, Axes,
+                "Returned object axs1 is not a single Matplotlib Axes object")
+
+    def test_minimize_const(self):
+        self.df = self.df[
+            (self.df.cName == self.CNAME) & (self.df.sName == self.SNAME)]
+        self.df.dropna(subset=['samples', 'columns'], inplace=True)
+        self.df[self.col_name] = self.df.samples * self.df['columns']
+        fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False)
+
+        bm, options = qdb.util._resource_allocation_plot_helper(
+            self.df, axs[0], self.CNAME, self.SNAME, 'MaxRSSRaw',
+            qdb.util.MODELS_MEM, self.col_name)
+        # check that the algorithm chooses correct model for MaxRSSRaw and
+        # has 0 failures
+        k, a, b = options.x
+        failures_df = qdb.util._resource_allocation_failures(
+            self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')
+        failures = failures_df.shape[0]
+        self.assertEqual(bm, qdb.util.mem_model4, msg="""Best memory model
+                                                doesn't match""")
+        self.assertEqual(failures, 0, "Number of failures must be 0")
+
+        # check that the algorithm chooses correct model for ElapsedRaw and
+        # has 1 failure
+        bm, options = qdb.util._resource_allocation_plot_helper(
+            self.df, axs[1], self.CNAME, self.SNAME, 'ElapsedRaw',
+            qdb.util.MODELS_TIME, self.col_name)
+        k, a, b = options.x
+        failures_df = qdb.util._resource_allocation_failures(
+            self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')
+        failures = failures_df.shape[0]
+
+        self.assertEqual(bm, qdb.util.time_model1, msg="""Best time model
+                                                   doesn't match""")
+        self.assertEqual(failures, 1, "Number of failures must be 1")
+
+
 STUDY_INFO = {
     'study_id': 1,
     'owner': 'Dude',