Merge pull request #45 from terraref/improve-datapoint-creation-logic

Improve datapoint creation logic & BETY handling

terrautils/betydb.py

@@ -265,4 +265,5 @@ def submit_traits(csv, filetype='csv', betykey='', betyurl=''):
     if resp.status_code in [200,201]:
         logging.info("Data successfully submitted to BETYdb.")
     else:
-        logging.error("Error submitting data to BETYdb: %s" % resp.status_code)
+        logging.error("Error submitting data to BETYdb: %s -- %s" % (resp.status_code, resp.reason))
+        resp.raise_for_status()

terrautils/geostreams.py

@@ -150,7 +150,7 @@ def create_datapoints(connector, host, key, streamid, datapoint_list):
 
 
 def create_datapoint_with_dependencies(connector, host, key, streamprefix, latlon, starttime, endtime,
-                                       metadata={}, filter_date='', geom=None):
+                                       metadata={}, filter_date='', geom=None, plot_name=None):
     """Create a new datapoint in Geostreams. Will create sensor and stream as necessary.
 
     Keyword arguments:
@@ -162,25 +162,39 @@ def create_datapoint_with_dependencies(connector, host, key, streamprefix, latlo
     starttime -- start time, in format 2017-01-25T09:33:02-06:00
     endtime -- end time, in format 2017-01-25T09:33:02-06:00
     metadata -- JSON object with any desired properties
+    filter_date -- date used to restrict number of sites returned from BETYdb
+    geom -- geometry for datapoint (use plot if not provided)
+    plot_name -- name of plot to map datapoint into if possible, otherwise query BETY
     """
 
     # SENSOR is the plot
-    sitelist = get_sites_by_latlon(latlon, filter_date)
     matched_sites = {}
-    for s in sitelist:
-        plot_name = s['sitename']
-        plot_geom = json.loads(wkt_to_geojson(s['geometry']))
-
-        # Get existing sensor with this plot name from geostreams, or create if it doesn't exist
+    if plot_name:
+        # If provided a plot name, see if the sensor exists before going into more expensive logic
         sensor_data = get_sensor_by_name(connector, host, key, plot_name)
-        if not sensor_data:
-            sensor_id = create_sensor(connector, host, key, plot_name, plot_geom,
-                                      {"id": "MAC Field Scanner", "title": "MAC Field Scanner", "sensorType": 4},
-                                      "Maricopa")
-            matched_sites[sensor_id] = {"name": plot_name, "geom": plot_geom}
-        else:
-            sensor_id = sensor_data['id']
-            matched_sites[sensor_id] = {"name": plot_name, "geom": plot_geom}
+        if sensor_data:
+            matched_sites[sensor_data['id']] = {
+                "name": plot_name,
+                "geom": sensor_data['geometry']
+            }
+
+    if matched_sites == {}:
+        # If we don't have existing sensor to use quickly, we must query geographically
+        sitelist = get_sites_by_latlon(latlon, filter_date)
+        for s in sitelist:
+            plot_name = s['sitename']
+            plot_geom = json.loads(wkt_to_geojson(s['geometry']))
+
+            # Get existing sensor with this plot name from geostreams, or create if it doesn't exist
+            sensor_data = get_sensor_by_name(connector, host, key, plot_name)
+            if not sensor_data:
+                sensor_id = create_sensor(connector, host, key, plot_name, plot_geom,
+                                          {"id": "MAC Field Scanner", "title": "MAC Field Scanner", "sensorType": 4},
+                                          "Maricopa")
+                matched_sites[sensor_id] = {"name": plot_name, "geom": plot_geom}
+            else:
+                sensor_id = sensor_data['id']
+                matched_sites[sensor_id] = {"name": plot_name, "geom": plot_geom}
 
     for sensor_id in matched_sites:
         plot_geom = matched_sites[sensor_id]["geom"]

terrautils/lemnatec.py

@@ -400,7 +400,10 @@ def _standardize_gantry_system_variable_metadata(lem_md, filepath=""):
             script_name = match.group(1).lower()
             script_name = re.sub(" ", "_", script_name)
             properties['script_name'] = script_name
-            properties['fullfield_eligible'] = scan_program["fullfield_eligible"]
+            if isinstance(scan_program, list):
+                properties['fullfield_eligible'] = scan_program["fullfield_eligible"]
+            else:
+                properties['fullfield_eligible'] = "True"
 
 
     # Limit output to the following fields for now

terrautils/sensors.py

@@ -174,6 +174,9 @@
         },
 
         'rgb_canopyCover': {
+            'template': '{base}/{station}/Level_1/' + \
+                        '{sensor}/{date}/{filename}',
+            'pattern': '{sensor}_L2_{station}_{date}{opts}.csv',
             'bety_traits': {
                 'canopy_cover': 'canopy_cover'
             },
@@ -283,8 +286,11 @@
 
         'ir_meanTemp': {
             'template': '{base}/{station}/Level_1/' + \
-                        '{sensor}/{date}/{timestamp}/{filename}',
-            'pattern': '{sensor}_L2_{station}_{timestamp}{opts}.tif'
+                        '{sensor}/{date}/{filename}',
+            'pattern': '{sensor}_L2_{station}_{date}{opts}.csv',
+            'bety_traits': {
+                'surface_temperature': 'surface_temperature'
+            },
         }
     },
 }

terrautils/spatial.py

@@ -218,6 +218,28 @@ def geojson_to_tuples(bounding_box):
     return (lat_min, lat_max, long_min, long_max)
 
 
+def geojson_to_tuples_betydb(bounding_box):
+    """Convert GeoJSON from BETYdb to
+        ( lat (y) min, lat (y) max,
+          long (x) min, long (x) max) for geotiff creation"""
+    min_x, min_y, max_x, max_y  = None, None, None, None
+
+    if isinstance(bounding_box, dict):
+        bounding_box = bounding_box["coordinates"]
+
+    for coord in bounding_box[0][0]:
+        if not min_x or coord[0] < min_x:
+            min_x = coord[0]
+        if not max_x or coord[0] > max_x:
+            max_x = coord[0]
+        if not min_y or coord[1] < min_y:
+            min_y = coord[1]
+        if not max_y or coord[1] > max_y:
+            max_y = coord[1]
+
+    return (min_y, max_y, min_x, max_x)
+
+
 def scanalyzer_to_mac(scan_x, scan_y):
     # TODO: Hard-coded
     # Linear transformation coefficients