Merge pull request #556 from emthompson-usgs/refactor

Some initial refactoring

docs/developer/readers.md

@@ -93,8 +93,8 @@ import numpy as np
 from obspy.core.utcdatetime import UTCDateTime
 
 # local imports
-from gmprocess.stationtrace import StationTrace
-from gmprocess.stationstream import StationStream
+from gmprocess.core.stationtrace import StationTrace
+from gmprocess.core.stationstream import StationStream
 from gmprocess.io.seedname import get_channel_name, is_channel_north
 
 TEXT_HDR_ROWS = 17

docs/scripting/data-structures/stream-collection.md

@@ -10,7 +10,7 @@ together.
 ```python
 import glob
 from gmprocess.io.read import read_data
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.core.streamcollection import StreamCollection
 
 # these sample files can be found in the repository
 # under gmprocess/data/testdata/knet/us2000cnnl

docs/scripting/scripting.md

@@ -8,10 +8,10 @@ processing steps, and the make plots of the processed waveforms.
 import os
 import pkg_resources
 
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.processing import process_streams
-from gmprocess.event import get_event_object
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.utils.event import get_event_object
 
 # Path to example data
 datapath = os.path.join('data', 'testdata', 'demo', 'ci38457511', 'raw')

docs/scripting/waveform-metrics.md

@@ -7,10 +7,10 @@ metrics.
 import os
 import pkg_resources
 
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.processing import process_streams
-from gmprocess.event import get_event_object
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.utils.event import get_event_object
 from gmprocess.metrics.station_summary import StationSummary
 
 # Path to example data

gmprocess/bin/gmconvert.py

@@ -7,9 +7,9 @@
 import logging
 
 # local imports
-from gmprocess.logging import setup_logger
-from gmprocess.args import add_shared_args
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.utils.logging import setup_logger
+from gmprocess.utils.args import add_shared_args
+from gmprocess.core.streamcollection import StreamCollection
 from gmprocess.io.read import read_data
 from gmprocess.io.read_directory import directory_to_streams
 

gmprocess/bin/gminfo.py

@@ -14,8 +14,8 @@
 
 # local imports
 from gmprocess.io.read import _get_format, read_data
-from gmprocess.args import add_shared_args
-from gmprocess.stationtrace import REV_PROCESS_LEVELS
+from gmprocess.utils.args import add_shared_args
+from gmprocess.core.stationtrace import REV_PROCESS_LEVELS
 
 COLUMNS = ['Filename', 'Format', 'Process Level',
            'Start Time', 'End Time',
@@ -179,6 +179,7 @@ def main():
     # Shared arguments
     parser = add_shared_args(parser)
     args = parser.parse_args()
+
     if not args.concise and args.save:
         msg = '''
         ****************************************************************

gmprocess/bin/gmprocess.py

@@ -17,18 +17,18 @@
 import numpy as np
 
 # local imports
-from gmprocess.args import add_shared_args
+from gmprocess.utils.args import add_shared_args
 from gmprocess.io.fetch_utils import (get_events, update_config,
                                       save_shakemap_amps, download,
                                       draw_stations_map)
-from gmprocess.logging import setup_logger
+from gmprocess.utils.logging import setup_logger
 from gmprocess.io.asdf.stream_workspace import StreamWorkspace
-from gmprocess.processing import process_streams
-from gmprocess.report import build_report_latex
-from gmprocess.plot import summary_plots, plot_regression, plot_moveout
-from gmprocess.config import get_config
-from gmprocess.tables import set_precisions
-from gmprocess.constants import DEFAULT_FLOAT_FORMAT, DEFAULT_NA_REP
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.io.report import build_report_latex
+from gmprocess.utils.plot import summary_plots, plot_regression, plot_moveout
+from gmprocess.utils.config import get_config
+from gmprocess.utils.tables import set_precisions
+from gmprocess.utils.constants import DEFAULT_FLOAT_FORMAT, DEFAULT_NA_REP
 
 NON_IMT_COLS = set(['EarthquakeId',
                     'EarthquakeTime',

gmprocess/bin/gmsetup.py

@@ -10,8 +10,8 @@
 import pkg_resources
 
 # local imports
-from gmprocess.constants import CONFIG_FILE_PRODUCTION
-from gmprocess.args import add_shared_args
+from gmprocess.utils.constants import CONFIG_FILE_PRODUCTION
+from gmprocess.utils.args import add_shared_args
 
 
 def main():

gmprocess/stationstream.py → gmprocess/core/stationstream.py

@@ -11,7 +11,7 @@
                                   Response, InstrumentSensitivity)
 # local imports
 from .stationtrace import StationTrace
-from gmprocess.exception import GMProcessException
+from gmprocess.utils.exception import GMProcessException
 
 UNITS = {
     'acc': 'cm/s/s',

gmprocess/stationtrace.py → gmprocess/core/stationtrace.py

@@ -17,7 +17,7 @@
 
 # local imports
 from gmprocess._version import get_versions
-from gmprocess.config import get_config
+from gmprocess.utils.config import get_config
 from gmprocess.io.seedname import get_units_type
 
 UNITS = {'acc': 'cm/s^2',

gmprocess/streamcollection.py → gmprocess/core/streamcollection.py

@@ -16,12 +16,12 @@
 import pandas as pd
 import numpy as np
 
-from gmprocess.exception import GMProcessException
+from gmprocess.utils.exception import GMProcessException
 from gmprocess.metrics.station_summary import StationSummary
-from gmprocess.stationtrace import REV_PROCESS_LEVELS
-from gmprocess.stationstream import StationStream
+from gmprocess.core.stationtrace import REV_PROCESS_LEVELS
+from gmprocess.core.stationstream import StationStream
 from gmprocess.io.read_directory import directory_to_streams
-from gmprocess.config import get_config
+from gmprocess.utils.config import get_config
 
 
 INDENT = 2

gmprocess/io/asdf/core.py

@@ -27,7 +27,8 @@ def is_asdf(filename):
 
 
 def read_asdf(filename, eventid=None, stations=None, label=None):
-    """Read Streams of data (complete with processing metadata) from an ASDF file.
+    """Read Streams of data (complete with processing metadata) from an ASDF
+    file.
 
     Args:
         filename (str):

gmprocess/io/asdf/stream_workspace.py

@@ -18,13 +18,13 @@
 from mapio.gmt import GMTGrid
 
 # local imports
-from gmprocess.stationtrace import (StationTrace, TIMEFMT_MS, NS_SEIS,
-                                    _get_person_agent, _get_software_agent)
-from gmprocess.stationstream import StationStream
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.core.stationtrace import (
+    StationTrace, TIMEFMT_MS, NS_SEIS, _get_person_agent, _get_software_agent)
+from gmprocess.core.stationstream import StationStream
+from gmprocess.core.streamcollection import StreamCollection
 from gmprocess.metrics.station_summary import StationSummary, XML_UNITS
-from gmprocess.exception import GMProcessException
-from gmprocess.event import ScalarEvent
+from gmprocess.utils.exception import GMProcessException
+from gmprocess.utils.event import ScalarEvent
 
 TIMEPAT = '[0-9]{4}-[0-9]{2}-[0-9]{2}T'
 EVENT_TABLE_COLUMNS = ['id', 'time', 'latitude',
@@ -139,7 +139,7 @@ def format_netsta(stats):
 
 
 def format_nslc(stats):
-    #loc = '' if stats.location == '--' else stats.location
+    # loc = '' if stats.location == '--' else stats.location
     return '{st.network}.{st.station}.{st.location}.{st.channel}'.format(
         st=stats)
 
@@ -149,7 +149,7 @@ def format_nslct(stats, tag):
 
 
 def format_nslit(stats, inst, tag):
-    #loc = '' if stats.location == '--' else stats.location
+    # loc = '' if stats.location == '--' else stats.location
     return '{st.network}.{st.station}.{st.location}.{inst}_{tag}'.format(
         st=stats, inst=inst, tag=tag)
 
@@ -362,15 +362,16 @@ def addStreams(self, event, streams, label=None):
                     base_dtype = ''.join([part.capitalize()
                                           for part in name_parts])
                     for array_name, array in spectrum.items():
-                        path = base_dtype + array_name.capitalize() + "/" + procname
+                        path = base_dtype + array_name.capitalize() \
+                            + "/" + procname
                         try:
                             self.dataset.add_auxiliary_data(
                                 array,
                                 data_type='Cache',
                                 path=path,
                                 parameters={}
                             )
-                        except Exception as e:
+                        except Exception:
                             pass
 
             inventory = stream.getInventory()
@@ -418,10 +419,11 @@ def getStreams(self, eventid, stations=None, labels=None):
         """
         trace_auxholder = []
         stream_auxholder = []
-        if 'TraceProcessingParameters' in self.dataset.auxiliary_data:
-            trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters
-        if 'StreamProcessingParameters' in self.dataset.auxiliary_data:
-            stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters
+        auxdata = self.dataset.auxiliary_data
+        if 'TraceProcessingParameters' in auxdata:
+            trace_auxholder = auxdata.TraceProcessingParameters
+        if 'StreamProcessingParameters' in auxdata:
+            stream_auxholder = auxdata.StreamProcessingParameters
         streams = []
 
         if stations is None:
@@ -477,9 +479,10 @@ def getStreams(self, eventid, stations=None, labels=None):
                     # get the trace spectra arrays from auxiliary,
                     # repack into stationtrace object
                     spectra = {}
-                    if 'Cache' in self.dataset.auxiliary_data:
-                        for aux in self.dataset.auxiliary_data['Cache'].list():
-                            auxarray = self.dataset.auxiliary_data['Cache'][aux]
+
+                    if 'Cache' in auxdata:
+                        for aux in auxdata['Cache'].list():
+                            auxarray = auxdata['Cache'][aux]
                             if top not in auxarray.list():
                                 continue
                             auxarray_top = auxarray[top]
@@ -650,7 +653,8 @@ def calcMetrics(self, eventid, stations=None, labels=None, config=None,
                 self.insert_aux(xmlstr, 'StationMetrics', metricpath)
 
     def getTables(self, label, streams=None, stream_label=None):
-        '''Retrieve dataframes containing event information and IMC/IMT metrics.
+        '''Retrieve dataframes containing event information and IMC/IMT
+        metrics.
 
         Args:
             label (str):
@@ -850,7 +854,8 @@ def getFitSpectraTable(self, eventid, label, streams):
 
     def getStreamMetrics(self, eventid, network, station, label, streams=None,
                          stream_label=None):
-        """Extract a StationSummary object from the ASDF file for a given input Stream.
+        """Extract a StationSummary object from the ASDF file for a given
+        input Stream.
 
         Args:
             eventid (str):
@@ -906,7 +911,8 @@ def getStreamMetrics(self, eventid, network, station, label, streams=None,
         if top in auxholder:
             tauxholder = auxholder[top]
             if metricpath not in tauxholder:
-                fmt = 'Stream metrics path (%s) not in WaveFormMetrics auxiliary_data.'
+                fmt = ('Stream metrics path (%s) not in WaveFormMetrics '
+                       'auxiliary_data.')
                 logging.warning(fmt % metricpath)
                 return None
 

gmprocess/io/asdf/utils.py

@@ -4,6 +4,7 @@
 # third party imports
 import h5py
 
+
 class TallyStorage(object):
     """Tally storage used within each group.
     """
@@ -72,7 +73,8 @@ def compute_storage(self, items, store_subtotals=False):
                         'groups': {}
                     }
             else:
-                raise ValueError("Group item '{}' is of type '{}', expected "
-                                 "'h5.Dataset' or 'h5.Group'".format(name, type(item)))
+                raise ValueError(
+                    "Group item '{}' is of type '{}', expected "
+                    "'h5.Dataset' or 'h5.Group'".format(name, type(item)))
             subtotal_bytes += item_bytes
         return (subtotal_bytes, storage)

gmprocess/io/bhrc/core.py

@@ -6,12 +6,12 @@
 
 # third party imports
 import numpy as np
-from gmprocess.constants import UNIT_CONVERSIONS
 from obspy.core.utcdatetime import UTCDateTime
 
 # local
-from gmprocess.stationstream import StationStream
-from gmprocess.stationtrace import StationTrace, PROCESS_LEVELS
+from gmprocess.utils.constants import UNIT_CONVERSIONS
+from gmprocess.core.stationstream import StationStream
+from gmprocess.core.stationtrace import StationTrace, PROCESS_LEVELS
 from gmprocess.io.seedname import get_channel_name, get_units_type
 
 

gmprocess/io/cosmos/cesmd_fetcher.py

@@ -11,11 +11,12 @@
 # local imports
 from gmprocess.io.fetcher import DataFetcher, _get_first_value
 from gmprocess.io.read import read_data
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.exception import GMProcessException
-from gmprocess.io.cosmos.cesmd_search import (get_records, get_metadata,
-                                              get_stations_dataframe)
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.utils.exception import GMProcessException
+from gmprocess.io.cosmos.cesmd_search import (
+    get_records, get_metadata,
+    get_stations_dataframe)
 
 # default values for this fetcher
 # if None specified in constructor, AND no parameters specified in
@@ -271,8 +272,8 @@ def retrieveData(self, event_dict):
         else:
             # web service has a maximum number of stations you're allowed to
             # fetch (note that this may not be the same as the number of files)
-            # so we're splitting up the stations by distance and downloading them
-            # in chunks.
+            # so we're splitting up the stations by distance and downloading
+            # them in chunks.
             dataframe = get_stations_dataframe(event)
             distances = dataframe['epidist'].to_numpy()
             distances.sort()
@@ -300,14 +301,18 @@ def retrieveData(self, event_dict):
                 except GMProcessException as gpe:
                     eqfmt = 'M%.1f %s'
                     eqdesc = eqfmt % (
-                        self.magnitude, self.time.strftime('%Y-%m-%d %H:%M:%S'))
+                        self.magnitude,
+                        self.time.strftime('%Y-%m-%d %H:%M:%S')
+                    )
                     if '404' in str(gpe):
                         fmt = ('Could not find data records for %s '
                                'between %.1f km and %.1f km')
                         logging.info(fmt % (eqdesc, mindist, maxdist))
                     else:
                         logging.warning(
-                            'Unplanned exception getting records for %s' % eqdesc)
+                            'Unplanned exception getting records for %s'
+                            % eqdesc
+                        )
                     continue
                 datafiles += tfiles