diff --git a/spec/ndx-pose.extensions.yaml b/spec/ndx-pose.extensions.yaml index b67af9c..180833b 100644 --- a/spec/ndx-pose.extensions.yaml +++ b/spec/ndx-pose.extensions.yaml @@ -86,7 +86,8 @@ groups: shape: - null doc: Paths to the original video files. The number of files should equal the number - of camera devices. + of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in original_videos_series + instead quantity: '?' - name: labeled_videos dtype: text @@ -95,7 +96,8 @@ groups: shape: - null doc: Paths to the labeled video files. The number of files should equal the number - of camera devices. + of camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in labeled_videos_series + instead quantity: '?' - name: dimensions dtype: uint8 @@ -105,7 +107,8 @@ groups: shape: - null - 2 - doc: Dimensions of each labeled video file. + doc: Dimensions of each labeled video file. Deprecated in version 0.2.0. Use "dimension" + in original_videos_series instead. quantity: '?' - name: scorer dtype: text @@ -200,9 +203,35 @@ groups: doc: Group that holds images, ground-truth annotations, and metadata for training a pose estimator. groups: + - neurodata_type_inc: PoseEstimationSeries + doc: Estimated position data for each body part. Deprecated in version 0.2.0. + Place the PoseEstimationSeries in the "pose_estimation_series" group instead. - neurodata_type_inc: Skeleton doc: Skeleton used in project where each skeleton corresponds to a unique morphology. quantity: '*' - neurodata_type_inc: TrainingFrame doc: Frames and ground-truth annotations for training a pose estimator. quantity: '*' + - name: pose_estimation_series + doc: Estimated position data for each body part. + quantity: '?' + groups: + - neurodata_type_inc: PoseEstimationSeries + doc: Estimated position data for each body part. + quantity: '*' + - name: original_videos_series + doc: Links to the original video files. + quantity: '?' + links: + - target_type: ImageSeries + doc: Links to the original video files. + quantity: '*' + - name: labeled_videos_series + doc: The labeled videos. The number of files should equal the number of original + videos. + quantity: '?' + datasets: + - neurodata_type_inc: ImageSeries + doc: The labeled videos. The number of files should equal the number of original + videos. + quantity: '*' diff --git a/spec/ndx-pose.namespace.yaml b/spec/ndx-pose.namespace.yaml index d249699..8ca949e 100644 --- a/spec/ndx-pose.namespace.yaml +++ b/spec/ndx-pose.namespace.yaml @@ -19,4 +19,4 @@ namespaces: - SpatialSeries - NWBDataInterface - source: ndx-pose.extensions.yaml - version: 0.1.1 + version: 0.2.0 diff --git a/src/pynwb/ndx_pose/io/pose.py b/src/pynwb/ndx_pose/io/pose.py index 9d7b873..787d492 100644 --- a/src/pynwb/ndx_pose/io/pose.py +++ b/src/pynwb/ndx_pose/io/pose.py @@ -21,3 +21,18 @@ def __init__(self, spec): super().__init__(spec) source_software_spec = self.spec.get_dataset('source_software') self.map_spec('source_software_version', source_software_spec.get_attribute('version')) + + # TODO if reading a file without the pose_estimates group, load the PoseEstimationSeries from the + # main PoseEstimation group into the pose_estimation_series variable + + pose_estimates_spec = self.spec.get_group('pose_estimation_series') + self.unmap(pose_estimates_spec) + self.map_spec('pose_estimation_series', pose_estimates_spec.get_neurodata_type('PoseEstimationSeries')) + + original_videos_series_spec = self.spec.get_group('original_videos_series') + self.unmap(original_videos_series_spec) + self.map_spec('original_videos_series', original_videos_series_spec.get_target_type('ImageSeries')) + + labeled_videos_series_spec = self.spec.get_group('labeled_videos_series') + self.unmap(labeled_videos_series_spec) + self.map_spec('labeled_videos_series', labeled_videos_series_spec.get_neurodata_type('ImageSeries')) diff --git a/src/pynwb/ndx_pose/pose.py b/src/pynwb/ndx_pose/pose.py index c3533df..9a97b12 100644 --- a/src/pynwb/ndx_pose/pose.py +++ b/src/pynwb/ndx_pose/pose.py @@ -1,9 +1,11 @@ -from hdmf.utils import docval, popargs, get_docval, call_docval_func, AllowPositional +from hdmf.utils import docval, popargs, get_docval, AllowPositional from pynwb import register_class, TimeSeries from pynwb.behavior import SpatialSeries from pynwb.core import MultiContainerInterface -from pynwb.device import Device +from pynwb.image import ImageSeries + +import warnings @register_class('PoseEstimationSeries', 'ndx-pose') @@ -23,7 +25,7 @@ class PoseEstimationSeries(SpatialSeries): 'doc': ('Estimated position (x, y) or (x, y, z).')}, {'name': 'reference_frame', 'type': str, # required 'doc': 'Description defining what the zero-position (0, 0) or (0, 0, 0) is.'}, - {'name': 'confidence', 'type': ('array_data', 'data'), 'shape': (None, ), + {'name': 'confidence', 'type': ('array_data', 'data'), 'shape': (None, ), 'doc': ('Confidence or likelihood of the estimated positions, scaled to be between 0 and 1.'), 'default': None,}, {'name': 'unit', 'type': str, @@ -42,7 +44,7 @@ class PoseEstimationSeries(SpatialSeries): def __init__(self, **kwargs): """Construct a new PoseEstimationSeries representing pose estimates for a particular body part.""" confidence, confidence_definition = popargs('confidence', 'confidence_definition', kwargs) - call_docval_func(super().__init__, kwargs) + super().__init__(**kwargs) self.confidence = confidence self.confidence_definition = confidence_definition @@ -55,30 +57,40 @@ class PoseEstimation(MultiContainerInterface): __clsconf__ = [ { + # NOTE pose_estimation_series was remapped in version 0.2.0 to live in the pose_estimation_series subgroup 'add': 'add_pose_estimation_series', 'get': 'get_pose_estimation_series', 'create': 'create_pose_estimation_series', 'type': PoseEstimationSeries, 'attr': 'pose_estimation_series' }, - # { - # 'add': 'add_device', - # 'get': 'get_devices', - # 'type': Device, - # 'attr': 'devices' - # # TODO prevent these from being children / add better support for links - # # may require update to HDMF to add a key 'child': False - # } + { + 'add': 'add_original_videos_series', + 'get': 'get_original_videos_series', + 'create': 'create_original_videos_series', + 'type': ImageSeries, + 'attr': 'original_videos_series' + }, + { # TODO how to check that these are links and not subgroups? + 'add': 'add_labeled_videos_series', + 'get': 'get_labeled_videos_series', + 'create': 'create_labeled_videos_series', + 'type': ImageSeries, + 'attr': 'labeled_videos_series' + }, ] - __nwbfields__ = ('description', 'original_videos', 'labeled_videos', 'dimensions', 'scorer', 'source_software', - 'source_software_version', 'nodes', 'edges') + __nwbfields__ = ('description', 'original_videos', 'labeled_videos', 'dimensions', 'scorer', + 'source_software', 'source_software_version', 'nodes', 'edges') # custom mapper in ndx_pose.io.pose maps: - # 'source_software' dataset -> 'version' attribute to 'source_software_version' field + # 'source_software' dataset, 'version' attribute to 'source_software_version' field + # 'pose_estimation_series' untyped group, 'PoseEstimationSeries' subgroup to 'pose_estimation_series' field + # 'original_videos_series' untyped group, 'ImageSeries' subgroup to 'original_videos_series' field + # 'labeled_videos_series' untyped group, 'ImageSeries' subgroup to 'labeled_videos_series' field @docval( # all fields optional - {'name': 'pose_estimation_series', 'type': ('array_data', 'data'), + {'name': 'pose_estimation_series', 'type': (list, tuple), 'doc': ('Estimated position data for each body part.'), 'default': None}, {'name': 'name', 'type': str, @@ -88,13 +100,20 @@ class PoseEstimation(MultiContainerInterface): 'doc': ('Description of the pose estimation procedure and output.'), 'default': None}, {'name': 'original_videos', 'type': ('array_data', 'data'), 'shape': (None, ), - 'doc': ('Paths to the original video files. The number of files should equal the number of camera devices.'), + 'doc': ('The original video files.'), + 'default': None}, + {'name': 'original_videos_series', 'type': (list, tuple), 'shape': (None, ), + 'doc': ('The original video files.'), 'default': None}, {'name': 'labeled_videos', 'type': ('array_data', 'data'), 'shape': (None, ), - 'doc': ('Paths to the labeled video files. The number of files should equal the number of camera devices.'), + 'doc': ('Links to the labeled video files. The number of files should equal the number of original videos.'), + 'default': None}, + {'name': 'labeled_videos_series', 'type': (list, tuple), 'shape': (None, ), + 'doc': ('Links to the labeled videos. The number of files should equal the number of original videos.'), 'default': None}, {'name': 'dimensions', 'type': ('array_data', 'data'), 'shape': ((None, 2)), - 'doc': ('Dimensions of each labeled video file.'), + 'doc': ('Dimensions of each labeled video file. Deprecated in version 0.2.0. ' + 'Use "dimension" in original_videos instead.'), 'default': None}, {'name': 'scorer', 'type': str, 'doc': ('Name of the scorer / algorithm used.'), @@ -113,38 +132,76 @@ class PoseEstimation(MultiContainerInterface): 'doc': ("Array of pairs of indices corresponding to edges between nodes. Index values correspond to row " "indices of the 'nodes' field. Index values use 0-indexing."), 'default': None}, - # {'name': 'devices', 'type': ('array_data', 'data'), - # 'doc': ('Cameras used to record the videos.'), - # 'default': None}, allow_positional=AllowPositional.ERROR ) def __init__(self, **kwargs): pose_estimation_series, description = popargs('pose_estimation_series', 'description', kwargs) - original_videos, labeled_videos, = popargs('original_videos', 'labeled_videos', kwargs) + original_videos, labeled_videos = popargs('original_videos', 'labeled_videos', kwargs) + original_videos_series = popargs('original_videos_series', kwargs) + labeled_videos_series = popargs('labeled_videos_series', kwargs) dimensions, scorer = popargs('dimensions', 'scorer', kwargs) source_software, source_software_version = popargs('source_software', 'source_software_version', kwargs) nodes, edges = popargs('nodes', 'edges', kwargs) - # devices = popargs('devices', kwargs) - call_docval_func(super().__init__, kwargs) + + super().__init__(**kwargs) self.pose_estimation_series = pose_estimation_series self.description = description + + if original_videos is not None: + warnings.warn("The 'original_videos' field has been deprecated in version 0.2.0. Use " + "'original_videos_series' instead. The provided " + "file paths will be converted to ImageSeries objects where the 'external_file' field is set " + "to each file path.", + DeprecationWarning) + if original_videos_series is None: + warnings.warn( + "The provided file paths in 'original_videos' will be converted to ImageSeries objects where the " + "'external_file' field is set to each file path.", + DeprecationWarning + ) + original_videos_series = list() + for i, file_path in enumerate(original_videos): + image_series = ImageSeries( + name="original_video" + str(i), + format="external", + external_file=file_path, + dimension=dimensions[0] if dimensions is not None and dimensions[0] is not None else None, + ) + original_videos_series.append(image_series) self.original_videos = original_videos + self.original_videos_series = original_videos_series + + if labeled_videos is not None: + warnings.warn("The 'labeled_videos' field has been deprecated in version 0.2.0. Use " + "'labeled_videos_series' instead.", DeprecationWarning) + if labeled_videos_series is None: + warnings.warn( + "The provided file paths in 'labeled_videos' will be converted to ImageSeries objects where the " + "'external_file' field is set to each file path.", + DeprecationWarning + ) + labeled_videos_series = list() + for i, file_path in enumerate(labeled_videos): + image_series = ImageSeries( + name="labeled_video" + str(i), + format="external", + external_file=file_path, + dimension=dimensions[0] if dimensions is not None and dimensions[0] is not None else None, + ) + labeled_videos_series.append(image_series) self.labeled_videos = labeled_videos + self.labeled_videos_series = labeled_videos_series + + if dimensions is not None: + warnings.warn("The 'dimensions' field has been deprecated in version 0.2.0. " + "Use 'dimension' in 'original_videos' instead.", DeprecationWarning) self.dimensions = dimensions self.scorer = scorer self.source_software = source_software self.source_software_version = source_software_version self.nodes = nodes self.edges = edges - # self.devices = devices # TODO include calibration images for 3D estimates? - # if original_videos is not None and (devices is None or len(original_videos) != len(devices)): - # raise ValueError("The number of original videos should equal the number of camera devices.") - # if labeled_videos is not None and (devices is None or len(labeled_videos) != len(devices)): - # raise ValueError("The number of labeled videos should equal the number of camera devices.") - # if dimensions is not None and (devices is None or len(dimensions) != len(devices)): - # raise ValueError("The number of dimensions should equal the number of camera devices.") - # TODO validate nodes and edges correspondence, convert edges to uint diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index ee086d5..299ad0b 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -9,7 +9,7 @@ def main(): ns_builder = NWBNamespaceBuilder( doc='NWB extension to store pose estimation data', name='ndx-pose', - version='0.1.1', + version='0.2.0', author=['Ryan Ly', 'Ben Dichter', 'Alexander Mathis', 'Liezl Maree', 'Chris Brozdowski'], contact=['rly@lbl.gov', 'bdichter@lbl.gov', 'alexander.mathis@epfl.ch', 'lmaree@salk.edu', 'cbroz@datajoint.com'], ) @@ -80,6 +80,7 @@ def main(): dtype='float32', dims=['num_frames'], shape=[None], + quantity='?', attributes=[ NWBAttributeSpec( name='definition', @@ -102,9 +103,46 @@ def main(): groups=[ NWBGroupSpec( neurodata_type_inc='PoseEstimationSeries', - doc='Estimated position data for each body part.', + doc=('Estimated position data for each body part. Deprecated in version 0.2.0. Place the ' + 'PoseEstimationSeries in the "pose_estimation_series" group instead.'), quantity='*', ), + NWBGroupSpec( + name="pose_estimation_series", + doc="Estimated position data for each body part.", + groups=[ + NWBGroupSpec( + neurodata_type_inc='PoseEstimationSeries', + doc='Estimated position data for each body part.', + quantity='*', + ) + ], + quantity='?', + ), + NWBGroupSpec( + name="original_videos_series", + doc="Links to the original video files.", + links=[ + NWBLinkSpec( + target_type='ImageSeries', + doc='Links to the original video files.', + quantity='*', + ), + ], + quantity='?', + ), + NWBGroupSpec( + name="labeled_videos_series", + doc="The labeled videos. The number of files should equal the number of original videos.", + datasets=[ + NWBDatasetSpec( + neurodata_type_inc="ImageSeries", + doc='The labeled videos. The number of files should equal the number of original videos.', + quantity='*', + ), + ], + quantity='?', + ), ], links=[ NWBLinkSpec( @@ -122,7 +160,9 @@ def main(): ), NWBDatasetSpec( name='original_videos', - doc='Paths to the original video files. The number of files should equal the number of camera devices.', + doc=('Paths to the original video files. The number of files should equal the number of ' + 'camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in original_videos_series ' + 'instead'), dtype='text', dims=['num_files'], shape=[None], @@ -130,7 +170,9 @@ def main(): ), NWBDatasetSpec( name='labeled_videos', - doc='Paths to the labeled video files. The number of files should equal the number of camera devices.', + doc=('Paths to the labeled video files. The number of files should equal the number of ' + 'camera devices. Deprecated in version 0.2.0. Use ImageSeries objects in labeled_videos_series ' + 'instead'), dtype='text', dims=['num_files'], shape=[None], @@ -138,7 +180,8 @@ def main(): ), NWBDatasetSpec( name='dimensions', - doc='Dimensions of each labeled video file.', + doc=('Dimensions of each labeled video file. Deprecated in version 0.2.0. ' + 'Use "dimension" in original_videos_series instead.'), dtype='uint8', dims=['num_files', 'width, height'], shape=[None, 2], @@ -165,14 +208,6 @@ def main(): ], ), ], - # TODO: collections of multiple links is currently buggy in PyNWB/HDMF - # links=[ - # NWBLinkSpec( - # target_type='Device', - # doc='Cameras used to record the videos.', - # quantity='*', - # ), - # ], ) training_frame = NWBGroupSpec(