Source code for McsPy.McsData

"""
    McsData
    ~~~~~~~

    Data classes to wrap and hide raw data handling of the HDF5 data files.
    It is based on the MCS-HDF5 definitions of the given compatible versions.

    :copyright: (c) 2018 by Multi Channel Systems MCS GmbH
    :license: see LICENSE for more details
"""

import h5py
from builtins import IndexError
import datetime
import math
import uuid
import collections
import numpy as np

from McsPy import *
from pint import UndefinedUnitError

MCS_TICK = 1 * ureg.us
CLR_TICK = 100 * ureg.ns

# day -> number of clr ticks (100 ns)
DAY_TO_CLR_TIME_TICK = 24 * 60 * 60 * (10**7)

VERBOSE = True

def dprint_name_value(n, v):
    if VERBOSE:
        print(n, v)

[docs]class RawData(object): """ This class holds the information of a complete MCS raw data file """ def __init__(self, raw_data_path): """ Creates and initializes a RawData object that provides access to the content of the given MCS-HDF5 file :param raw_data_path: path to a HDF5 file that contains raw data encoded in a supported MCS-HDF5 format version """ self.raw_data_path = raw_data_path self.h5_file = h5py.File(raw_data_path, 'r') self.__validate_mcs_hdf5_version() self.__get_session_info() self.__recordings = None def __del__(self): self.h5_file.close() # Stub for with-Statement: #def __enter_(self): # return self # #def __exit__(self, type, value, traceback): # self.h5_file.close() def __str__(self): return super(RawData, self).__str__() def __validate_mcs_hdf5_version(self): "Check if the MCS-HDF5 protocol type and version of the file is supported by this class" root_grp = self.h5_file['/'] if 'McsHdf5ProtocolType' in root_grp.attrs: self.mcs_hdf5_protocol_type = root_grp.attrs['McsHdf5ProtocolType'].decode('UTF-8') if self.mcs_hdf5_protocol_type == "RawData": self.mcs_hdf5_protocol_type_version = root_grp.attrs['McsHdf5ProtocolVersion'] supported_versions = McsHdf5Protocols.SUPPORTED_PROTOCOLS[self.mcs_hdf5_protocol_type] if ((self.mcs_hdf5_protocol_type_version < supported_versions[0]) or (supported_versions[1] < self.mcs_hdf5_protocol_type_version)): raise IOError('Given HDF5 file has MCS-HDF5 RawData protocol version %s and supported are all versions from %s to %s' % (self.mcs_hdf5_protocol_type_version, supported_versions[0], supported_versions[1])) else: raise IOError("The root group of this HDF5 file has no 'McsHdf5ProtocolVersion' attribute -> so it could't be checked if the version is supported!") else: raise IOError("The root group of this HDF5 file has no 'McsHdf5ProtocolType attribute' -> this file is not supported by McsPy!") def __get_session_info(self): "Read all session metadata" data_attrs = self.h5_file['Data'].attrs.items() session_attributes = data_attrs session_info = {} for (name, value) in session_attributes: #print(name, value) if hasattr(value, "decode"): session_info[name] = value.decode('utf-8') else: session_info[name] = value self.comment = session_info['Comment'].rstrip() self.clr_date = session_info['Date'].rstrip() self.date_in_clr_ticks = session_info['DateInTicks'] # self.date = datetime.datetime.fromordinal(int(math.ceil(self.date_in_clr_ticks / day_to_clr_time_tick)) + 1) self.date = datetime.datetime(1, 1, 1) + datetime.timedelta(microseconds=int(self.date_in_clr_ticks)/10) # self.file_guid = session_info['FileGUID'].rstrip() self.file_guid = uuid.UUID(session_info['FileGUID'].rstrip()) self.mea_layout = session_info['MeaLayout'].rstrip() self.mea_sn = session_info['MeaSN'].rstrip() self.mea_name = session_info['MeaName'].rstrip() self.program_name = session_info['ProgramName'].rstrip() self.program_version = session_info['ProgramVersion'].rstrip() #return session_info def __read_recordings(self): "Read all recordings" data_folder = self.h5_file['Data'] if len(data_folder) > 0: self.__recordings = {} for (name, value) in data_folder.items(): dprint_name_value(name, value) recording_name = name.split('_') if (len(recording_name) == 2) and (recording_name[0] == 'Recording'): self.__recordings[int(recording_name[1])] = Recording(value) @property def recordings(self): "Access recordings" if self.__recordings is None: self.__read_recordings() return self.__recordings
[docs]class Recording(object): """ Container class for one recording """ def __init__(self, recording_grp): self.__recording_grp = recording_grp self.__get_recording_info() self.__analog_streams = None self.__frame_streams = None self.__event_streams = None self.__segment_streams = None self.__timestamp_streams = None def __get_recording_info(self): "Read metadata for this recording" recording_info = {} for (name, value) in self.__recording_grp.attrs.items(): if hasattr(value, "decode"): recording_info[name] = value.decode('UTF-8') else: recording_info[name] = value self.comment = recording_info['Comment'].rstrip() self.duration = recording_info['Duration'] self.label = recording_info['Label'].rstrip() self.recording_id = recording_info['RecordingID'] self.recording_type = recording_info['RecordingType'].rstrip() self.timestamp = recording_info['TimeStamp'] def __read_analog_streams(self): "Read all contained analog streams" if 'AnalogStream' in self.__recording_grp: analog_stream_folder = self.__recording_grp['AnalogStream'] if len(analog_stream_folder) > 0: self.__analog_streams = {} for (name, value) in analog_stream_folder.items(): dprint_name_value(name, value) stream_name = name.split('_') if (len(stream_name) == 2) and (stream_name[0] == 'Stream'): self.__analog_streams[int(stream_name[1])] = AnalogStream(value) def __read_frame_streams(self): "Read all contained frame streams" if 'FrameStream' in self.__recording_grp: frame_stream_folder = self.__recording_grp['FrameStream'] if len(frame_stream_folder) > 0: self.__frame_streams = {} for (name, value) in frame_stream_folder.items(): dprint_name_value(name, value) stream_name = name.split('_') if (len(stream_name) == 2) and (stream_name[0] == 'Stream'): self.__frame_streams[int(stream_name[1])] = FrameStream(value) def __read_event_streams(self): "Read all contained event streams" if 'EventStream' in self.__recording_grp: event_stream_folder = self.__recording_grp['EventStream'] if len(event_stream_folder) > 0: self.__event_streams = {} for (name, value) in event_stream_folder.items(): dprint_name_value(name, value) stream_name = name.split('_') if (len(stream_name) == 2) and (stream_name[0] == 'Stream'): index = int(stream_name[1]) self.__event_streams[index] = EventStream(value) def __read_segment_streams(self): "Read all contained segment streams" if 'SegmentStream' in self.__recording_grp: segment_stream_folder = self.__recording_grp['SegmentStream'] if len(segment_stream_folder) > 0: self.__segment_streams = {} for (name, value) in segment_stream_folder.items(): dprint_name_value(name, value) stream_name = name.split('_') if (len(stream_name) == 2) and (stream_name[0] == 'Stream'): self.__segment_streams[int(stream_name[1])] = SegmentStream(value) def __read_timestamp_streams(self): "Read all contained timestamp streams" if 'TimeStampStream' in self.__recording_grp: timestamp_stream_folder = self.__recording_grp['TimeStampStream'] if len(timestamp_stream_folder) > 0: self.__timestamp_streams = {} for (name, value) in timestamp_stream_folder.items(): dprint_name_value(name, value) stream_name = name.split('_') if (len(stream_name) == 2) and (stream_name[0] == 'Stream'): self.__timestamp_streams[int(stream_name[1])] = TimeStampStream(value) @property def analog_streams(self): "Access all analog streams - collection of :class:`~McsPy.McsData.AnalogStream` objects" if self.__analog_streams is None: self.__read_analog_streams() return self.__analog_streams @property def frame_streams(self): "Access all frame streams - collection of :class:`~McsPy.McsData.FrameStream` objects" if self.__frame_streams is None: self.__read_frame_streams() return self.__frame_streams @property def event_streams(self): "Access event streams - collection of :class:`~McsPy.McsData.EventStream` objects" if self.__event_streams is None: self.__read_event_streams() return self.__event_streams @property def segment_streams(self): "Access segment streams - - collection of :class:`~McsPy.McsData.SegementStream` objects" if self.__segment_streams is None: self.__read_segment_streams() return self.__segment_streams @property def timestamp_streams(self): "Access timestamp streams - collection of :class:`~McsPy.McsData.TimestampStream` objects" if self.__timestamp_streams is None: self.__read_timestamp_streams() return self.__timestamp_streams @property def duration_time(self): "Duration of the recording" dur_time = (self.duration - self.timestamp) * ureg.us return dur_time
[docs]class Stream(object): """ Base class for all stream types """ def __init__(self, stream_grp, info_type_name=None): """ Initializes a stream object with its associated HDF5 folder :param stream_grp: folder of the HDF5 file that contains the data of this stream :param info_type_name: name of the Info-Type as given in class McsHdf5Protocols (default None -> no version check is executed) """ self.stream_grp = stream_grp info_version = self.stream_grp.attrs["StreamInfoVersion"] if info_type_name != None: McsHdf5Protocols.check_protocol_type_version(info_type_name, info_version) self.__get_stream_info() def __get_stream_info(self): "Read all describing meta data common to each stream -> HDF5 folder attributes" stream_info = {} for (name, value) in self.stream_grp.attrs.items(): if hasattr(value, "decode"): stream_info[name] = value.decode('UTF-8') else: stream_info[name] = value self.info_version = stream_info['StreamInfoVersion'] self.data_subtype = stream_info['DataSubType'].rstrip() self.label = stream_info['Label'].rstrip() self.source_stream_guid = uuid.UUID(stream_info['SourceStreamGUID'].rstrip()) self.stream_guid = uuid.UUID(stream_info['StreamGUID'].rstrip()) self.stream_type = stream_info['StreamType'].rstrip() Stream_Types = ["analog", "event", "segment", "timestamp", "frame"]
[docs]class AnalogStream(Stream): """ Container class for one analog stream of several channels. Description for each channel is provided by a channel-associated object of :class:`~McsPy.McsData.ChannelInfo` """ def __init__(self, stream_grp): """ Initializes an analog stream object containing several analog channels :param stream_grp: folder of the HDF5 file that contains the data of this analog stream """ #McsHdf5Protocols.check_protocol_type_version("AnalogStreamInfoVersion", info_version) Stream.__init__(self, stream_grp, "AnalogStreamInfoVersion") self.__read_channels() def __read_channels(self): "Read all channels -> create Info structure and connect datasets" assert len(self.stream_grp) == 3 for (name, value) in self.stream_grp.items(): dprint_name_value(name, value) # Read timestamp index of channels: self.timestamp_index = self.stream_grp['ChannelDataTimeStamps'][...] # Read infos per channel ch_infos = self.stream_grp['InfoChannel'][...] ch_info_version = self.stream_grp['InfoChannel'].attrs['InfoVersion'] self.channel_infos = {} self.__map_row_to_channel_id = {} for channel_info in ch_infos: self.channel_infos[channel_info['ChannelID']] = ChannelInfo(ch_info_version, channel_info) self.__map_row_to_channel_id[channel_info['RowIndex']] = channel_info['ChannelID'] # Connect the data set self.channel_data = self.stream_grp['ChannelData']
[docs] def get_channel_in_range(self, channel_id, idx_start, idx_end): """ Get the signal of the given channel over the curse of time and in its measured range. :param channel_id: ID of the channel :param idx_start: index of the first sampled signal value that should be returned (0 <= idx_start < idx_end <= count samples) :param idx_end: index of the last sampled signal value that should be returned (0 <= idx_start < idx_end <= count samples) :return: Tuple (vector of the signal, unit of the values) """ if channel_id in self.channel_infos.keys(): if idx_start < 0: idx_start = 0 if idx_end > self.channel_data.shape[1]: idx_end = self.channel_data.shape[1] else: idx_end += 1 signal = self.channel_data[self.channel_infos[channel_id].row_index, idx_start : idx_end] scale = self.channel_infos[channel_id].adc_step.magnitude #scale = self.channel_infos[channel_id].get_field('ConversionFactor') * (10**self.channel_infos[channel_id].get_field('Exponent')) signal_corrected = (signal - self.channel_infos[channel_id].get_field('ADZero')) * scale return (signal_corrected, self.channel_infos[channel_id].adc_step.units)
[docs] def get_channel_sample_timestamps(self, channel_id, idx_start, idx_end): """ Get the timestamps of the sampled values. :param channel_id: ID of the channel :param idx_start: index of the first signal timestamp that should be returned (0 <= idx_start < idx_end <= count samples) :param idx_end: index of the last signal timestamp that should be returned (0 <= idx_start < idx_end <= count samples) :return: Tuple (vector of the timestamps, unit of the timestamps) """ if channel_id in self.channel_infos.keys(): start_ts = 0 channel = self.channel_infos[channel_id] tick = channel.get_field('Tick') for ts_range in self.timestamp_index: if idx_end < ts_range[1]: # nothing to do anymore -> break if ts_range[2] < idx_start: # start is behind the end of this range -> continue else: idx_segment = idx_start - ts_range[1] start_ts = ts_range[0] + idx_segment * tick # timestamp of first index if idx_end <= ts_range[2]: time_range = start_ts + np.arange(0, (idx_end - ts_range[1] + 1) - idx_segment, 1) * tick else: time_range = start_ts + np.arange(0, (ts_range[2] - ts_range[1] + 1) - idx_segment, 1) * tick idx_start = ts_range[2] + 1 if 'time' in locals(): time = np.append(time, time_range) else: time = time_range return (time, MCS_TICK.units)
[docs]class Info(object): """ Base class of all info classes Derived classes contain meta information for data structures and fields. """ def __init__(self, info_data): self.info = {} for name in info_data.dtype.names: if hasattr(info_data[name], "decode"): self.info[name] = info_data[name].decode('UTF-8') else: self.info[name] = info_data[name]
[docs] def get_field(self, name): "Get the field with that name -> access to the raw info array" return self.info[name]
@property def group_id(self): "Get the id of the group that the objects belongs to" return self.info["GroupID"] @property def label(self): "Label of this object" return self.info['Label'] @property def data_type(self): "Raw data type of this object" return self.info['RawDataType']
[docs]class InfoSampledData(Info): """ Base class of all info classes for evenly sampled data """ def __init__(self, info): """ Initialize an info object for sampled data :param info: array of info descriptors for this info object """ Info.__init__(self, info) @property def sampling_frequency(self): "Get the used sampling frequency in Hz" frequency = 1 / self.sampling_tick.to_base_units() return frequency.to(ureg.Hz) @property def sampling_tick(self): "Get the used sampling tick" tick_time = self.info['Tick'] * MCS_TICK return tick_time
[docs]class ChannelInfo(InfoSampledData): """ Contains all describing meta data for one sampled channel """ def __init__(self, info_version, info): """ Initialize an info object for sampled channel data :param info_version: number of the protocol version used by the following info structure :param info: array of info descriptors for this channel info object """ InfoSampledData.__init__(self, info) McsHdf5Protocols.check_protocol_type_version("InfoChannel", info_version) self.__version = info_version @property def channel_id(self): "Get the ID of the channel" return self.info['ChannelID'] @property def row_index(self): "Get the index of the row that contains the associated channel data inside the data matrix" return self.info['RowIndex'] @property def adc_step(self): "Size and unit of one ADC step for this channel" unit_name = self.info['Unit'] if unit_name == 'g': # Acceleration stream unit = ureg['standard_gravity'] elif unit_name == 'DegreePerSecond': # Gyroscope stream unit = ureg.degree / ureg.second else: unit = ureg[unit_name] # Should be tested that unit_name is a available in ureg (unit register) step = self.info['ConversionFactor'] * (10 ** self.info['Exponent'].astype(np.float64)) * unit return step @property def version(self): "Version number of the Type-Definition" return self.__version
[docs]class FrameStream(Stream): """ Container class for one frame stream with different entities """ def __init__(self, stream_grp): """ Initializes an frame stream object that contains all frame entities that belong to it. :param stream_grp: folder of the HDF5 file that contains the data of this frame stream """ Stream.__init__(self, stream_grp, "FrameStreamInfoVersion") self.__read_frame_entities() def __read_frame_entities(self): "Read all fream entities for this frame stream inside the associated frame entity folder" #assert len(self.stream_grp) == 3 for (name, value) in self.stream_grp.items(): dprint_name_value(name, value) # Read infos per frame fr_infos = self.stream_grp['InfoFrame'][...] fr_info_version = self.stream_grp['InfoFrame'].attrs['InfoVersion'] self.frame_entity = {} for frame_entity_info in fr_infos: frame_entity_group = "FrameDataEntity_" + str(frame_entity_info['FrameDataID']) conv_fact = self.__read_conversion_factor_matrix(frame_entity_group) frame_info = FrameEntityInfo(fr_info_version, frame_entity_info, conv_fact) self.frame_entity[frame_entity_info['FrameID']] = FrameEntity(self.stream_grp[frame_entity_group], frame_info) def __read_conversion_factor_matrix(self, frame_entity_group): "Read matrix of conversion factors inside the frame data entity folder" frame_entity_conv_matrix = frame_entity_group + "/ConversionFactors" conv_fact = self.stream_grp[frame_entity_conv_matrix][...] return conv_fact
[docs]class FrameEntity(object): """ Contains the stream of a specific frame entity. Meta-Information for this entity is available via an associated object of :class:`~McsPy.McsData.FrameEntityInfo` """ def __init__(self, frame_entity_group, frame_info): """ Initializes an frame entity object :param frame_entity_group: folder/group of the HDF5 file that contains the data for this frame entity :param frame_info: object of type FrameEntityInfo that contains the description of this frame entity """ self.info = frame_info self.group = frame_entity_group self.timestamp_index = self.group['FrameDataTimeStamps'][...] # Connect the data set self.data = self.group['FrameData']
[docs] def get_sensor_signal(self, sensor_x, sensor_y, idx_start, idx_end): """ Get the signal of a single sensor over the curse of time and in its measured range. :param sensor_x: x coordinate of the sensor :param sensor_y: y coordinate of the sensor :param idx_start: index of the first sampled frame that should be returned (0 <= idx_start < idx_end <= count frames) :param idx_end: index of the last sampled frame that should be returned (0 <= idx_start < idx_end <= count frames) :return: Tuple (vector of the signal, unit of the values) """ if sensor_x < 0 or self.data.shape[0] < sensor_x or sensor_y < 0 or self.data.shape[1] < sensor_y: raise IndexError if idx_start < 0: idx_start = 0 if idx_end > self.data.shape[2]: idx_end = self.data.shape[2] else: idx_end += 1 sensor_signal = self.data[sensor_x, sensor_y, idx_start : idx_end] scale_factor = self.info.adc_step_for_sensor(sensor_x, sensor_y) scale = scale_factor.magnitude sensor_signal_corrected = (sensor_signal - self.info.get_field('ADZero')) * scale return (sensor_signal_corrected, scale_factor.units)
[docs] def get_frame_timestamps(self, idx_start, idx_end): """ Get the timestamps of the sampled frames. :param idx_start: index of the first sampled frame that should be returned (0 <= idx_start < idx_end <= count frames) :param idx_end: index of the last sampled frame that should be returned (0 <= idx_start < idx_end <= count frames) :return: Tuple (vector of the timestamps, unit of the timestamps) """ if idx_start < 0 or self.data.shape[2] < idx_start or idx_end < idx_start or self.data.shape[2] < idx_end: raise IndexError start_ts = 0 tick = self.info.get_field('Tick') for ts_range in self.timestamp_index: if idx_end < ts_range[1]: # nothing to do anymore -> break if ts_range[2] < idx_start: # start is behind the end of this range -> continue else: idx_segment = idx_start - ts_range[1] start_ts = ts_range[0] + idx_segment * tick # timestamp of first index if idx_end <= ts_range[2]: time_range = start_ts + np.arange(0, (idx_end - ts_range[1] + 1) - idx_segment, 1) * tick else: time_range = start_ts + np.arange(0, (ts_range[2] - ts_range[1] + 1) - idx_segment, 1) * tick idx_start = ts_range[2] + 1 if 'time' in locals(): time = np.append(time, time_range) else: time = time_range return (time, MCS_TICK.units)
class Frame(object): """ Frame definition """ def __init__(self, left, top, right, bottom): self.__left = left self.__top = top self.__right = right self.__bottom = bottom @property def left(self): return self.__left @property def top(self): return self.__top @property def right(self): return self.__right @property def bottom(self): return self.__bottom @property def width(self): return self.__right - self.__left + 1 @property def height(self): return self.__bottom - self.__top + 1 class FrameEntityInfo(InfoSampledData): """ Contains all describing meta data for one frame entity """ def __init__(self, info_version, info, conv_factor_matrix): """ Initializes an describing info object that contains all descriptions of this frame entity. :param info_version: number of the protocol version used by the following info structure :param info: array of frame entity descriptors as represented by one row of the InfoFrame structure inside the HDF5 file :param conv_factor_matrix: matrix of conversion factor as represented by the ConversionFactors structure inside one FrameDataEntity folder of the HDF5 file """ InfoSampledData.__init__(self, info) McsHdf5Protocols.check_protocol_type_version("FrameEntityInfo", info_version) self.__version = info_version self.frame = Frame(info['FrameLeft'], info['FrameTop'], info['FrameRight'], info['FrameBottom']) self.reference_frame = Frame(info['ReferenceFrameLeft'], info['ReferenceFrameTop'], info['ReferenceFrameRight'], info['ReferenceFrameBottom']) self.conversion_factors = conv_factor_matrix @property def frame_id(self): "ID of the frame" return self.info['FrameID'] @property def sensor_spacing(self): "Returns the spacing of the sensors in micro-meter" return self.info['SensorSpacing'] @property def adc_basic_step(self): "Returns the value of one basic ADC-Step" unit_name = self.info['Unit'] # Should be tested that unit_name is a available in ureg (unit register) basic_step = (10 ** self.info['Exponent'].astype(np.float64)) * ureg[unit_name] return basic_step def adc_step_for_sensor(self, x, y): "Returns the combined (virtual) ADC-Step for the sensor (x,y)" adc_sensor_step = self.conversion_factors[x, y] * self.adc_basic_step return adc_sensor_step @property def version(self): "Version number of the Type-Definition" return self.__version
[docs]class EventStream(Stream): """ Container class for one event stream with different entities """ def __init__(self, stream_grp): """ Initializes an event stream object that contains all entities that belong to it. :param stream_grp: folder of the HDF5 file that contains the data of this event stream """ Stream.__init__(self, stream_grp, "EventStreamInfoVersion") self.__read_event_entities() def __read_event_entities(self): "Create all event entities of this event stream" for (name, value) in self.stream_grp.items(): dprint_name_value(name, value) # Read infos per event entity event_infos = self.stream_grp['InfoEvent'][...] event_entity_info_version = self.stream_grp['InfoEvent'].attrs['InfoVersion'] self.event_entity = {} for event_entity_info in event_infos: event_entity_name = "EventEntity_" + str(event_entity_info['EventID']) event_info = EventEntityInfo(event_entity_info_version, event_entity_info) if event_entity_name in self.stream_grp: self.event_entity[event_entity_info['EventID']] = EventEntity(self.stream_grp[event_entity_name], event_info)
[docs]class EventEntity(object): """ Contains the event data of a specific entity. Meta-Information for this entity is available via an associated object of :class:`~McsPy.McsData.EventEntityInfo` """ def __init__(self, event_data, event_info): """ Initializes an event entity object :param event_data: dataset of the HDF5 file that contains the data for this event entity :param event_info: object of type EventEntityInfo that contains the description of this entity """ self.info = event_info # Connect the data set self.data = event_data @property def count(self): """Number of contained events""" dim = self.data.shape return dim[1] def __handle_indices(self, idx_start, idx_end): """Check indices for consistency and set default values if nothing was provided""" if idx_start == None: idx_start = 0 if idx_end == None: idx_end = self.count if idx_start < 0 or self.data.shape[1] < idx_start or idx_end < idx_start or self.data.shape[1] < idx_end: raise IndexError return (idx_start, idx_end)
[docs] def get_events(self, idx_start=None, idx_end=None): """Get all n events of this entity of the given index range (idx_start <= idx < idx_end) :param idx_start: start index of the range (including), if nothing is given -> 0 :param idx_end: end index of the range (excluding, if nothing is given -> last index :return: Tuple of (2 x n matrix of timestamp (1. row) and duration (2. row), Used unit of time) """ idx_start, idx_end = self.__handle_indices(idx_start, idx_end) events = self.data[..., idx_start:idx_end] return (events * MCS_TICK.magnitude, MCS_TICK.units)
[docs] def get_event_timestamps(self, idx_start=None, idx_end=None): """Get all n event timestamps of this entity of the given index range :param idx_start: start index of the range, if nothing is given -> 0 :param idx_end: end index of the range, if nothing is given -> last index :return: Tuple of (n-length array of timestamps, Used unit of time) """ idx_start, idx_end = self.__handle_indices(idx_start, idx_end) events = self.data[0, idx_start:idx_end] return (events * MCS_TICK.magnitude, MCS_TICK.units)
[docs] def get_event_durations(self, idx_start=None, idx_end=None): """Get all n event durations of this entity of the given index range :param idx_start: start index of the range, if nothing is given -> 0 :param idx_end: end index of the range, if nothing is given -> last index :return: Tuple of (n-length array of duration, Used unit of time) """ idx_start, idx_end = self.__handle_indices(idx_start, idx_end) events = self.data[1, idx_start:idx_end] return (events * MCS_TICK.magnitude, MCS_TICK.units)
[docs]class EventEntityInfo(Info): """ Contains all meta data for one event entity """ def __init__(self, info_version, info): """ Initializes an describing info object with an array that contains all descriptions of this event entity. :param info_version: number of the protocol version used by the following info structure :param info: array of event entity descriptors as represented by one row of the InfoEvent structure inside the HDF5 file """ Info.__init__(self, info) McsHdf5Protocols.check_protocol_type_version("EventEntityInfo", info_version) self.__version = info_version if info["SourceChannelIDs"] == "": source_channel_ids = [-1] source_channel_labels = ["N/A"] else: source_channel_ids = [int(x) for x in info['SourceChannelIDs'].decode('utf-8').split(',')] source_channel_labels = [x.strip() for x in info['SourceChannelLabels'].decode('utf-8').split(',')] self.__source_channels = {} for idx, channel_id in enumerate(source_channel_ids): self.__source_channels[channel_id] = source_channel_labels[idx] @property def id(self): "Event ID" return self.info['EventID'] @property def raw_data_bytes(self): "Lenght of raw data in bytes" return self.info['RawDataBytes'] @property def source_channel_ids(self): "ID's of all channels that were involved in the event generation." return list(self.__source_channels.keys()) @property def source_channel_labels(self): "Labels of the channels that were involved in the event generation." return self.__source_channels @property def version(self): "Version number of the Type-Definition" return self.__version
[docs]class SegmentStream(Stream): """ Container class for one segment stream of different segment entities """ def __init__(self, stream_grp): Stream.__init__(self, stream_grp, "SegmentStreamInfoVersion") self.__read_segment_entities() def __read_segment_entities(self): "Read and initialize all segment entities" for (name, value) in self.stream_grp.items(): dprint_name_value(name, value) # Read infos per segment entity segment_infos = self.stream_grp['InfoSegment'][...] segment_info_version = self.stream_grp['InfoSegment'].attrs['InfoVersion'] self.segment_entity = {} for segment_entity_info in segment_infos: ch_info_version = self.stream_grp['SourceInfoChannel'].attrs['InfoVersion'] source_channel_infos = self.__get_source_channel_infos(ch_info_version, self.stream_grp['SourceInfoChannel'][...]) segment_info = SegmentEntityInfo(segment_info_version, segment_entity_info, source_channel_infos) if self.data_subtype == "Average": segment_entity_data_name = "AverageData_" + str(segment_entity_info['SegmentID']) segment_entity_average_annotation_name = "AverageData_Range_" + str(segment_entity_info['SegmentID']) if segment_entity_data_name in self.stream_grp: self.segment_entity[segment_entity_info['SegmentID']] = AverageSegmentEntity(self.stream_grp[segment_entity_data_name], self.stream_grp[segment_entity_average_annotation_name], segment_info) else: segment_entity_data_name = "SegmentData_" + str(segment_entity_info['SegmentID']) segment_entity_ts_name = "SegmentData_ts_" + str(segment_entity_info['SegmentID']) if segment_entity_data_name in self.stream_grp: self.segment_entity[segment_entity_info['SegmentID']] = SegmentEntity(self.stream_grp[segment_entity_data_name], self.stream_grp[segment_entity_ts_name], segment_info) def __get_source_channel_infos(self, ch_info_version, source_channel_infos): "Create a dictionary of all present source channels" source_channels = {} for source_channel_info in source_channel_infos: source_channels[source_channel_info['ChannelID']] = ChannelInfo(ch_info_version, source_channel_info) return source_channels
[docs]class SegmentEntity(object): """ Segment entity class, Meta-Information for this entity is available via an associated object of :class:`~McsPy.McsData.SegmentEntityInfo` """ def __init__(self, segment_data, segment_ts, segment_info): """ Initializes a segment entity. :param segment_data: 2d-matrix (one segment) or 3d-cube (n segments) of segment data :param segment_ts: timestamp vector for every segment (2d) or multi-segments (3d) :param segment_info: segment info object that contains all meta data for this segment entity :return: Segment entity """ self.info = segment_info # connect the data set self.data = segment_data # connect the timestamp vector self.data_ts = segment_ts assert self.segment_sample_count == self.data_ts.shape[1], 'Timestamp index is not compatible with dataset!!!' @property def segment_sample_count(self): "Number of contained samples of segments (2d) or multi-segments (3d)" dim = self.data.shape if len(dim) == 3: return dim[2] else: return dim[1] @property def segment_count(self): "Number of segments that are sampled for one time point (2d) -> 1 and (3d) -> n" dim = self.data.shape if len(dim) == 3: return dim[1] else: return 1 def __handle_indices(self, idx_start, idx_end): """Check indices for consistency and set default values if nothing was provided""" sample_count = self.segment_sample_count if idx_start == None: idx_start = 0 if idx_end == None: idx_end = sample_count if idx_start < 0 or sample_count < idx_start or idx_end < idx_start or sample_count < idx_end: raise IndexError return (idx_start, idx_end)
[docs] def get_segment_in_range(self, segment_id, flat=False, idx_start=None, idx_end=None): """ Get the a/the segment signals in its measured range. :param segment_id: id resp. number of the segment (0 if only one segment is present or the index inside the multi-segment collection) :param flat: true -> one-dimensional vector of the sequentially ordered segments, false -> k x n matrix of the n segments of k sample points :param idx_start: index of the first segment that should be returned (0 <= idx_start < idx_end <= count segments) :param idx_end: index of the last segment that should be returned (0 <= idx_start < idx_end <= count segments) :return: Tuple (of a flat vector of the sequentially ordered segments or a k x n matrix of the n segments of k sample points depending on the value of *flat* , and the unit of the values) """ if segment_id in self.info.source_channel_of_segment.keys(): idx_start, idx_end = self.__handle_indices(idx_start, idx_end) if self.segment_count == 1: signal = self.data[..., idx_start : idx_end] else: signal = self.data[..., segment_id, idx_start : idx_end] source_channel = self.info.source_channel_of_segment[segment_id] scale = source_channel.adc_step.magnitude signal_corrected = (signal - source_channel.get_field('ADZero')) * scale if flat: signal_corrected = np.reshape(signal_corrected, -1, 'F') return (signal_corrected, source_channel.adc_step.units)
[docs] def get_segment_sample_timestamps(self, segment_id, flat=False, idx_start=None, idx_end=None): """ Get the timestamps of the sample points of the measured segment. :param segment_id: id resp. number of the segment (0 if only one segment is present or the index inside the multi-segment collection) :param flat: true -> one-dimensional vector of the sequentially ordered segment timestamps, false -> k x n matrix of the k timestamps of n segments :param idx_start: index of the first segment for that timestamps should be returned (0 <= idx_start < idx_end <= count segments) :param idx_end: index of the last segment for that timestamps should be returned (0 <= idx_start < idx_end <= count segments) :return: Tuple (of a flat vector of the sequentially ordered segments or a k x n matrix of the n segments of k sample points depending on the value of *flat* , and the unit of the values) """ if segment_id in self.info.source_channel_of_segment.keys(): idx_start, idx_end = self.__handle_indices(idx_start, idx_end) data_ts = self.data_ts[idx_start:idx_end] source_channel = self.info.source_channel_of_segment[segment_id] signal_ts = np.zeros((self.data.shape[0], data_ts.shape[1]), dtype=np.long) segment_ts = np.zeros(self.data.shape[0], dtype=np.long) + source_channel.sampling_tick.magnitude segment_ts[0] = 0 segment_ts = np.cumsum(segment_ts) for i in range(data_ts.shape[1]): col = (data_ts[0, i] - self.info.pre_interval.magnitude) + segment_ts signal_ts[:, i] = col if flat: signal_ts = np.reshape(signal_ts, -1, 'F') return (signal_ts, source_channel.sampling_tick.units)
AverageSegmentTuple = collections.namedtuple('AverageSegmentTuple', ['mean', 'std_dev', 'time_tick_unit', 'signal_unit']) """ Named tuple that describe one or more average segments (mean, std_dev, time_tick_unit, signal_unit). .. note:: * :class:`~AverageSegmentTuple.mean` - mean signal values * :class:`~AverageSegmentTuple.std_dev` - standard deviation of the signal value (it is 0 if there was only one sample segment) * :class:`~AverageSegmentTuple.time_tick_unit` - sampling interval with time unit * :class:`~AverageSegmentTuple.signal_unit` - measured unit of the signal """
[docs]class AverageSegmentEntity(object): """ Contains a number of signal segments that are calcualted as averages of number of segments occured in a given time range. Meta-Information for this entity is available via an associated object of :class:`~McsPy.McsData.SegmentEntityInfo` """ def __init__(self, segment_average_data, segment_average_annotation, segment_info): """ Initializes an average segment entity :param segment_avarage_data: 2d-matrix (one average) or 3d-cube (n averages) of average segments :param segment_annotation: annotation vector for every average segment :param segment_info: segment info object that contains all meta data for this segment entity :return: Average segment entity """ self.info = segment_info # connect the data set self.data = segment_average_data # connect the timestamp vector self.data_annotation = segment_average_annotation assert self.number_of_averages == self.data_annotation.shape[1], 'Timestamp index is not compatible with dataset!!!' @property def number_of_averages(self): "Number of average segments inside this average entity" dim = self.data.shape return dim[2] @property def sample_length(self): "Number of sample points of an average segment" dim = self.data.shape return dim[1]
[docs] def time_ranges(self): """ List of time range tuples for all contained average segments :return: List of tuple with start and end time point """ time_range_list = [] for idx in range(self.data_annotation.shape[-1]): time_range_list.append((self.data_annotation[0, idx] * MCS_TICK, self.data_annotation[1, idx] * MCS_TICK)) return time_range_list
[docs] def time_range(self, average_segment_idx): """ Get the time range for that the average segment was calculated :param average_segment_idx: index resp. number of the average segment :return: Tuple with start and end time point """ return (self.data_annotation[0, average_segment_idx] * MCS_TICK, self.data_annotation[1, average_segment_idx] * MCS_TICK)
[docs] def average_counts(self): """ List of counts of samples for all contained average segments :param average_segment_idx: id resp. number of the average segment :return: sample count """ sample_count_list = [] for idx in range(self.data_annotation.shape[-1]): sample_count_list.append(self.data_annotation[2, idx]) return sample_count_list
[docs] def average_count(self, average_segment_idx): """ Count of samples that were used to calculate the average :param average_segment_idx: id resp. number of the average segment :return: sample count """ return self.data_annotation[2, average_segment_idx]
def __calculate_scaled_average(self, mean_data, std_dev_data): """ Shift and scale average segments appropriate """ assert len(self.info.source_channel_of_segment) == 1, "There should be only one source channel for one average segment entity!" source_channel = self.info.source_channel_of_segment[0] # take the first and only source channel scale = source_channel.adc_step.magnitude mean_shifted_and_scaled = (mean_data - source_channel.get_field('ADZero')) * scale std_dev_scaled = std_dev_data * scale data_tuple = AverageSegmentTuple(mean=mean_shifted_and_scaled, std_dev=std_dev_scaled, time_tick_unit=source_channel.sampling_tick, signal_unit=source_channel.adc_step.units) return data_tuple
[docs] def get_scaled_average_segments(self): """ Get all contained average segments in its measured physical range. :return: :class:`~McsPy.McsData.AverageSegmentTuple` containing the k x n matrices for mean and standard deviation of all contained average segments n with the associated sampling and measuring information """ mean = self.data[0, ...] std_dev = self.data[1, ...] return self.__calculate_scaled_average(mean, std_dev)
[docs] def get_scaled_average_segment(self, average_segment_idx): """ Get the selected average segment in its measured physical range. :param segment_idx: index resp. number of the average segment :return: :class:`~McsPy.McsData.AverageSegmentTuple` containing the mean and standard deviation vector of the average segment with the associated sampling and measuring information """ mean = self.data[0, ..., average_segment_idx] std_dev = self.data[1, ..., average_segment_idx] return self.__calculate_scaled_average(mean, std_dev)
def __calculate_shifted_average(self, mean_data, std_dev_data): """ Shift average segments appropriate """ assert len(self.info.source_channel_of_segment) == 1, "There should be only one source channel for one average segment entity!" source_channel = self.info.source_channel_of_segment[0] # take the first and only source channel mean = mean_data mean_shifted = mean - source_channel.get_field('ADZero') data_tuple = AverageSegmentTuple(mean=mean_shifted, std_dev=std_dev_data, time_tick_unit=source_channel.sampling_tick, signal_unit=source_channel.adc_step) return data_tuple
[docs] def get_average_segments(self): """ Get all contained average segments AD-offset in ADC values with its measuring conditions :return: :class:`~McsPy.McsData.AverageSegmentTuple` containing the mean and standard deviation vector of the average segment in ADC steps with sampling tick and ADC-Step definition """ mean = self.data[0, ...] std_dev = self.data[1, ...] return self.__calculate_shifted_average(mean, std_dev)
[docs] def get_average_segment(self, average_segment_idx): """ Get the AD-offset corrected average segment in ADC values with its measuring conditions :param segment_id: id resp. number of the segment :return: :class:`~McsPy.McsData.AverageSegmentTuple` containing the k x n matrices for mean and standard deviation of all contained average segments in ADC steps with sampling tick and ADC-Step definition """ mean = self.data[0, ..., average_segment_idx] std_dev = self.data[1, ..., average_segment_idx] return self.__calculate_shifted_average(mean, std_dev)
[docs]class SegmentEntityInfo(Info): """ Contains all meta data for one segment entity """ def __init__(self, info_version, info, source_channel_infos): """ Initializes an describing info object with an array that contains all descriptions of this segment entity. :param info_version: number of the protocol version used by the following info structure :param info: array of segment entity descriptors as represented by one row of the SegmentEvent structure inside the HDF5 file :param source_channel_infos: dictionary of source channels from where the segments were taken """ Info.__init__(self, info) McsHdf5Protocols.check_protocol_type_version("SegmentEntityInfo", info_version) self.__version = info_version source_channel_ids = [int(x) for x in info['SourceChannelIDs'].decode('utf-8').split(',')] self.source_channel_of_segment = {} for idx, channel_id in enumerate(source_channel_ids): self.source_channel_of_segment[idx] = source_channel_infos[channel_id] @property def id(self): "Segment ID" return self.info['SegmentID'] @property def pre_interval(self): "Interval [start of the segment <- defining event timestamp]" return self.info['PreInterval'] * MCS_TICK @property def post_interval(self): "Interval [defining event timestamp -> end of the segment]" return self.info['PostInterval'] * MCS_TICK @property def type(self): "Type of the segment like 'Average' or 'Cutout'" return self.info['SegmentType'] @property def count(self): "Count of segments inside the segment entity" return len(self.source_channel_of_segment) @property def version(self): "Version number of the Type-Definition" return self.__version
[docs]class TimeStampStream(Stream): """ Container class for one timestamp stream with different entities """ def __init__(self, stream_grp): """ Initializes an timestamp stream object that contains all entities that belong to it. :param stream_grp: folder of the HDF5 file that contains the data of this timestamp stream """ Stream.__init__(self, stream_grp, "TimeStampStreamInfoVersion") self.__read_timestamp_entities() def __read_timestamp_entities(self): "Create all timestamp entities of this timestamp stream" for (name, value) in self.stream_grp.items(): dprint_name_value(name, value) # Read infos per timestamp entity timestamp_infos = self.stream_grp['InfoTimeStamp'][...] timestamp_info_version = self.stream_grp['InfoTimeStamp'].attrs['InfoVersion'] self.timestamp_entity = {} for timestamp_entity_info in timestamp_infos: timestamp_entity_name = "TimeStampEntity_" + str(timestamp_entity_info['TimeStampEntityID']) timestamp_info = TimeStampEntityInfo(timestamp_info_version, timestamp_entity_info) if timestamp_entity_name in self.stream_grp: self.timestamp_entity[timestamp_entity_info['TimeStampEntityID']] = TimeStampEntity(self.stream_grp[timestamp_entity_name], timestamp_info)
[docs]class TimeStampEntity(object): """ Time-Stamp entity class, Meta-Information for this entity is available via an associated object of :class:`~McsPy.McsData.TimestampEntityInfo` """ def __init__(self, timestamp_data, timestamp_info): """ Initializes an timestamp entity object :param timestamp_data: dataset of the HDF5 file that contains the data for this timestamp entity :param timestamp_info: object of type TimeStampEntityInfo that contains the description of this entity """ self.info = timestamp_info # Connect the data set self.data = timestamp_data @property def count(self): """Number of contained timestamps""" dim = self.data.shape return dim[1] def __handle_indices(self, idx_start, idx_end): """Check indices for consistency and set default values if nothing was provided""" if idx_start == None: idx_start = 0 if idx_end == None: idx_end = self.count if idx_start < 0 or self.data.shape[1] < idx_start or idx_end < idx_start or self.data.shape[1] < idx_end: raise IndexError return (idx_start, idx_end)
[docs] def get_timestamps(self, idx_start=None, idx_end=None): """Get all n time stamps of this entity of the given index range (idx_start <= idx < idx_end) :param idx_start: start index of the range (including), if nothing is given -> 0 :param idx_end: end index of the range (excluding, if nothing is given -> last index :return: Tuple of (n-length array of timestamps, Used unit of time) """ idx_start, idx_end = self.__handle_indices(idx_start, idx_end) timestamps = self.data[idx_start:idx_end] scale = self.info.measuring_unit return (timestamps, scale)
[docs]class TimeStampEntityInfo(Info): """ Contains all meta data for one timestamp entity """ def __init__(self, info_version, info): """ Initializes an describing info object with an array that contains all descriptions of this timestamp entity. :param info_version: number of the protocol version used by the following info structure :param info: array of event entity descriptors as represented by one row of the InfoTimeStamp structure inside the HDF5 file """ Info.__init__(self, info) McsHdf5Protocols.check_protocol_type_version("TimeStampEntityInfo", info_version) self.__version = info_version source_channel_ids = [int(x) for x in info['SourceChannelIDs'].decode('utf-8').split(',')] source_channel_labels = [x.strip() for x in info['SourceChannelLabels'].decode('utf-8').split(',')] self.__source_channels = {} for idx, channel_id in enumerate(source_channel_ids): self.__source_channels[channel_id] = source_channel_labels[idx] @property def id(self): "Timestamp entity ID" return self.info['TimeStampEntityID'] @property def unit(self): "Unit in which the timestamps are measured" return self.info['Unit'] @property def exponent(self): "Exponent for the unit in which the timestamps are measured" return int(self.info['Exponent']) @property def measuring_unit(self): "Unit in which the timestamp entity was measured" try: provided_base_unit = ureg.parse_expression(self.unit) except UndefinedUnitError as unit_undefined: print ("Could not find unit \'%s\' in the Unit-Registry" % self.unit) #unit_undefined.unit_names return None else: return (10**self.exponent) * provided_base_unit @property def data_type(self): "DataType for the timestamps" return 'Long' @property def source_channel_ids(self): "ID's of all channels that were involved in the timestamp generation." return list(self.__source_channels.keys()) @property def source_channel_labels(self): "Labels of the channels that were involved in the timestamp generation." return self.__source_channels @property def version(self): "Version number of the Type-Definition" return self.__version