Module thunderfish.eodexplorer
View and explore properties of EOD waveforms.
Functions
def load_waveform(idx)-
Expand source code
def load_waveform(idx): eodf = data[idx,'EODf'] if not np.isfinite(eodf): if load_spec: return None, None else: return None file_name = data[idx,'file'] if 'file' in data else '-'.join(basename.split('-')[:-1]) file_index = data[idx,'index'] if 'index' in data else 0 eod_filename = os.path.join(data_path, '%s-eodwaveform-%d.csv' % (file_name, file_index)) eod_table = TableData(eod_filename) eod = eod_table[:,'mean'] norm = np.max(eod) if wave_fish: eod = np.column_stack((eod_table[:,'time']*0.001*eodf, eod/norm)) else: eod = np.column_stack((eod_table[:,'time'], eod/norm)) if not load_spec: return eod fish_type = 'wave' if wave_fish else 'pulse' spec_table = TableData(os.path.join(data_path, '%s-%sspectrum-%d.csv' % (file_name, fish_type, file_index))) spec_data = spec_table.array() if not wave_fish: spec_data = spec_data[spec_data[:,0]<2000.0,:] spec_data = spec_data[::5,:] return (eod, spec_data) def main(cargs=None)-
Expand source code
def main(cargs=None): global data global wave_fish global load_spec global data_path global basename # command line arguments: if cargs is None: cargs = sys.argv[1:] parser = argparse.ArgumentParser(add_help=False, description='View and explore properties of EOD waveforms.', epilog='version %s by Benda-Lab (2019-%s)' % (__version__, __year__)) parser.add_argument('-h', '--help', nargs=0, action=PrintHelp, help='show this help message and exit') parser.add_argument('--version', action='version', version=__version__) parser.add_argument('-v', dest='verbose', action='store_true', default=False, help='verbose output') parser.add_argument('-l', dest='list_columns', action='store_true', help='list all available data columns and exit') parser.add_argument('-j', dest='jobs', nargs='?', type=int, default=None, const=0, help='number of jobs run in parallel for loading waveform data. Without argument use all CPU cores.') parser.add_argument('-D', dest='column_groups', default=[], type=str, action='append', choices=EODExplorer.groups, help='default selection of data columns, check them with the -l option') parser.add_argument('-d', dest='add_data_cols', action='append', default=[], metavar='COLUMN', help='data columns to be appended or removed (if already listed) for analysis') parser.add_argument('-n', dest='max_n', default=0, type=int, metavar='MAX', help='maximum number of harmonics or peaks to be used') parser.add_argument('-w', dest='add_waveforms', default=[], type=str, action='append', choices=['first', 'second', 'ampl', 'power', 'phase'], help='add first or second derivative of EOD waveform, or relative amplitude, power, or phase to the plot of selected EODs.') parser.add_argument('-s', dest='save_pca', action='store_true', help='save PCA components and exit') parser.add_argument('-c', dest='color_col', default='EODf', type=str, metavar='COLUMN', help='data column to be used for color code or "row"') parser.add_argument('-m', dest='color_map', default='jet', type=str, metavar='CMAP', help='name of color map') parser.add_argument('-p', dest='data_path', default='.', type=str, metavar='PATH', help='path to the analyzed EOD waveform data') parser.add_argument('-P', dest='rawdata_path', default='.', type=str, metavar='PATH', help='path to the raw EOD recordings') parser.add_argument('-f', dest='format', default='auto', type=str, choices=TableData.formats + ['same'], help='file format used for saving PCA data ("same" uses same format as input file)') parser.add_argument('file', default='', type=str, help='a wavefish.* or pulsefish.* summary file as generated by collectfish') args = parser.parse_args(cargs) # read in command line arguments: verbose = args.verbose list_columns = args.list_columns jobs = args.jobs file_name = args.file column_groups = args.column_groups add_data_cols = args.add_data_cols max_n = args.max_n add_waveforms = args.add_waveforms save_pca = args.save_pca color_col = args.color_col color_map = args.color_map data_path = args.data_path rawdata_path = args.rawdata_path data_format = args.format # read configuration: cfgfile = __package__ + '.cfg' cfg = ConfigFile() add_eod_quality_config(cfg) add_harmonic_groups_config(cfg) add_species_config(cfg) add_write_table_config(cfg, table_format='csv', unit_style='row', align_columns=True, shrink_width=False) cfg.load_files(cfgfile, file_name, 4) # output format: if data_format == 'same': ext = os.path.splitext(file_name)[1][1:] if ext in TableData.ext_formats: data_format = TableData.ext_formats[ext] else: data_format = 'dat' if data_format != 'auto': cfg.set('fileFormat', data_format) # check color map: if not color_map in plt.colormaps(): parser.error('"%s" is not a valid color map' % color_map) # load summary data: wave_fish = 'wave' in file_name data = TableData(file_name) # basename: basename = os.path.splitext(os.path.basename(file_name))[0] # check quality: skipped = 0 for r in reversed(range(data.rows())): idx = 0 if 'index' in data: idx = data[r,'index'] skips = '' if wave_fish: harm_rampl = np.array([0.01*data[r,'relampl%d'%(k+1)] for k in range(3) if 'relampl%d'%(k+1) in data]) props = data.row_dict(r) if 'clipped' in props: props['clipped'] *= 0.01 if 'noise' in props: props['noise'] *= 0.01 if 'rmserror' in props: props['rmserror'] *= 0.01 if 'thd' in props: props['thd'] *= 0.01 _, skips, msg = wave_quality(props, harm_rampl, **wave_quality_args(cfg)) else: props = data.row_dict(r) if 'clipped' in props: props['clipped'] *= 0.01 if 'noise' in props: props['noise'] *= 0.01 skips, msg, _ = pulse_quality(props, **pulse_quality_args(cfg)) if len(skips) > 0: if verbose: print('skip fish %2d from %s: %s' % (idx, data[r,'file'] if 'file' in data else basename, skips)) del data[r,:] skipped += 1 if verbose and skipped > 0: print('') # select columns (EOD properties) to be shown: data_cols, error = \ EODExplorer.select_EOD_properties(data, wave_fish, max_n, column_groups, add_data_cols) if error: parser.error(error) # select column used for coloring the data: colors, color_label, color_idx, error = \ select_coloring(data, data_cols, color_col) if error: parser.error(error) # list columns: if list_columns: list_available_features(data, data_cols, color_idx) parser.exit() # load waveforms: load_spec = 'ampl' in add_waveforms or 'power' in add_waveforms or 'phase' in add_waveforms if jobs is not None: cpus = cpu_count() if jobs == 0 else jobs p = Pool(cpus) eod_data = p.map(load_waveform, range(data.rows())) del p else: eod_data = list(map(load_waveform, range(data.rows()))) # explore: eod_expl = EODExplorer(data, data_cols, wave_fish, eod_data, add_waveforms, load_spec, rawdata_path) # write pca: if save_pca: eod_expl.compute_pca(False) eod_expl.save_pca(basename, False, **write_table_args(cfg)) eod_expl.compute_pca(True) eod_expl.save_pca(basename, True, **write_table_args(cfg)) else: eod_expl.set_colors(colors, color_label, color_map) eod_expl.show()
Classes
class EODExplorer (data, data_cols, wave_fish, eod_data, add_waveforms, loaded_spec, rawdata_path)-
Expand source code
class EODExplorer(MultivariateExplorer): """Simple GUI for viewing and exploring properties of EOD waveforms. EODExplorer adapts a MultivariateExplorer to specific needs of EODs. Static members -------------- - `groups`: names of groups of data columns that can be selected. - `select_EOD_properties()`: select data columns to be explored. """ def __init__(self, data, data_cols, wave_fish, eod_data, add_waveforms, loaded_spec, rawdata_path): """ Parameters ---------- data: TableData Full table of EOD properties. Each row is a fish. data_cols: list of string or ints Names or indices of columns in `data` to be explored. You may use the static function `select_EOD_properties()` for assisting the selection of columns. wave_fish: boolean True if data are about wave-type weakly electric fish. False if data are about pulse-type weakly electric fish. eod_data: list of waveform data Either waveform data is only the EOD waveform, a ndarray of shape (time, ['time', 'voltage']), or it is a list with the first element being the EOD waveform, and the second element being a 2D ndarray of spectral properties of the EOD waveform with first column being the frequency or harmonics. add_waveforms: list of string List of what should be shown as waveform. Elements can be 'first', 'second', 'ampl', 'power', or 'phase'. For 'first' and 'second' the first and second derivatives of the supplied EOD waveform a computed and shown. 'ampl', 'power', and 'phase' select properties of the provided spectral properties. loaded_spec: boolean Indicates whether eod_data contains spectral properties. rawdata_path: string Base path to the raw recording, needed to show thunderfish when double clicking on a single EOD. """ self.wave_fish = wave_fish self.eoddata = data self.path = rawdata_path MultivariateExplorer.__init__(self, data[:,data_cols], None, 'EODExplorer') tunit = 'ms' dunit = '1/ms' if wave_fish: tunit = '1/EODf' dunit = 'EODf' wave_data = eod_data xlabels = ['Time [%s]' % tunit] ylabels = ['Voltage'] if 'first' in add_waveforms: # first derivative: if loaded_spec: if hasattr(sig, 'savgol_filter'): derivative = lambda x: (np.column_stack((x[0], \ sig.savgol_filter(x[0][:,1], 5, 2, 1, x[0][1,0]-x[0][0,0]))), x[1]) else: derivative = lambda x: (np.column_stack((x[0][:-1,:], \ np.diff(x[0][:,1])/(x[0][1,0]-x[0][0,0]))), x[1]) else: if hasattr(sig, 'savgol_filter'): derivative = lambda x: np.column_stack((x, \ sig.savgol_filter(x[:,1], 5, 2, 1, x[1,0]-x[0,0]))) else: derivative = lambda x: np.column_stack((x[:-1,:], \ np.diff(x[:,1])/(x[1,0]-x[0,0]))) wave_data = list(map(derivative, wave_data)) ylabels.append('dV/dt [%s]' % dunit) if 'second' in add_waveforms: # second derivative: if loaded_spec: if hasattr(sig, 'savgol_filter'): derivative = lambda x: (np.column_stack((x[0], \ sig.savgol_filter(x[0][:,1], 5, 2, 2, x[0][1,0]-x[0][0,0]))), x[1]) else: derivative = lambda x: (np.column_stack((x[0][:-1,:], \ np.diff(x[0][:,2])/(x[0][1,0]-x[0][0,0]))), x[1]) else: if hasattr(sig, 'savgol_filter'): derivative = lambda x: np.column_stack((x, \ sig.savgol_filter(x[:,1], 5, 2, 2, x[1,0]-x[0,0]))) else: derivative = lambda x: np.column_stack((x[:-1,:], \ np.diff(x[:,2])/(x[1,0]-x[0,0]))) wave_data = list(map(derivative, wave_data)) ylabels.append('d^2V/dt^2 [%s^2]' % dunit) if loaded_spec: if wave_fish: indices = [0] phase = False xlabels.append('Harmonics') if 'ampl' in add_waveforms: indices.append(3) ylabels.append('Ampl [%]') if 'power' in add_waveforms: indices.append(4) ylabels.append('Power [dB]') if 'phase' in add_waveforms: indices.append(5) ylabels.append('Phase') phase = True def get_spectra(x): y = x[1][:,indices] if phase: y[y[:,-1]<0.0,-1] += 2.0*np.pi return (x[0], y) wave_data = list(map(get_spectra, wave_data)) else: xlabels.append('Frequency [Hz]') ylabels.append('Power [dB]') def get_spectra(x): y = x[1] y[:,1] = decibel(y[:,1], None) return (x[0], y) wave_data = list(map(get_spectra, wave_data)) self.set_wave_data(wave_data, xlabels, ylabels, True) def fix_scatter_plot(self, ax, data, label, axis): """Customize an axes of a scatter plot. - Limits for amplitude and time like quantities start at zero. - Phases a labeled with multuples of pi. - Species labels are rotated. """ if any(l in label for l in ['ampl', 'power', 'width', 'time', 'tau', 'P2-P1-dist', 'var', 'peak', 'trough', 'P2-P1-dist', 'rms', 'noise']): if np.all(data[np.isfinite(data)] >= 0.0): if axis == 'x': ax.set_xlim(0.0, None) elif axis == 'y': ax.set_ylim(0.0, None) elif axis == 'c': return 0.0, np.max(data), None else: if axis == 'x': ax.set_xlim(None, 0.0) elif axis == 'y': ax.set_ylim(None, 0.0) elif axis == 'c': return np.min(data), 0.0, None elif 'phase' in label: if axis == 'x': ax.set_xlim(-np.pi, np.pi) ax.set_xticks(np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi)) ax.set_xticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) elif axis == 'y': ax.set_ylim(-np.pi, np.pi) ax.set_yticks(np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi)) ax.set_yticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) elif axis == 'c': if ax is not None: ax.set_yticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) return -np.pi, np.pi, np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi) elif 'species' in label: if axis == 'x': for label in ax.get_xticklabels(): label.set_rotation(30) ax.set_xlabel('') ax.set_xlim(np.min(data)-0.5, np.max(data)+0.5) elif axis == 'y': ax.set_ylabel('') ax.set_ylim(np.min(data)-0.5, np.max(data)+0.5) elif axis == 'c': if ax is not None: ax.set_ylabel('') return np.min(data), np.max(data), None def fix_waveform_plot(self, axs, indices): """Adapt waveform plots to EOD waveforms, derivatives, and spectra. """ if len(indices) == 0: axs[0].text(0.5, 0.5, 'Click to plot EOD waveforms', transform = axs[0].transAxes, ha='center', va='center') axs[0].text(0.5, 0.3, 'n = %d' % len(self.raw_data), transform = axs[0].transAxes, ha='center', va='center') elif len(indices) == 1: file_name = self.eoddata[indices[0],'file'] if 'file' in self.eoddata else basename if 'index' in self.eoddata and np.isfinite(self.eoddata[indices[0],'index']) and \ np.any(self.eoddata[:,'index'] != self.eoddata[0,'index']): axs[0].set_title('%s: %d' % (file_name, self.eoddata[indices[0],'index'])) else: axs[0].set_title(file_name) if np.isfinite(self.eoddata[indices[0],'index']): axs[0].text(0.05, 0.85, '%.1fHz' % self.eoddata[indices[0],'EODf'], transform = axs[0].transAxes) else: axs[0].set_title('%d EOD waveforms selected' % len(indices)) for ax in axs: for l in ax.lines: l.set_linewidth(3.0) for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_ylim(top=1.1) ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if 'dV/dt' in xl: ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if 'd^2V/dt^2' in xl: ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if self.wave_fish: for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_xlim(-0.7, 0.7) if 'Ampl' in xl or 'Power' in xl or 'Phase' in xl: ax.set_xlim(-0.5, 8.5) for l in ax.lines: l.set_marker('.') l.set_markersize(15.0) l.set_markeredgewidth(0.5) l.set_markeredgecolor('k') l.set_markerfacecolor(l.get_color()) if 'Ampl' in xl: ax.set_ylim(0.0, 100.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(25.0)) if 'Power' in xl: ax.set_ylim(-60.0, 2.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0)) if 'Phase' in xl: ax.set_ylim(0.0, 2.0*np.pi) ax.set_yticks(np.arange(0.0, 2.5*np.pi, 0.5*np.pi)) ax.set_yticklabels(['0', u'\u03c0/2', u'\u03c0', u'3\u03c0/2', u'2\u03c0']) else: for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_xlim(-1.0, 1.5) if 'Power' in xl: ax.set_xlim(1.0, 2000.0) ax.set_xscale('log') ax.set_ylim(-60.0, 2.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0)) if len(indices) > 0: for ax in axs: ax.axhline(c='k', lw=1) def list_selection(self, indices): """List file names and indices of selection. If only a single EOD is selected, list all of its properties. """ print('') print('selected EODs:') if 'index' in self.eoddata and \ np.any(self.eoddata[:,'index'] != self.eoddata[0,'index']): for i in indices: file_name = self.eoddata[i,'file'] if 'file' in self.eoddata else basename if np.isfinite(self.eoddata[i,'index']): print('%s : %d' % (file_name, self.eoddata[i,'index'])) else: print(file_name) elif 'file' in self.eoddata: for i in indices: print(self.eoddata[i,'file']) if len(indices) == 1: # write eoddata line on terminal: keylen = 0 keys = [] values = [] for c in range(self.eoddata.columns()): k, v = self.eoddata.key_value(indices[0], c) keys.append(k) values.append(v) if keylen < len(k): keylen = len(k) for k, v in zip(keys, values): fs = '%%-%ds: %%s' % keylen print(fs % (k, v.strip())) def analyze_selection(self, index): """Launch thunderfish on the selected EOD. """ # load data: file_base = self.eoddata[index,'file'] if 'file' in self.eoddata else basename bp = os.path.join(self.path, file_base) fn = glob.glob(bp + '.*') if len(fn) == 0: print('no recording found for %s' % bp) return recording = fn[0] channel = 0 try: raw_data, rate, unit, ampl_max = load_data(recording) raw_data = raw_data[:,channel] except IOError as e: print('%s: failed to open file: did you provide a path to the raw data (-P option)?' % (recording)) return if len(raw_data) <= 1: print('%s: empty data file' % recording) return # load configuration: cfgfile = __package__ + '.cfg' cfg = configuration(cfgfile, False, recording) cfg.load_files(cfgfile, recording, 4) if 'flipped' in self.eoddata: fs = 'flip' if self.eoddata[index,'flipped'] else 'none' cfg.set('flipWaveEOD', fs) cfg.set('flipPulseEOD', fs) # best_window: data, idx0, idx1, clipped, min_clip, max_clip = \ analysis_window(raw_data, rate, ampl_max, cfg.value('windowPosition'), cfg) # detect EODs in the data: psd_data, fishlist, _, eod_props, mean_eods, \ spec_data, peak_data, power_thresh, skip_reason, zoom_window = \ detect_eods(data, rate, min_clip, max_clip, recording, 0, 0, cfg) # plot EOD: idx = int(self.eoddata[index,'index']) if 'index' in self.eoddata else 0 for k in ['toolbar', 'keymap.back', 'keymap.forward', 'keymap.zoom', 'keymap.pan']: plt.rcParams[k] = self.plt_params[k] fig = plot_eods(file_base, None, raw_data, rate, None, idx0, idx1, clipped, psd_data[0], fishlist, None, mean_eods, eod_props, peak_data, spec_data, [idx], unit, zoom_window, 10, None, True, False, 'auto', False, 0.0, 3000.0, interactive=True, verbose=0) fig.canvas.manager.set_window_title('thunderfish: %s' % file_base) plt.show(block=False) """Names of groups of data columns that can be selected by the select_EOD_properties() function. """ groups = ['all', 'allpower', 'noise', 'timing', 'wave', 'ampl', 'relampl', 'power', 'relpower', 'phase', 'time', 'width', 'peaks', 'none'] @staticmethod def select_EOD_properties(data, wave_fish, max_n, column_groups, add_columns): """Select data columns to be explored. First, groups of columns are selected, then individual columns. Columns that are selected twice are removed from the selection. Parameters ---------- data: TableData Table with EOD properties from which columns are selected. wave_fish: boolean. Indicates if data contains properties of wave- or pulse-type electric fish. max_n: int Maximum number of harmonics (wave-type fish) or peaks (pulse-type fish) to be selected. column_groups: list of string List of name denoting groups of columns to be selected. Supported groups are listed in `EODExplor.groups`. add_columns: list of string or int List of further individual columns to be selected. Returns ------- data_cols: list of int Indices of data columns to be shown by EODExplorer. error: string In case of an invalid column group, an error string. """ if wave_fish: # maximum number of harmonics: if max_n == 0: max_n = 100 else: max_n += 1 for k in range(1, max_n): if not ('phase%d' % k) in data: max_n = k break else: # minimum number of peaks: min_peaks = -10 for k in range(1, min_peaks, -1): if not ('P%dampl' % k) in data or not np.all(np.isfinite(data[:,'P%dampl' % k])): min_peaks = k+1 break # maximum number of peaks: if max_n == 0: max_peaks = 20 else: max_peaks = max_n + 1 for k in range(1, max_peaks): if not ('P%dampl' % k) in data or not np.all(np.isfinite(data[:,'P%dampl' % k])): max_peaks = k break # default columns: group_cols = ['EODf'] if 'EODf_adjust' in data: group_cols.append('EODf_adjust') if len(column_groups) == 0: column_groups = ['all'] for group in column_groups: if group == 'none': group_cols = [] elif wave_fish: if group == 'noise': group_cols.extend(['noise', 'rmserror', 'power', 'thd', 'dbdiff', 'maxdb', 'p-p-amplitude', 'relampl1', 'relampl2', 'relampl3']) elif group == 'timing' or group == 'time': group_cols.extend(['peakwidth', 'troughwidth', 'p-p-distance', 'leftpeak', 'rightpeak', 'lefttrough', 'righttrough']) elif group == 'wave': group_cols.extend(['thd', 'minwidth', 'min-p-p-distance', 'relpeakampl']) elif group == 'ampl': for k in range(0, max_n): group_cols.append('ampl%d' % k) elif group == 'relampl': group_cols.append('thd') group_cols.append('relpeakampl') for k in range(1, max_n): group_cols.append('relampl%d' % k) elif group == 'relpower' or group == 'power': for k in range(1, max_n): group_cols.append('relpower%d' % k) elif group == 'phase': for k in range(0, max_n): group_cols.append('phase%d' % k) elif group == 'all': group_cols.extend(['thd', 'minwidth', 'min-p-p-distance', 'relpeakampl']) for k in range(1, max_n): group_cols.append('relampl%d' % k) group_cols.append('phase%d' % k) elif group == 'allpower': group_cols.append('thd') for k in range(1, max_n): group_cols.append('relpower%d' % k) group_cols.append('phase%d' % k) else: return None, '"%s" is not a valid data group for wavefish' % group else: # pulse fish if group == 'noise': group_cols.extend(['noise', 'p-p-amplitude', 'min-ampl', 'max-ampl']) elif group == 'timing': group_cols.extend(['tstart', 'tend', 'width', 'tau', 'P2-P1-dist', 'firstpeak', 'lastpeak']) elif group == 'power': group_cols.extend(['peakfreq', 'peakpower', 'poweratt5', 'poweratt50', 'lowcutoff']) elif group == 'time': for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%dtime' % k) elif group == 'ampl': for k in range(min_peaks, max_peaks): group_cols.append('P%dampl' % k) elif group == 'relampl': for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%drelampl' % k) elif group == 'width': for k in range(min_peaks, max_peaks): group_cols.append('P%dwidth' % k) elif group == 'peaks': group_cols.append('firstpeak') group_cols.append('lastpeak') elif group == 'all': group_cols.extend(['firstpeak', 'lastpeak']) for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%drelampl' % k) group_cols.append('P%dtime' % k) group_cols.append('P%dwidth' % k) group_cols.extend(['tau', 'P2-P1-dist', 'peakfreq', 'poweratt5']) else: return None, '"%s" is not a valid data group for pulsefish' % group # additional data columns: group_cols.extend(add_columns) # translate to indices: data_cols = select_features(data, group_cols) return data_cols, NoneSimple GUI for viewing and exploring properties of EOD waveforms.
EODExplorer adapts a MultivariateExplorer to specific needs of EODs.
Static Members
groups: names of groups of data columns that can be selected.select_EOD_properties(): select data columns to be explored.
Parameters
data:TableData- Full table of EOD properties. Each row is a fish.
data_cols:listofstringorints- Names or indices of columns in
datato be explored. You may use the static functionselect_EOD_properties()for assisting the selection of columns. wave_fish:boolean- True if data are about wave-type weakly electric fish. False if data are about pulse-type weakly electric fish.
eod_data:listofwaveform data- Either waveform data is only the EOD waveform, a ndarray of shape (time, ['time', 'voltage']), or it is a list with the first element being the EOD waveform, and the second element being a 2D ndarray of spectral properties of the EOD waveform with first column being the frequency or harmonics.
add_waveforms:listofstring- List of what should be shown as waveform. Elements can be 'first', 'second', 'ampl', 'power', or 'phase'. For 'first' and 'second' the first and second derivatives of the supplied EOD waveform a computed and shown. 'ampl', 'power', and 'phase' select properties of the provided spectral properties.
loaded_spec:boolean- Indicates whether eod_data contains spectral properties.
rawdata_path:string- Base path to the raw recording, needed to show thunderfish when double clicking on a single EOD.
Ancestors
- thunderlab.multivariateexplorer.MultivariateExplorer
Class variables
var groups
Static methods
def select_EOD_properties(data, wave_fish, max_n, column_groups, add_columns)-
Expand source code
@staticmethod def select_EOD_properties(data, wave_fish, max_n, column_groups, add_columns): """Select data columns to be explored. First, groups of columns are selected, then individual columns. Columns that are selected twice are removed from the selection. Parameters ---------- data: TableData Table with EOD properties from which columns are selected. wave_fish: boolean. Indicates if data contains properties of wave- or pulse-type electric fish. max_n: int Maximum number of harmonics (wave-type fish) or peaks (pulse-type fish) to be selected. column_groups: list of string List of name denoting groups of columns to be selected. Supported groups are listed in `EODExplor.groups`. add_columns: list of string or int List of further individual columns to be selected. Returns ------- data_cols: list of int Indices of data columns to be shown by EODExplorer. error: string In case of an invalid column group, an error string. """ if wave_fish: # maximum number of harmonics: if max_n == 0: max_n = 100 else: max_n += 1 for k in range(1, max_n): if not ('phase%d' % k) in data: max_n = k break else: # minimum number of peaks: min_peaks = -10 for k in range(1, min_peaks, -1): if not ('P%dampl' % k) in data or not np.all(np.isfinite(data[:,'P%dampl' % k])): min_peaks = k+1 break # maximum number of peaks: if max_n == 0: max_peaks = 20 else: max_peaks = max_n + 1 for k in range(1, max_peaks): if not ('P%dampl' % k) in data or not np.all(np.isfinite(data[:,'P%dampl' % k])): max_peaks = k break # default columns: group_cols = ['EODf'] if 'EODf_adjust' in data: group_cols.append('EODf_adjust') if len(column_groups) == 0: column_groups = ['all'] for group in column_groups: if group == 'none': group_cols = [] elif wave_fish: if group == 'noise': group_cols.extend(['noise', 'rmserror', 'power', 'thd', 'dbdiff', 'maxdb', 'p-p-amplitude', 'relampl1', 'relampl2', 'relampl3']) elif group == 'timing' or group == 'time': group_cols.extend(['peakwidth', 'troughwidth', 'p-p-distance', 'leftpeak', 'rightpeak', 'lefttrough', 'righttrough']) elif group == 'wave': group_cols.extend(['thd', 'minwidth', 'min-p-p-distance', 'relpeakampl']) elif group == 'ampl': for k in range(0, max_n): group_cols.append('ampl%d' % k) elif group == 'relampl': group_cols.append('thd') group_cols.append('relpeakampl') for k in range(1, max_n): group_cols.append('relampl%d' % k) elif group == 'relpower' or group == 'power': for k in range(1, max_n): group_cols.append('relpower%d' % k) elif group == 'phase': for k in range(0, max_n): group_cols.append('phase%d' % k) elif group == 'all': group_cols.extend(['thd', 'minwidth', 'min-p-p-distance', 'relpeakampl']) for k in range(1, max_n): group_cols.append('relampl%d' % k) group_cols.append('phase%d' % k) elif group == 'allpower': group_cols.append('thd') for k in range(1, max_n): group_cols.append('relpower%d' % k) group_cols.append('phase%d' % k) else: return None, '"%s" is not a valid data group for wavefish' % group else: # pulse fish if group == 'noise': group_cols.extend(['noise', 'p-p-amplitude', 'min-ampl', 'max-ampl']) elif group == 'timing': group_cols.extend(['tstart', 'tend', 'width', 'tau', 'P2-P1-dist', 'firstpeak', 'lastpeak']) elif group == 'power': group_cols.extend(['peakfreq', 'peakpower', 'poweratt5', 'poweratt50', 'lowcutoff']) elif group == 'time': for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%dtime' % k) elif group == 'ampl': for k in range(min_peaks, max_peaks): group_cols.append('P%dampl' % k) elif group == 'relampl': for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%drelampl' % k) elif group == 'width': for k in range(min_peaks, max_peaks): group_cols.append('P%dwidth' % k) elif group == 'peaks': group_cols.append('firstpeak') group_cols.append('lastpeak') elif group == 'all': group_cols.extend(['firstpeak', 'lastpeak']) for k in range(min_peaks, max_peaks): if k != 1: group_cols.append('P%drelampl' % k) group_cols.append('P%dtime' % k) group_cols.append('P%dwidth' % k) group_cols.extend(['tau', 'P2-P1-dist', 'peakfreq', 'poweratt5']) else: return None, '"%s" is not a valid data group for pulsefish' % group # additional data columns: group_cols.extend(add_columns) # translate to indices: data_cols = select_features(data, group_cols) return data_cols, NoneSelect data columns to be explored.
First, groups of columns are selected, then individual columns. Columns that are selected twice are removed from the selection.
Parameters
data:TableData- Table with EOD properties from which columns are selected.
- wave_fish: boolean.
- Indicates if data contains properties of wave- or pulse-type electric fish.
max_n:int- Maximum number of harmonics (wave-type fish) or peaks (pulse-type fish) to be selected.
column_groups:listofstring- List of name denoting groups of columns to be selected. Supported groups are
listed in
EODExplor.groups. add_columns:listofstringorint- List of further individual columns to be selected.
Returns
data_cols:listofint- Indices of data columns to be shown by EODExplorer.
error:string- In case of an invalid column group, an error string.
Methods
def fix_scatter_plot(self, ax, data, label, axis)-
Expand source code
def fix_scatter_plot(self, ax, data, label, axis): """Customize an axes of a scatter plot. - Limits for amplitude and time like quantities start at zero. - Phases a labeled with multuples of pi. - Species labels are rotated. """ if any(l in label for l in ['ampl', 'power', 'width', 'time', 'tau', 'P2-P1-dist', 'var', 'peak', 'trough', 'P2-P1-dist', 'rms', 'noise']): if np.all(data[np.isfinite(data)] >= 0.0): if axis == 'x': ax.set_xlim(0.0, None) elif axis == 'y': ax.set_ylim(0.0, None) elif axis == 'c': return 0.0, np.max(data), None else: if axis == 'x': ax.set_xlim(None, 0.0) elif axis == 'y': ax.set_ylim(None, 0.0) elif axis == 'c': return np.min(data), 0.0, None elif 'phase' in label: if axis == 'x': ax.set_xlim(-np.pi, np.pi) ax.set_xticks(np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi)) ax.set_xticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) elif axis == 'y': ax.set_ylim(-np.pi, np.pi) ax.set_yticks(np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi)) ax.set_yticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) elif axis == 'c': if ax is not None: ax.set_yticklabels([u'-\u03c0', u'-\u03c0/2', '0', u'\u03c0/2', u'\u03c0']) return -np.pi, np.pi, np.arange(-np.pi, 1.5*np.pi, 0.5*np.pi) elif 'species' in label: if axis == 'x': for label in ax.get_xticklabels(): label.set_rotation(30) ax.set_xlabel('') ax.set_xlim(np.min(data)-0.5, np.max(data)+0.5) elif axis == 'y': ax.set_ylabel('') ax.set_ylim(np.min(data)-0.5, np.max(data)+0.5) elif axis == 'c': if ax is not None: ax.set_ylabel('') return np.min(data), np.max(data), NoneCustomize an axes of a scatter plot.
- Limits for amplitude and time like quantities start at zero.
- Phases a labeled with multuples of pi.
- Species labels are rotated.
def fix_waveform_plot(self, axs, indices)-
Expand source code
def fix_waveform_plot(self, axs, indices): """Adapt waveform plots to EOD waveforms, derivatives, and spectra. """ if len(indices) == 0: axs[0].text(0.5, 0.5, 'Click to plot EOD waveforms', transform = axs[0].transAxes, ha='center', va='center') axs[0].text(0.5, 0.3, 'n = %d' % len(self.raw_data), transform = axs[0].transAxes, ha='center', va='center') elif len(indices) == 1: file_name = self.eoddata[indices[0],'file'] if 'file' in self.eoddata else basename if 'index' in self.eoddata and np.isfinite(self.eoddata[indices[0],'index']) and \ np.any(self.eoddata[:,'index'] != self.eoddata[0,'index']): axs[0].set_title('%s: %d' % (file_name, self.eoddata[indices[0],'index'])) else: axs[0].set_title(file_name) if np.isfinite(self.eoddata[indices[0],'index']): axs[0].text(0.05, 0.85, '%.1fHz' % self.eoddata[indices[0],'EODf'], transform = axs[0].transAxes) else: axs[0].set_title('%d EOD waveforms selected' % len(indices)) for ax in axs: for l in ax.lines: l.set_linewidth(3.0) for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_ylim(top=1.1) ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if 'dV/dt' in xl: ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if 'd^2V/dt^2' in xl: ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=4)) if self.wave_fish: for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_xlim(-0.7, 0.7) if 'Ampl' in xl or 'Power' in xl or 'Phase' in xl: ax.set_xlim(-0.5, 8.5) for l in ax.lines: l.set_marker('.') l.set_markersize(15.0) l.set_markeredgewidth(0.5) l.set_markeredgecolor('k') l.set_markerfacecolor(l.get_color()) if 'Ampl' in xl: ax.set_ylim(0.0, 100.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(25.0)) if 'Power' in xl: ax.set_ylim(-60.0, 2.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0)) if 'Phase' in xl: ax.set_ylim(0.0, 2.0*np.pi) ax.set_yticks(np.arange(0.0, 2.5*np.pi, 0.5*np.pi)) ax.set_yticklabels(['0', u'\u03c0/2', u'\u03c0', u'3\u03c0/2', u'2\u03c0']) else: for ax, xl in zip(axs, self.wave_ylabels): if 'Voltage' in xl: ax.set_xlim(-1.0, 1.5) if 'Power' in xl: ax.set_xlim(1.0, 2000.0) ax.set_xscale('log') ax.set_ylim(-60.0, 2.0) ax.yaxis.set_major_locator(ticker.MultipleLocator(20.0)) if len(indices) > 0: for ax in axs: ax.axhline(c='k', lw=1)Adapt waveform plots to EOD waveforms, derivatives, and spectra.
def list_selection(self, indices)-
Expand source code
def list_selection(self, indices): """List file names and indices of selection. If only a single EOD is selected, list all of its properties. """ print('') print('selected EODs:') if 'index' in self.eoddata and \ np.any(self.eoddata[:,'index'] != self.eoddata[0,'index']): for i in indices: file_name = self.eoddata[i,'file'] if 'file' in self.eoddata else basename if np.isfinite(self.eoddata[i,'index']): print('%s : %d' % (file_name, self.eoddata[i,'index'])) else: print(file_name) elif 'file' in self.eoddata: for i in indices: print(self.eoddata[i,'file']) if len(indices) == 1: # write eoddata line on terminal: keylen = 0 keys = [] values = [] for c in range(self.eoddata.columns()): k, v = self.eoddata.key_value(indices[0], c) keys.append(k) values.append(v) if keylen < len(k): keylen = len(k) for k, v in zip(keys, values): fs = '%%-%ds: %%s' % keylen print(fs % (k, v.strip()))List file names and indices of selection.
If only a single EOD is selected, list all of its properties.
def analyze_selection(self, index)-
Expand source code
def analyze_selection(self, index): """Launch thunderfish on the selected EOD. """ # load data: file_base = self.eoddata[index,'file'] if 'file' in self.eoddata else basename bp = os.path.join(self.path, file_base) fn = glob.glob(bp + '.*') if len(fn) == 0: print('no recording found for %s' % bp) return recording = fn[0] channel = 0 try: raw_data, rate, unit, ampl_max = load_data(recording) raw_data = raw_data[:,channel] except IOError as e: print('%s: failed to open file: did you provide a path to the raw data (-P option)?' % (recording)) return if len(raw_data) <= 1: print('%s: empty data file' % recording) return # load configuration: cfgfile = __package__ + '.cfg' cfg = configuration(cfgfile, False, recording) cfg.load_files(cfgfile, recording, 4) if 'flipped' in self.eoddata: fs = 'flip' if self.eoddata[index,'flipped'] else 'none' cfg.set('flipWaveEOD', fs) cfg.set('flipPulseEOD', fs) # best_window: data, idx0, idx1, clipped, min_clip, max_clip = \ analysis_window(raw_data, rate, ampl_max, cfg.value('windowPosition'), cfg) # detect EODs in the data: psd_data, fishlist, _, eod_props, mean_eods, \ spec_data, peak_data, power_thresh, skip_reason, zoom_window = \ detect_eods(data, rate, min_clip, max_clip, recording, 0, 0, cfg) # plot EOD: idx = int(self.eoddata[index,'index']) if 'index' in self.eoddata else 0 for k in ['toolbar', 'keymap.back', 'keymap.forward', 'keymap.zoom', 'keymap.pan']: plt.rcParams[k] = self.plt_params[k] fig = plot_eods(file_base, None, raw_data, rate, None, idx0, idx1, clipped, psd_data[0], fishlist, None, mean_eods, eod_props, peak_data, spec_data, [idx], unit, zoom_window, 10, None, True, False, 'auto', False, 0.0, 3000.0, interactive=True, verbose=0) fig.canvas.manager.set_window_title('thunderfish: %s' % file_base) plt.show(block=False)Launch thunderfish on the selected EOD.