Coverage for src/thunderlab/dataloader.py: 84%
1144 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 15:01 +0000
1"""Load time-series data from files.
3```
4data, rate, unit, amax = load_data('data/file.wav')
5```
7The function `data_loader()` loads the whole time-series from the file
8as a numpy array of floats. First dimension is frames, second is
9channels. In contrast to the `audioio.load_audio()` function, the
10values of the data array are not restricted between -1 and 1. They can
11assume any value wihin the range `-amax` to `+amax` with the returned
12`unit`.
14```
15data = DataLoader('data/file.wav', 60.0)
16```
17or
18```
19with DataLoader('data/file.wav', 60.0) as data:
20```
21Create an `DataLoader` object that loads chuncks of 60 seconds long data
22on demand. `data` can be used like a read-only numpy array of floats.
25## Supported file formats
27- python pickle files
28- numpy .npz files
29- matlab .mat files
30- audio files via [`audioio`](https://github.com/bendalab/audioio) package
31- LabView .scandat files
32- raw files
33- relacs files (https://www.relacs.net)
34- fishgrid files (https://github.com/bendalab/fishgrid)
37## Metadata
39Many file formats allow to store metadata that further describe the
40stored time series data. We handle them as nested dictionary of key-value
41pairs. Load them with the `metadata()` function:
42```
43metadata = metadata('data/file.mat')
44```
46## Markers
48Some file formats also allow to store markers that mark specific
49positions in the time series data. Load marker positions and spans (in
50the 2-D array `locs`) and label and text strings (in the 2-D array
51`labels`) with the `markers()` function:
52```
53locs, labels = markers('data.wav')
54```
56## Aditional, format specific functions
58- `extract_container_metadata()`: extract metadata from dictionary loaded from a container file.
59- `relacs_samplerate_unit()`: retrieve sampling rate and unit from a relacs stimuli.dat file.
60- `relacs_header()`: read key-value pairs from relacs *.dat file headers.
61- `fishgrid_grids()`: retrieve grid sizes from a fishgrid.cfg file.
62- `fishgrid_spacings()`: spacing between grid electrodes.
64"""
66import gc
67import os
68import sys
69import gzip
70import pickle
71import numpy as np
72try:
73 import matplotlib.pyplot as plt
74except ImportError:
75 pass
77from pathlib import Path
78from datetime import timedelta
80from audioio import load_audio, AudioLoader, unflatten_metadata
81from audioio import get_number_unit, get_number, get_int, get_bool, get_gain
82from audioio import default_starttime_keys, default_gain_keys
83from audioio import get_datetime, flatten_metadata, add_metadata, set_starttime
84from audioio import metadata as metadata_audioio
85from audioio import markers as markers_audioio
88def relacs_samplerate_unit(filepath, channel=0):
89 """Retrieve sampling rate and unit from a relacs stimuli.dat file.
91 Parameters
92 ----------
93 filepath: str or Path
94 Path to a relacs data directory, or a file in a relacs data directory.
95 channel: int
96 Channel (trace) number, if `filepath` does not specify a
97 trace-*.raw file.
99 Returns
100 -------
101 samplerate: float
102 Sampling rate in Hertz
103 unit: str
104 Unit of the trace, can be empty if not found
106 Raises
107 ------
108 IOError/FileNotFoundError:
109 If the stimuli.dat file does not exist.
110 ValueError:
111 stimuli.dat file does not contain sampling rate.
112 """
113 trace = channel + 1
114 relacs_dir = Path(filepath)
115 # check for relacs data directory:
116 if not relacs_dir.is_dir():
117 bn = relacs_dir.stem.lower()
118 ext = relacs_dir.suffix.lower()
119 relacs_dir = relacs_dir.parent
120 if len(bn) > 6 and bn[:6] == 'trace-':
121 trace = int(bn[6:])
123 # retreive sampling rate and unit from stimuli.dat file:
124 samplerate = None
125 sampleinterval = None
126 unit = ""
128 # load stimuli.dat file:
129 lines = []
130 stimuli_file = relacs_dir / 'stimuli.dat.gz'
131 if stimuli_file.is_file():
132 with gzip.open(stimuli_file, 'r', encoding='latin-1') as sf:
133 for line in sf:
134 line = line.strip()
135 if len(line) == 0 or line[0] != '#':
136 break
137 lines.append(line)
138 else:
139 stimuli_file = relacs_dir / 'stimuli.dat'
140 with open(stimuli_file, 'r', encoding='latin-1') as sf:
141 for line in sf:
142 line = line.strip()
143 if len(line) == 0 or line[0] != '#':
144 break
145 lines.append(line)
146 # extract unit and sampling rate:
147 for line in lines:
148 if f'unit{trace}' in line:
149 unit = line.split(':')[1].strip()
150 if f'sampling rate{trace}' in line:
151 value = line.split(':')[1].strip()
152 samplerate = float(value.replace('Hz',''))
153 elif f'sample interval{trace}' in line:
154 value = line.split(':')[1].strip()
155 sampleinterval = float(value.replace('ms',''))
157 if samplerate is not None:
158 return samplerate, unit
159 if sampleinterval is not None:
160 return 1000/sampleinterval, unit
161 raise ValueError(f'could not retrieve sampling rate from {stimuli_file}')
164def relacs_header(filepath, store_empty=False, first_only=False,
165 lower_keys=False, flat=False,
166 add_sections=False):
167 """Read key-value pairs from a relacs *.dat file header.
169 Parameters
170 ----------
171 filepath: str or Path
172 A relacs *.dat file, can be also a zipped .gz file.
173 store_empty: bool
174 If `False` do not add meta data with empty values.
175 first_only: bool
176 If `False` only store the first element of a list.
177 lower_keys: bool
178 Make all keys lower case.
179 flat: bool
180 Do not make a nested dictionary.
181 Use this option also to read in very old relacs metadata with
182 ragged left alignment.
183 add_sections: bool
184 If `True`, prepend keys with sections names separated by
185 '.' to make them unique.
187 Returns
188 -------
189 data: dict
190 Nested dictionary with key-value pairs of the file header.
192 Raises
193 ------
194 IOError/FileNotFoundError:
195 If `filepath` cannot be opened.
196 """
197 filepath = Path(filepath)
198 # read in header from file:
199 lines = []
200 gzfilepath = filepath.with_suffix(filepath.suffix + '.gz')
201 if gzfilepath.is_file():
202 with gzip.open(gzfilepath, 'r', encoding='latin-1') as sf:
203 for line in sf:
204 line = line.strip()
205 if len(line) == 0 or line[0] != '#':
206 break
207 lines.append(line)
208 else:
209 with open(filepath, 'r', encoding='latin-1') as sf:
210 for line in sf:
211 line = line.strip()
212 if len(line) == 0 or line[0] != '#':
213 break
214 lines.append(line)
215 # parse:
216 data = {}
217 cdatas = [data]
218 sections = ['']
219 ident_offs = None
220 ident = None
221 for line in lines:
222 words = line.split(':')
223 value = ':'.join(words[1:]).strip() if len(words) > 1 else ''
224 if len(words) >= 1:
225 key = words[0].strip('#')
226 # get section level:
227 level = 0
228 if not flat or len(value) == 0:
229 nident = len(key) - len(key.lstrip())
230 if ident_offs is None:
231 ident_offs = nident
232 elif ident is None:
233 if nident > ident_offs:
234 ident = nident - ident_offs
235 level = 1
236 else:
237 level = (nident - ident_offs)//ident
238 # close sections:
239 if not flat:
240 while len(cdatas) > level + 1:
241 cdatas[-1][sections.pop()] = cdatas.pop()
242 else:
243 while len(sections) > level + 1:
244 sections.pop()
245 # key:
246 key = key.strip().strip('"')
247 if lower_keys:
248 key = key.lower()
249 skey = key
250 if add_sections:
251 key = '.'.join(sections[1:] + [key])
252 if len(value) == 0:
253 # new sub-section:
254 if flat:
255 if store_empty:
256 cdatas[-1][key] = None
257 else:
258 cdatas.append({})
259 sections.append(skey)
260 else:
261 # key-value pair:
262 value = value.strip('"')
263 if len(value) > 0 or value != '-' or store_empty:
264 if len(value) > 0 and value[0] == '[' and value[-1] == ']':
265 value = [v.strip() for v in value.lstrip('[').rstrip(']').split(',')]
266 if first_only:
267 value = value[0]
268 cdatas[-1][key] = value
269 while len(cdatas) > 1:
270 cdatas[-1][sections.pop()] = cdatas.pop()
271 return data
274def check_relacs(filepath):
275 """Check for valid relacs file.
277 Parameters
278 ----------
279 filepath: str or Path
280 Path to a relacs data directory, or a file in a relacs data directory.
282 Returns
283 -------
284 is_relacs: boolean
285 `True` if `filepath` is a valid relacs directory or is a file therein.
286 """
287 # relacs data directory:
288 relacs_dir = Path(filepath)
289 if not relacs_dir.is_dir():
290 relacs_dir = relacs_dir.parent
291 # check for a valid relacs data directory:
292 has_stimuli = False
293 has_trace = False
294 for fname in ['stimuli.dat', 'stimuli.dat.gz']:
295 if (relacs_dir / fname).is_file():
296 has_stimuli = True
297 break
298 for fname in ['trace-1.raw', 'trace-1.raw.gz']:
299 if (relacs_dir / fname).is_file():
300 has_trace = True
301 break
302 return has_stimuli and has_trace
305def relacs_trace_files(filepath):
306 """Expand file path for relacs data to appropriate trace*.raw file names.
308 Parameters
309 ----------
310 filepath: str or Path
311 Path to a relacs data directory, or a file in a relacs data directory.
313 Returns
314 -------
315 trace_filepaths: list of Path
316 List of relacs trace*.raw files.
317 """
318 relacs_dir = Path(filepath)
319 if not relacs_dir.is_dir():
320 relacs_dir = relacs_dir.parent
321 trace_filepaths = []
322 for k in range(10000):
323 trace_file = relacs_dir / f'trace-{k+1}.raw'
324 gz_trace_file = relacs_dir / f'trace-{k+1}.raw.gz'
325 if trace_file.is_file():
326 trace_filepaths.append(trace_file)
327 elif gz_trace_file.is_file():
328 trace_filepaths.append(gz_trace_file)
329 else:
330 break
331 return trace_filepaths
334def load_relacs(filepath, amax=1.0):
335 """Load traces that have been recorded with relacs (https://github.com/relacs/relacs).
337 Parameters
338 ----------
339 filepath: str of Path
340 Path to a relacs data directory, or a file in a relacs data directory.
341 amax: float
342 The amplitude range of the data.
344 Returns
345 -------
346 data: 2-D array
347 All data traces as an 2-D numpy array, even for single channel data.
348 First dimension is time, second is channel.
349 rate: float
350 Sampling rate of the data in Hz
351 unit: str
352 Unit of the data
353 amax: float
354 Maximum amplitude of data range.
356 Raises
357 ------
358 FileNotFoundError:
359 Invalid or non existing relacs files.
360 ValueError:
361 - Invalid name for relacs trace-*.raw file.
362 - Sampling rates of traces differ.
363 - Unit of traces differ.
364 """
365 trace_filepaths = relacs_trace_files(filepath)
366 if len(trace_filepaths) == 0:
367 raise FileNotFoundError(f'no relacs files found')
368 # load trace*.raw files:
369 nchannels = len(trace_filepaths)
370 data = None
371 nrows = 0
372 rate = None
373 unit = ''
374 for c, path in enumerate(sorted(trace_filepaths)):
375 if path.suffix == '.gz':
376 with gzip.open(path, 'rb') as sf:
377 x = np.frombuffer(sf.read(), dtype=np.float32)
378 else:
379 x = np.fromfile(path, np.float32)
380 if data is None:
381 nrows = len(x)
382 data = np.zeros((nrows, nchannels))
383 n = min(len(x), nrows)
384 data[:n,c] = x[:n]
385 # retrieve sampling rate and unit:
386 crate, us = relacs_samplerate_unit(path, c)
387 if rate is None:
388 rate = crate
389 elif crate != rate:
390 raise ValueError('sampling rates of traces differ')
391 if len(unit) == 0:
392 unit = us
393 elif us != unit:
394 raise ValueError('unit of traces differ')
395 return data, rate, unit, amax
398def metadata_relacs(filepath, store_empty=False, first_only=False,
399 lower_keys=False, flat=False, add_sections=False):
400 """ Read meta-data of a relacs data set.
402 Parameters
403 ----------
404 filepath: str or Path
405 A relacs data directory or a file therein.
406 store_empty: bool
407 If `False` do not add meta data with empty values.
408 first_only: bool
409 If `False` only store the first element of a list.
410 lower_keys: bool
411 Make all keys lower case.
412 flat: bool
413 Do not make a nested dictionary.
414 Use this option also to read in very old relacs metadata with
415 ragged left alignment.
416 add_sections: bool
417 If `True`, prepend keys with sections names separated by
418 '.' to make them unique.
420 Returns
421 -------
422 data: nested dict
423 Nested dictionary with key-value pairs of the meta data.
424 """
425 relacs_dir = Path(filepath)
426 if not relacs_dir.is_dir():
427 relacs_dir = relacs_dir.parent
428 info_path = relacs_dir / 'info.dat'
429 if not info_path.is_file():
430 return dict()
431 data = relacs_header(info_path, store_empty, first_only,
432 lower_keys, flat, add_sections)
433 return data
436def fishgrid_spacings(metadata, unit='m'):
437 """Spacing between grid electrodes.
439 Parameters
440 ----------
441 metadata: dict
442 Fishgrid metadata obtained from `metadata_fishgrid()`.
443 unit: str
444 Unit in which to return the spacings.
446 Returns
447 -------
448 grid_dist: list of tuple of float
449 For each grid the distances between rows and columns in `unit`.
450 """
451 grids_dist = []
452 for k in range(4):
453 row_dist = get_number(metadata, unit, f'RowDistance{k+1}', default=0)
454 col_dist = get_number(metadata, unit, f'ColumnDistance{k+1}', default=0)
455 rows = get_int(metadata, f'Rows{k+1}', default=0)
456 cols = get_int(metadata, f'Columns{k+1}', default=0)
457 if get_bool(metadata, f'Used{k+1}', default=False) or \
458 cols > 0 and rows > 0:
459 grids_dist.append((row_dist, col_dist))
460 return grids_dist
463def fishgrid_grids(metadata):
464 """Retrieve grid sizes from a fishgrid.cfg file.
466 Parameters
467 ----------
468 metadata: dict
469 Fishgrid metadata obtained from `metadata_fishgrid()`.
471 Returns
472 -------
473 grids: list of tuple of int
474 For each grid the number of rows and columns.
475 """
476 grids = []
477 for k in range(4):
478 rows = get_int(metadata, f'Rows{k+1}', default=0)
479 cols = get_int(metadata, f'Columns{k+1}', default=0)
480 if get_bool(metadata, f'Used{k+1}', default=False) or \
481 cols > 0 and rows > 0:
482 grids.append((rows, cols))
483 return grids
486def check_fishgrid(filepath):
487 """Check for valid fishgrid file (https://github.com/bendalab/fishgrid).
489 Parameters
490 ----------
491 filepath: str or Path
492 Path to a fishgrid data directory or a file in a fishgrid
493 data directory.
495 Returns
496 -------
497 is_fishgrid: bool
498 `True` if `filepath` is a valid fishgrid data directory or
499 a file therein.
500 """
501 # fishgrid data directory:
502 fishgrid_dir = Path(filepath)
503 if not fishgrid_dir.is_dir():
504 fishgrid_dir = fishgrid_dir.parent
505 # check for a valid fishgrid data directory:
506 return ((fishgrid_dir / 'fishgrid.cfg').is_file() and
507 ((fishgrid_dir / 'traces-grid1.raw').is_file() or
508 (fishgrid_dir / 'traces.raw').is_file()))
511def fishgrid_trace_files(filepath):
512 """Expand file paths for fishgrid data to appropriate traces*.raw file names.
514 Parameters
515 ----------
516 filepath: str or Path
517 Path to a fishgrid data directory, or a file therein.
519 Returns
520 -------
521 trace_filepaths: list of Path
522 List of fishgrid traces*.raw files.
523 """
524 fishgrid_dir = Path(filepath)
525 if not fishgrid_dir.is_dir():
526 fishgrid_dir = fishgrid_dir.parent
527 # find grids:
528 trace_filepaths = []
529 for k in range(10000):
530 trace_file = fishgrid_dir / f'traces-grid{k+1}.raw'
531 gz_trace_file = fishgrid_dir / f'traces-grid{k+1}.raw.gz'
532 if trace_file.is_file():
533 trace_filepaths.append(trace_file)
534 elif gz_trace_file.is_file():
535 trace_filepaths.append(gz_trace_file)
536 else:
537 break
538 if len(trace_filepaths) == 0:
539 trace_file = fishgrid_dir / f'traces.raw'
540 gz_trace_file = fishgrid_dir / f'traces.raw.gz'
541 if trace_file.is_file():
542 trace_filepaths.append(trace_file)
543 elif gz_trace_file.is_file():
544 trace_filepaths.append(gz_trace_file)
545 return trace_filepaths
548def load_fishgrid(filepath):
549 """Load traces that have been recorded with fishgrid (https://github.com/bendalab/fishgrid).
551 Parameters
552 ----------
553 filepath: str
554 Path to a fishgrid data directory, or a file therein.
556 Returns
557 -------
558 data: 2-D array
559 All data traces as an 2-D numpy array, even for single channel data.
560 First dimension is time, second is channel.
561 rate: float
562 Sampling rate of the data in Hz.
563 unit: str
564 Unit of the data.
565 amax: float
566 Maximum amplitude of data range.
568 Raises
569 ------
570 FileNotFoundError:
571 Invalid or non existing fishgrid files.
572 """
573 trace_filepaths = fishgrid_trace_files(filepath)
574 if len(trace_filepaths) == 0:
575 raise FileNotFoundError(f'no fishgrid files found')
576 md = metadata_fishgrid(filepath)
577 grids = fishgrid_grids(md)
578 grid_sizes = [r*c for r, c in grids]
580 # load traces-grid*.raw files:
581 grid_channels = []
582 nchannels = 0
583 for g, path in enumerate(trace_filepaths):
584 grid_channels.append(grid_sizes[g])
585 nchannels += grid_sizes[g]
586 data = None
587 nrows = 0
588 c = 0
589 rate = get_number(md, 'Hz', 'AISampleRate')
590 for path, channels in zip(trace_filepaths, grid_channels):
591 if path.suffix == '.gz':
592 with gzip.open(path, 'rb') as sf:
593 x = np.frombuffer(sf.read(), dtype=np.float32)
594 else:
595 x = np.fromfile(path, np.float32).reshape((-1, channels))
596 if data is None:
597 nrows = len(x)
598 data = np.zeros((nrows, nchannels))
599 n = min(len(x), nrows)
600 data[:n, c:c + channels] = x[:n, :]
601 c += channels
602 amax, unit = get_number_unit(md, 'AIMaxVolt')
603 return data, rate, unit, amax
606# add fishgrid keys:
607default_starttime_keys.append(['StartDate', 'StartTime'])
608default_gain_keys.insert(0, 'AIMaxVolt')
611def metadata_fishgrid(filepath):
612 """ Read meta-data of a fishgrid data set.
614 Parameters
615 ----------
616 filepath: str or Path
617 A fishgrid data directory or a file therein.
619 Returns
620 -------
621 data: nested dict
622 Nested dictionary with key-value pairs of the meta data.
623 """
624 fishgrid_dir = Path(filepath)
625 if not fishgrid_dir.is_dir():
626 fishgrid_dir = fishgrid_dir.parent
627 config_path = fishgrid_dir / 'fishgrid.cfg'
628 gz_config_path = fishgrid_dir / 'fishgrid.cfg.gz'
629 # read in header from file:
630 lines = []
631 if gz_config_path.is_file():
632 with gzip.open(gz_config_path, 'r', encoding='latin-1') as sf:
633 for line in sf:
634 lines.append(line)
635 elif config_path.is_file():
636 with open(config_path, 'r', encoding='latin-1') as sf:
637 for line in sf:
638 lines.append(line)
639 else:
640 return {}
641 # parse:
642 data = {}
643 cdatas = [data]
644 ident_offs = None
645 ident = None
646 old_style = False
647 grid_n = False
648 for line in lines:
649 if len(line.strip()) == 0:
650 continue
651 if line[0] == '*':
652 key = line[1:].strip()
653 data[key] = {}
654 cdatas = [data, data[key]]
655 elif '----' in line:
656 old_style = True
657 key = line.strip().strip(' -').replace('&', '')
658 if key.upper() == 'SETUP':
659 key = 'Grid 1'
660 grid_n = False
661 if key[:4].lower() == 'grid':
662 grid_n = key[5]
663 cdatas = cdatas[:2]
664 cdatas[1][key] = {}
665 cdatas.append(cdatas[1][key])
666 else:
667 words = line.split(':')
668 key = words[0].strip().strip('"')
669 value = None
670 if len(words) > 1 and (len(words[1].strip()) > 0 or old_style):
671 value = ':'.join(words[1:]).strip().strip('"')
672 if old_style:
673 if value is None:
674 cdatas = cdatas[:3]
675 cdatas[2][key] = {}
676 cdatas.append(cdatas[2][key])
677 else:
678 if grid_n and key[-1] != grid_n:
679 key = key + grid_n
680 cdatas[-1][key] = value
681 else:
682 # get section level:
683 level = 0
684 nident = len(line) - len(line.lstrip())
685 if ident_offs is None:
686 ident_offs = nident
687 elif ident is None:
688 if nident > ident_offs:
689 ident = nident - ident_offs
690 level = 1
691 else:
692 level = (nident - ident_offs)//ident
693 # close sections:
694 cdatas = cdatas[:2 + level]
695 if value is None:
696 # new section:
697 cdatas[-1][key] = {}
698 cdatas.append(cdatas[-1][key])
699 else:
700 # key-value pair:
701 cdatas[-1][key] = value.replace(r'\n', '\n')
702 # remove unused grids:
703 fgm = data.get('FishGrid', {})
704 for i in range(4):
705 gs = f'Grid {i+1}'
706 if gs in fgm:
707 gm = fgm[gs]
708 us = f'Used{i+1}'
709 if us in gm and gm[us].upper() == 'FALSE':
710 del fgm[gs]
711 return data
714def markers_fishgrid(filepath):
715 """ Read markers of a fishgrid data set.
717 Parameters
718 ----------
719 filepath: str or Path
720 A fishgrid data directory or a file therein.
722 Returns
723 -------
724 locs: 2-D array of ints
725 Marker positions (first column) and spans (second column)
726 for each marker (rows).
727 labels: 2-D array of string objects
728 Labels (first column) and texts (second column)
729 for each marker (rows).
730 """
731 def add_marker():
732 if 'index1' in marker:
733 index1 = int(marker['index1'])//nchannels
734 else:
735 index1 = int(marker['index'])//nchannels
736 span1 = int(marker.get('span1', 0))//nchannels
737 locs.append([index1, span1])
738 ls = marker.get('label', 'M')
739 cs = marker.get('comment', '')
740 labels.append([ls, cs])
742 fishgrid_dir = Path(filepath)
743 if not fishgrid_dir.is_dir():
744 fishgrid_dir = fishgrid_dir.parent
745 path = fishgrid_dir / 'timestamps.dat'
746 if not path.is_file():
747 return np.zeros((0, 2), dtype=int), np.zeros((0, 2), dtype=object)
748 # get number of channels:
749 md = metadata_fishgrid(path.with_name('fishgrid.cfg'))
750 grids = fishgrid_grids(md)
751 nchannels = np.prod(grids[0])
752 # read timestamps:
753 locs = []
754 labels = []
755 marker = {}
756 with open(path, 'r') as sf:
757 for line in sf:
758 if len(line.strip()) == 0:
759 add_marker()
760 marker = {}
761 else:
762 words = line.split(':')
763 if len(words) > 1:
764 v = words[1].strip()
765 v = v.strip('"')
766 marker[words[0].strip().lower()] = v
767 if len(marker) > 0:
768 add_marker()
769 if len(locs) > 2:
770 return np.array(locs[1:-1]), np.array(labels[1:-1])
771 else:
772 return np.zeros((0, 2), dtype=int), np.zeros((0, 2), dtype=object)
775default_container_samplekeys = ['rate', 'fs', 'samplerate', 'samplingrate']
776""" Lower-case keys that specify a sampling rate in a data container. """
778default_container_timekeys = ['time', 'timescale']
779""" Lower-case keys that specify a time axis in a data container. """
781default_container_amplkeys = ['amax', 'ampl_max', 'ampl', 'amplitude']
782""" Lower-case keys that specify a maximum amplitude in a data container. """
784default_container_unitkeys = ['unit']
785""" Lower-case keys that specify a unit in a data container. """
787default_container_metadatakeys = ['metadata', 'info']
788""" Lower-case keys that specify metadata in a data container. """
790default_container_poskeys = ['positions']
791""" Lower-case keys that specify event positions in a data container. """
793default_container_spankeys = ['spans']
794""" Lower-case keys that specify event spans in a data container. """
796default_container_labelkeys = ['labels']
797""" Lower-case keys that specify labels in a data container. """
799default_container_descrkeys = ['descriptions']
800""" Lower-case keys that specify descriptions in a data container. """
803def check_container(filepath):
804 """Check if file is a generic container file.
806 Supported file formats are:
808 - python pickle files (.pkl)
809 - numpy files (.npz)
810 - matlab files (.mat)
812 Parameters
813 ----------
814 filepath: str or Path
815 Path of the file to check.
817 Returns
818 -------
819 is_container: bool
820 `True`, if `filepath` is a supported container format.
821 """
822 ext = Path(filepath).suffix
823 return ext.lower() in ('.pkl', '.npz', '.mat')
826def extract_container_data(data_dict, datakey=None,
827 samplekey=default_container_samplekeys,
828 timekey=default_container_timekeys,
829 amplkey=default_container_amplkeys,
830 unitkey=default_container_unitkeys,
831 amax=1.0, unit='a.u.'):
832 """Extract data from dictionary loaded from a container file.
834 Parameters
835 ----------
836 data_dict: dict
837 Dictionary of the data items contained in the container.
838 datakey: None, str, or list of str
839 Name of the variable holding the data. If `None` take the
840 variable that is an 2D array and has the largest number of
841 elements.
842 samplekey: str or list of str
843 Name of the variable holding the sampling rate.
844 timekey: str or list of str
845 Name of the variable holding sampling times.
846 If no sampling rate is available, the sampling rate is retrieved
847 from the sampling times.
848 amplkey: str or list of str
849 Name of the variable holding the amplitude range of the data.
850 unitkey: str or list of str
851 Name of the variable holding the unit of the data.
852 amax: None or float
853 If specified and no amplitude range has been found in `data_dict`,
854 then this is the amplitude range of the data.
855 unit: None or str
856 If specified and no unit has been found in `data_dict`,
857 then return this as the unit of the data.
859 Returns
860 -------
861 data: 2-D array of floats
862 All data traces as an 2-D numpy array, even for single channel data.
863 First dimension is time, second is channel.
864 rate: float
865 Sampling rate of the data in Hz.
866 unit: str
867 Unit of the data.
868 amax: float
869 Maximum amplitude of data range in `unit`.
871 Raises
872 ------
873 ValueError:
874 Invalid key requested.
875 """
876 # extract format data:
877 if not isinstance(samplekey, (list, tuple, np.ndarray)):
878 samplekey = (samplekey,)
879 if not isinstance(timekey, (list, tuple, np.ndarray)):
880 timekey = (timekey,)
881 if not isinstance(amplkey, (list, tuple, np.ndarray)):
882 amplkey = (amplkey,)
883 if not isinstance(unitkey, (list, tuple, np.ndarray)):
884 unitkey = (unitkey,)
885 lower_keys = {k.lower():k for k in data_dict}
886 rate = 0.0
887 for skey in samplekey:
888 if skey in lower_keys:
889 rate = float(data_dict[lower_keys[skey]])
890 break
891 if rate == 0.0:
892 for tkey in timekey:
893 if tkey in lower_keys:
894 time = data_dict[lower_keys[tkey]]
895 rate = 1.0/(time[1] - time[0])
896 break
897 if rate == 0.0:
898 raise ValueError(f"invalid keys {', '.join(samplekey)} and {', '.join(timekey)} for requesting sampling rate or sampling times! Available keys in container: {', '.join(lower_keys)}")
899 for akey in amplkey:
900 if akey in lower_keys:
901 amax = float(data_dict[lower_keys[akey]])
902 break
903 for ukey in unitkey:
904 if ukey in lower_keys:
905 unit = data_dict[lower_keys[ukey]]
906 break
907 # get data array:
908 raw_data = np.array([])
909 if datakey:
910 # try data keys:
911 if not isinstance(datakey, (list, tuple, np.ndarray)):
912 datakey = (datakey,)
913 for dkey in datakey:
914 if dkey in lower_keys:
915 raw_data = data_dict[lower_keys[dkey]]
916 break
917 if len(raw_data) == 0:
918 raise ValueError(f"invalid key(s) {', '.join(datakey)} for requesting data! Available keys in container: {', '.join(lower_keys)}")
919 else:
920 # find largest 2D array:
921 for d in data_dict:
922 if hasattr(data_dict[d], 'shape'):
923 if 1 <= len(data_dict[d].shape) <= 2 and \
924 np.max(data_dict[d].shape) > np.max(raw_data.shape):
925 raw_data = data_dict[d]
926 if len(raw_data) == 0:
927 raise ValueError('no data found in container')
928 # make 2D:
929 if len(raw_data.shape) == 1:
930 raw_data = raw_data.reshape(-1, 1)
931 # transpose if necessary:
932 if np.argmax(raw_data.shape) > 0:
933 raw_data = raw_data.T
934 # recode:
935 if raw_data.dtype == np.dtype('int16'):
936 data = raw_data.astype('float32')
937 data *= amax/2**15
938 elif raw_data.dtype == np.dtype('int32'):
939 data = raw_data.astype(float)
940 data *= amax/2**31
941 elif raw_data.dtype == np.dtype('int64'):
942 data = raw_data.astype(float)
943 data *= amax/2**63
944 else:
945 data = raw_data
946 return data, rate, unit, amax
949def load_container(filepath, datakey=None,
950 samplekey=default_container_samplekeys,
951 timekey=default_container_timekeys,
952 amplkey=default_container_amplkeys,
953 unitkey=default_container_unitkeys,
954 amax=1.0, unit='a.u.'):
955 """Load data from a generic container file.
957 Supported file formats are:
959 - python pickle files (.pkl)
960 - numpy files (.npz)
961 - matlab files (.mat)
963 Parameters
964 ----------
965 filepath: str or Path
966 Path of the file to load.
967 datakey: None, str, or list of str
968 Name of the variable holding the data. If `None` take the
969 variable that is an 2D array and has the largest number of
970 elements.
971 samplekey: str or list of str
972 Name of the variable holding the sampling rate.
973 timekey: str or list of str
974 Name of the variable holding sampling times.
975 If no sampling rate is available, the sampling rate is retrieved
976 from the sampling times.
977 amplkey: str
978 Name of the variable holding the amplitude range of the data.
979 unitkey: str or list of str
980 Name of the variable holding the unit of the data.
981 amax: None or float
982 If specified and no amplitude range has been found in the data
983 container, then this is the amplitude range of the data.
984 unit: None or str
985 If specified and no unit has been found in the data container,
986 then return this as the unit of the data.
988 Returns
989 -------
990 data: 2-D array of floats
991 All data traces as an 2-D numpy array, even for single channel data.
992 First dimension is time, second is channel.
993 rate: float
994 Sampling rate of the data in Hz.
995 unit: str
996 Unit of the data.
997 amax: float
998 Maximum amplitude of data range.
1000 Raises
1001 ------
1002 ValueError:
1003 Invalid key requested.
1004 """
1005 # load data:
1006 data_dict = {}
1007 filepath = Path(filepath)
1008 ext = filepath.suffix.lower()
1009 if ext == '.pkl':
1010 with open(filepath, 'rb') as f:
1011 data_dict = pickle.load(f)
1012 elif ext == '.npz':
1013 data_dict = np.load(filepath)
1014 elif ext == '.mat':
1015 from scipy.io import loadmat
1016 data_dict = loadmat(filepath, squeeze_me=True)
1017 return extract_container_data(data_dict, datakey, samplekey,
1018 timekey, amplkey, unitkey, amax, unit)
1021def extract_container_metadata(data_dict,
1022 metadatakey=default_container_metadatakeys):
1023 """ Extract metadata from dictionary loaded from a container file.
1025 Parameters
1026 ----------
1027 data_dict: dict
1028 Dictionary of the data items contained in the container.
1029 metadatakey: str or list of str
1030 Name of the variable holding the metadata.
1032 Returns
1033 -------
1034 metadata: nested dict
1035 Nested dictionary with key-value pairs of the meta data.
1036 """
1037 if not isinstance(metadatakey, (list, tuple, np.ndarray)):
1038 metadatakey = (metadatakey,)
1039 lower_keys = {k.lower():k for k in data_dict}
1040 # get single metadata dictionary:
1041 for mkey in metadatakey:
1042 if mkey in lower_keys:
1043 return data_dict[lower_keys[mkey]]
1044 # collect all keys starting with metadatakey:
1045 metadata = {}
1046 for mkey in metadatakey:
1047 mkey += '__'
1048 for dkey in data_dict:
1049 if dkey[:len(mkey)].lower() == mkey:
1050 v = data_dict[dkey]
1051 if hasattr(v, 'size') and v.ndim == 0:
1052 v = v.item()
1053 metadata[dkey[len(mkey):]] = v
1054 if len(metadata) > 0:
1055 return unflatten_metadata(metadata, sep='__')
1056 return metadata
1059def metadata_container(filepath, metadatakey=default_container_metadatakeys):
1060 """ Read meta-data of a container file.
1062 Parameters
1063 ----------
1064 filepath: str or Path
1065 A container file.
1066 metadatakey: str or list of str
1067 Name of the variable holding the metadata.
1069 Returns
1070 -------
1071 metadata: nested dict
1072 Nested dictionary with key-value pairs of the meta data.
1073 """
1074 data_dict = {}
1075 filepath = Path(filepath)
1076 ext = filepath.suffix.lower()
1077 if ext == '.pkl':
1078 with open(filepath, 'rb') as f:
1079 data_dict = pickle.load(f)
1080 elif ext == '.npz':
1081 data_dict = np.load(filepath)
1082 elif ext == '.mat':
1083 from scipy.io import loadmat
1084 data_dict = loadmat(filepath, squeeze_me=True)
1085 return extract_container_metadata(data_dict, metadatakey)
1088def extract_container_markers(data_dict, poskey=default_container_poskeys,
1089 spankey=default_container_spankeys,
1090 labelkey=default_container_labelkeys,
1091 descrkey=default_container_descrkeys):
1092 """ Extract markers from dictionary loaded from a container file.
1094 Parameters
1095 ----------
1096 data_dict: dict
1097 Dictionary of the data items contained in the container.
1098 poskey: str or list of str
1099 Name of the variable holding positions of markers.
1100 spankey: str or list of str
1101 Name of the variable holding spans of markers.
1102 labelkey: str or list of str
1103 Name of the variable holding labels of markers.
1104 descrkey: str or list of str
1105 Name of the variable holding descriptions of markers.
1107 Returns
1108 -------
1109 locs: 2-D array of ints
1110 Marker positions (first column) and spans (second column)
1111 for each marker (rows).
1112 labels: 2-D array of string objects
1113 Labels (first column) and texts (second column)
1114 for each marker (rows).
1115 """
1116 if not isinstance(poskey, (list, tuple, np.ndarray)):
1117 poskey = (poskey,)
1118 if not isinstance(spankey, (list, tuple, np.ndarray)):
1119 spankey = (spankey,)
1120 if not isinstance(labelkey, (list, tuple, np.ndarray)):
1121 labelkey = (labelkey,)
1122 if not isinstance(descrkey, (list, tuple, np.ndarray)):
1123 descrkey = (descrkey,)
1124 lower_keys = {k.lower():k for k in data_dict}
1125 locs = np.zeros((0, 2), dtype=int)
1126 for pkey in poskey:
1127 if pkey in lower_keys:
1128 loc_data = data_dict[lower_keys[pkey]]
1129 locs = np.zeros((len(loc_data), 2), dtype=int)
1130 locs[:, 0] = loc_data
1131 break
1132 for skey in spankey:
1133 if skey in lower_keys:
1134 locs[:,1] = data_dict[lower_keys[skey]]
1135 break
1136 labels = np.zeros((0, 2), dtype=object)
1137 for lkey in labelkey:
1138 if lkey in lower_keys:
1139 label_data = data_dict[lower_keys[lkey]]
1140 labels = np.zeros((len(label_data), 2), dtype=object)
1141 labels[:,0] = label_data
1142 break
1143 for dkey in descrkey:
1144 if dkey in lower_keys:
1145 labels[:,1] = data_dict[lower_keys[dkey]]
1146 break
1147 return locs, labels
1150def markers_container(filepath, poskey=default_container_poskeys,
1151 spankey=default_container_spankeys,
1152 labelkey=default_container_labelkeys,
1153 descrkey=default_container_descrkeys):
1154 """ Read markers of a container file.
1156 Parameters
1157 ----------
1158 filepath: str or Path
1159 A container file.
1160 poskey: str or list of str
1161 Name of the variable holding positions of markers.
1162 spankey: str or list of str
1163 Name of the variable holding spans of markers.
1164 labelkey: str or list of str
1165 Name of the variable holding labels of markers.
1166 descrkey: str or list of str
1167 Name of the variable holding descriptions of markers.
1169 Returns
1170 -------
1171 locs: 2-D array of ints
1172 Marker positions (first column) and spans (second column)
1173 for each marker (rows).
1174 labels: 2-D array of string objects
1175 Labels (first column) and texts (second column)
1176 for each marker (rows).
1177 """
1178 data_dict = {}
1179 filepath = Path(filepath)
1180 ext = filepath.suffix.lower()
1181 if ext == '.pkl':
1182 with open(filepath, 'rb') as f:
1183 data_dict = pickle.load(f)
1184 elif ext == '.npz':
1185 data_dict = np.load(filepath)
1186 elif ext == '.mat':
1187 from scipy.io import loadmat
1188 data_dict = loadmat(filepath, squeeze_me=True)
1189 return extract_container_markers(data_dict, poskey, spankey,
1190 labelkey, descrkey)
1193def check_raw(filepath):
1194 """Check if file is a raw file.
1196 The following extensions are interpreted as raw files:
1198 - raw files (*.raw)
1199 - LabView scandata (*.scandat)
1201 Parameters
1202 ----------
1203 filepath: str or Path
1204 Path of the file to check.
1206 Returns
1207 -------
1208 is_raw: bool
1209 `True`, if `filepath` is a raw format.
1210 """
1211 ext = Path(filepath).suffix
1212 return ext.lower() in ('.raw', '.scandat')
1215def load_raw(filepath, rate=44000, channels=1, encoding='FLOAT',
1216 amax=1.0, unit='a.u.'):
1217 """Load data from a raw file.
1219 Raw files just contain the data and absolutely no metadata, not
1220 even the sampling rate, number of channels, etc.
1221 Supported file formats are:
1223 - raw files (*.raw)
1224 - LabView scandata (*.scandat)
1226 Parameters
1227 ----------
1228 filepath: str or Path
1229 Path of the file to load.
1230 rate: float
1231 Sampling rate of the data in Hertz.
1232 channels: int
1233 Number of channels multiplexed in the data.
1234 encoding: str
1235 The encoding of the data stored in the file.
1236 Valid encodings are 'PCM_16', 'PCM_32', 'PCM_64', 'FLOAT', or
1237 'DOUBLE' or lower-case versions thereof.
1238 amax: float
1239 The amplitude range of the data.
1240 unit: str
1241 The unit of the data.
1243 Returns
1244 -------
1245 data: 2-D array of floats
1246 All data traces as an 2-D numpy array, even for single channel data.
1247 First dimension is time, second is channel.
1248 rate: float
1249 Sampling rate of the data in Hz.
1250 unit: str
1251 Unit of the data.
1252 amax: float
1253 Maximum amplitude of data range.
1255 Raises
1256 ------
1257 ValueError:
1258 Invalid encoding.
1260 """
1261 encodings = {'PCM_16': 'i2',
1262 'PCM_32': 'i4',
1263 'PCM_64': 'i8',
1264 'FLOAT': 'f',
1265 'DOUBLE': 'd'}
1266 encoding = encoding.upper()
1267 if not encoding in encodings:
1268 raise ValueError(f'invalid encoding {encoding} for raw file!')
1269 dtype = np.dtype(encodings[encoding])
1270 raw_data = np.fromfile(filepath, dtype=dtype).reshape(-1, channels)
1271 # recode:
1272 if dtype == np.dtype('int16'):
1273 data = raw_data.astype('float32')
1274 data *= amax/2**15
1275 elif dtype == np.dtype('int32'):
1276 data = raw_data.astype(float)
1277 data *= amax/2**31
1278 elif dtype == np.dtype('int64'):
1279 data = raw_data.astype(float)
1280 data *= amax/2**63
1281 else:
1282 data = raw_data
1283 return data, rate, unit, amax
1286def load_audioio(filepath, verbose=0, gainkey=default_gain_keys, sep='.',
1287 amax=1.0, unit='a.u.'):
1288 """Load data from an audio file.
1290 See the
1291 [`load_audio()`](https://bendalab.github.io/audioio/api/audioloader.html#audioio.audioloader.load_audio)
1292 function of the [`audioio`](https://github.com/bendalab/audioio)
1293 package for more infos.
1295 Parameters
1296 ----------
1297 filepath: str or Path
1298 Path of the file to load.
1299 verbose: int
1300 If > 0 show detailed error/warning messages.
1301 gainkey: str or list of str
1302 Key in the file's metadata that holds some gain information.
1303 If found, the data will be multiplied with the gain,
1304 and if available, the corresponding unit is returned.
1305 See the [audioio.get_gain()](https://bendalab.github.io/audioio/api/audiometadata.html#audioio.audiometadata.get_gain) function for details.
1306 sep: str
1307 String that separates section names in `gainkey`.
1308 amax: float
1309 If specified and no gain has been found in the metadata,
1310 then use this as the amplitude range.
1311 unit: str
1312 If specified and no gain has been found in the metadata,
1313 then return this as the unit of the data.
1315 Returns
1316 -------
1317 data: 2-D array of floats
1318 All data traces as an 2-D numpy array, even for single channel data.
1319 First dimension is time, second is channel.
1320 rate: float
1321 Sampling rate of the data in Hz.
1322 unit: str
1323 Unit of the data if found in the metadata (see `gainkey`),
1324 otherwise `unit`.
1325 amax: float
1326 Maximum amplitude of data range.
1327 """
1328 # get gain:
1329 md = metadata_audioio(filepath)
1330 amax, unit = get_gain(md, gainkey, sep, amax, unit)
1331 # load data:
1332 data, rate = load_audio(filepath, verbose)
1333 if amax != 1.0:
1334 data *= amax
1335 return data, rate, unit, amax
1338data_loader_funcs = (
1339 ('relacs', check_relacs, load_relacs, metadata_relacs, None),
1340 ('fishgrid', check_fishgrid, load_fishgrid, metadata_fishgrid, markers_fishgrid),
1341 ('container', check_container, load_container, metadata_container, markers_container),
1342 ('raw', check_raw, load_raw, None, None),
1343 ('audioio', None, load_audioio, metadata_audioio, markers_audioio),
1344 )
1345"""List of implemented load functions.
1347Each element of the list is a tuple with the data format's name, its
1348check and its load function.
1350"""
1353def load_data(filepath, verbose=0, **kwargs):
1354 """Load time-series data from a file.
1356 Parameters
1357 ----------
1358 filepath: str or Path
1359 Path and name of the file to load.
1360 verbose: int
1361 If > 0 show detailed error/warning messages.
1362 **kwargs: dict
1363 Further keyword arguments that are passed on to the
1364 format specific loading functions.
1365 For example:
1366 - `amax`: the amplitude range of the data.
1367 - 'unit': the unit of the data.
1369 Returns
1370 -------
1371 data: 2-D array
1372 All data traces as an 2-D numpy array, even for single channel data.
1373 First dimension is time, second is channel.
1374 rate: float
1375 Sampling rate of the data in Hz.
1376 unit: str
1377 Unit of the data.
1378 amax: float
1379 Maximum amplitude of data range.
1380 """
1381 # load data:
1382 for name, check_file, load_file, _, _ in data_loader_funcs:
1383 if check_file is None or check_file(filepath):
1384 data, rate, unit, amax = load_file(filepath, **kwargs)
1385 if verbose > 0:
1386 print(f'loaded {name} data from file "{filepath}"')
1387 if verbose > 1:
1388 print(f' sampling rate: {rate:g} Hz')
1389 print(f' channels : {data.shape[1]}')
1390 print(f' frames : {len(data)}')
1391 print(f' range : {amax:g}{unit}')
1392 return data, rate, unit, amax
1393 return np.zeros((0, 1)), 0.0, '', 1.0
1396def metadata(filepath, **kwargs):
1397 """ Read meta-data from a data file.
1399 Parameters
1400 ----------
1401 filepath: str or Path
1402 The full path and name of the file to load. For some file
1403 formats several files can be provided in a list.
1404 **kwargs: dict
1405 Further keyword arguments that are passed on to the
1406 format specific loading functions.
1408 Returns
1409 -------
1410 meta_data: nested dict
1411 Meta data contained in the file. Keys of the nested
1412 dictionaries are always strings. If the corresponding
1413 values are dictionaries, then the key is the section name
1414 of the metadata contained in the dictionary. All other
1415 types of values are values for the respective key. In
1416 particular they are strings, or list of strings. But other
1417 simple types like ints or floats are also allowed.
1418 """
1419 # load metadata:
1420 for _, check_file, _, metadata_file, _ in data_loader_funcs:
1421 if check_file is None or check_file(filepath):
1422 if metadata_file is not None:
1423 return metadata_file(filepath, **kwargs)
1424 return {}
1427def markers(filepath):
1428 """ Read markers of a data file.
1430 Parameters
1431 ----------
1432 filepath: str or Path
1433 The data file.
1435 Returns
1436 -------
1437 locs: 2-D array of ints
1438 Marker positions (first column) and spans (second column)
1439 for each marker (rows).
1440 labels: 2-D array of string objects
1441 Labels (first column) and texts (second column)
1442 for each marker (rows).
1443 """
1444 # load markers:
1445 for _, check_file, _, _, markers_file in data_loader_funcs:
1446 if check_file is None or check_file(filepath):
1447 if markers_file is not None:
1448 return markers_file(filepath)
1449 return np.zeros((0, 2), dtype=int), np.zeros((0, 2), dtype=object)
1452class DataLoader(AudioLoader):
1453 """Buffered reading of time-series data for random access of the data in the file.
1455 This allows for reading very large data files that do not fit into
1456 memory. A `DataLoader` instance can be used like a huge
1457 read-only numpy array, i.e.
1458 ```
1459 data = DataLoader('path/to/data/file.dat')
1460 x = data[10000:20000,0]
1461 ```
1462 The first index specifies the frame, the second one the channel.
1464 `DataLoader` first determines the format of the data file and then
1465 opens the file (first line). It then reads data from the file as
1466 necessary for the requested data (second line).
1468 Supported file formats are
1470 - python pickle files
1471 - numpy .npz files
1472 - matlab .mat files
1473 - audio files via [`audioio`](https://github.com/bendalab/audioio) package
1474 - LabView .scandat files
1475 - raw files
1476 - relacs files (https://www.relacs.net)
1477 - fishgrid files (https://github.com/bendalab/fishgrid)
1479 Reading sequentially through the file is always possible. If
1480 previous data are requested, then the file is read from the
1481 beginning. This might slow down access to previous data
1482 considerably. Use the `backsize` argument to the open functions to
1483 make sure some data are loaded before the requested frame. Then a
1484 subsequent access to the data within `backsize` seconds before that
1485 frame can still be handled without the need to reread the file
1486 from the beginning.
1488 Usage:
1489 ------
1490 ```
1491 import thunderlab.dataloader as dl
1492 with dl.DataLoader(filepath, 60.0, 10.0) as data:
1493 # do something with the content of the file:
1494 x = data[0:10000,0]
1495 y = data[10000:20000,0]
1496 z = x + y
1497 ```
1499 Normal open and close:
1500 ```
1501 data = dl.DataLoader(filepath, 60.0)
1502 x = data[:,:] # read the whole file
1503 data.close()
1504 ```
1505 that is the same as:
1506 ```
1507 data = dl.DataLoader()
1508 data.open(filepath, 60.0)
1509 ```
1511 Parameters
1512 ----------
1513 filepath: str or Path
1514 Path of the data file.
1515 buffersize: float
1516 Size of internal buffer in seconds.
1517 backsize: float
1518 Part of the buffer to be loaded before the requested start index in seconds.
1519 verbose: int
1520 If larger than zero show detailed error/warning messages.
1521 meta_kwargs: dict
1522 Keyword arguments that are passed on to the _load_metadata() function.
1523 **kwargs: dict
1524 Further keyword arguments that are passed on to the
1525 specific open() functions.
1527 Attributes
1528 ----------
1529 filepath: Path
1530 Name and path of the opened file. In case of many files, the first one.
1531 file_paths: list of Path
1532 List of pathes of the opened files that are made accessible
1533 as a single array.
1534 file_indices: list of int
1535 For each file the index of its first sample.
1536 rate: float
1537 The sampling rate of the data in Hertz.
1538 channels: int
1539 The number of channels that are read in.
1540 frames: int
1541 The number of frames in the file.
1542 format: str or None
1543 Format of the audio file.
1544 encoding: str or None
1545 Encoding/subtype of the audio file.
1546 shape: tuple
1547 Number of frames and channels of the data.
1548 ndim: int
1549 Number of dimensions: always 2 (frames and channels).
1550 offset: int
1551 Index of first frame in the current buffer.
1552 buffer: ndarray of floats
1553 The curently available data from the file.
1554 unit: str
1555 Unit of the data.
1556 ampl_min: float
1557 Minimum amplitude the file format supports.
1558 ampl_max: float
1559 Maximum amplitude the file format supports.
1561 Methods
1562 -------
1564 - `len()`: the number of frames
1565 - `open()`: open a data file.
1566 - `open_*()`: open a data file of a specific format.
1567 - `close()`: close the file.
1568 - `basename()`: Base name of the audio data.
1569 - `format_dict()`: technical infos about how the data are stored.
1570 - `metadata()`: metadata of the file.
1571 - `markers()`: markers of the file.
1572 - `set_unwrap()`: Set parameters for unwrapping clipped data.
1574 See audioio.audioloader.AudioLoader for more methods.
1576 """
1578 def __init__(self, filepath=None, buffersize=10.0, backsize=0.0,
1579 verbose=0, meta_kwargs={}, **kwargs):
1580 super().__init__(None, buffersize, backsize,
1581 verbose, meta_kwargs)
1582 if filepath is not None:
1583 self.open(filepath, buffersize, backsize, verbose, **kwargs)
1585 def __getitem__(self, key):
1586 return super(DataLoader, self).__getitem__(key)
1588 def __next__(self):
1589 return super(DataLoader, self).__next__()
1592 # relacs interface:
1593 def open_relacs(self, filepath, buffersize=10.0, backsize=0.0,
1594 verbose=0, amax=1.0):
1595 """Open relacs data files (www.relacs.net) for reading.
1597 Parameters
1598 ----------
1599 filepath: str
1600 Path to a relacs data directory or a file therein.
1601 buffersize: float
1602 Size of internal buffer in seconds.
1603 backsize: float
1604 Part of the buffer to be loaded before the requested start index in seconds.
1605 verbose: int
1606 If > 0 show detailed error/warning messages.
1607 amax: float
1608 The amplitude range of the data.
1610 Raises
1611 ------
1612 FileNotFoundError:
1613 Invalid or non existing fishgrid files.
1614 ValueError:
1615 .gz files not supported.
1616 """
1617 self.verbose = verbose
1619 # open trace files:
1620 filepath = Path(filepath)
1621 self.trace_filepaths = relacs_trace_files(filepath)
1622 if len(self.trace_filepaths) == 0:
1623 raise FileNotFoundError('no relacs files found')
1624 self.sf = []
1625 self.frames = None
1626 self.rate = None
1627 self.unit = ''
1628 self.filepath = filepath
1629 self.file_paths = [self.filepath]
1630 self.file_indices = [0]
1631 for path in self.trace_filepaths:
1632 if path.suffix == '.gz':
1633 raise ValueError('.gz files not supported')
1634 sf = open(path, 'rb')
1635 self.sf.append(sf)
1636 if self.verbose > 0:
1637 print(f'open_relacs("{path}")')
1638 # file size:
1639 sf.seek(0, os.SEEK_END)
1640 frames = sf.tell()//4
1641 if self.frames is None:
1642 self.frames = frames
1643 elif self.frames != frames:
1644 diff = self.frames - frames
1645 if diff > 1 or diff < -2:
1646 raise ValueError('number of frames of traces differ')
1647 elif diff >= 0:
1648 self.frames = frames
1649 sf.seek(0)
1650 # retrieve sampling rate and unit:
1651 rate, us = relacs_samplerate_unit(path)
1652 if self.rate is None:
1653 self.rate = rate
1654 elif rate != self.rate:
1655 raise ValueError('sampling rates of traces differ')
1656 if len(self.unit) == 0:
1657 self.unit = us
1658 elif us != self.unit:
1659 raise ValueError('unit of traces differ')
1660 self.channels = len(self.sf)
1661 self.shape = (self.frames, self.channels)
1662 self.size = self.frames * self.channels
1663 self.ndim = len(self.shape)
1664 self.format = 'RELACS'
1665 self.encoding = 'FLOAT'
1666 self.bufferframes = int(buffersize*self.rate)
1667 self.backframes = int(backsize*self.rate)
1668 self.init_buffer()
1669 self.offset = 0
1670 self.close = self._close_relacs
1671 self.load_audio_buffer = self._load_buffer_relacs
1672 self.basename = self._basename_relacs
1673 self.ampl_min = -amax
1674 self.ampl_max = +amax
1675 self._load_metadata = metadata_relacs
1676 # TODO: load markers:
1677 self._locs = np.zeros((0, 2), dtype=int)
1678 self._labels = np.zeros((0, 2), dtype=object)
1679 self._load_markers = None
1680 return self
1682 def _close_relacs(self):
1683 """Close the relacs data files.
1684 """
1685 for f in self.sf:
1686 f.close()
1687 self.sf = []
1689 def _load_buffer_relacs(self, r_offset, r_size, buffer):
1690 """Load new data from relacs data file.
1692 Parameters
1693 ----------
1694 r_offset: int
1695 First frame to be read from file.
1696 r_size: int
1697 Number of frames to be read from file.
1698 buffer: ndarray
1699 Buffer where to store the loaded data.
1700 """
1701 if len(self.sf) == 0 and len(self.trace_filepaths) > 0:
1702 for path in self.trace_filepaths:
1703 self.sf.append(open(path, 'rb'))
1704 for i, f in enumerate(self.sf):
1705 f.seek(r_offset*4)
1706 data = f.read(r_size*4)
1707 buffer[:, i] = np.frombuffer(data, dtype=np.float32)
1710 def _basename_relacs(self, path=None):
1711 """ Base name of the relacs data files.
1713 Parameters
1714 ----------
1715 path: str or None
1716 Path of a relacs data file (*.raw, info.dat, or just the directory).
1717 If `None`, use `self.filepath`.
1719 Returns
1720 -------
1721 s: str
1722 The base name, i.e. the name of the directory containing the
1723 relacs data files.
1725 """
1726 if path is None:
1727 path = self.filepath
1728 else:
1729 path = Path(path)
1730 if path.is_dir():
1731 return path.name
1732 else:
1733 return path.parent.name
1736 # fishgrid interface:
1737 def open_fishgrid(self, filepath, buffersize=10.0, backsize=0.0,
1738 verbose=0):
1739 """Open fishgrid data files (https://github.com/bendalab/fishgrid) for reading.
1741 Parameters
1742 ----------
1743 filepath: str
1744 Path to a fishgrid data directory, or a file therein.
1745 buffersize: float
1746 Size of internal buffer in seconds.
1747 backsize: float
1748 Part of the buffer to be loaded before the requested start index in seconds.
1749 verbose: int
1750 If > 0 show detailed error/warning messages.
1752 Raises
1753 ------
1754 FileNotFoundError:
1755 Invalid or non existing fishgrid files.
1756 ValueError:
1757 .gz files not supported.
1758 """
1759 self.verbose = verbose
1761 filepath = Path(filepath)
1762 self.trace_filepaths = fishgrid_trace_files(filepath)
1763 if len(self.trace_filepaths) == 0:
1764 raise FileNotFoundError(f'no fishgrid files found')
1765 self.filepath = filepath
1766 self.file_paths = [self.filepath]
1767 self.file_indices = [0]
1768 self._load_metadata = metadata_fishgrid
1769 self._load_markers = markers_fishgrid
1771 # open grid files:
1772 grids = fishgrid_grids(self.metadata())
1773 grid_sizes = [r*c for r,c in grids]
1774 self.channels = 0
1775 for g, path in enumerate(self.trace_filepaths):
1776 self.channels += grid_sizes[g]
1777 self.sf = []
1778 self.grid_channels = []
1779 self.grid_offs = []
1780 offs = 0
1781 self.frames = None
1782 self.rate = get_number(self.metadata(), 'Hz', 'AISampleRate')
1783 v, self.unit = get_number_unit(self.metadata(), 'AIMaxVolt')
1784 if v is not None:
1785 self.ampl_min = -v
1786 self.ampl_max = +v
1788 for g, path in enumerate(self.trace_filepaths):
1789 if path.suffix == '.gz':
1790 raise ValueError('.gz files not supported')
1791 sf = open(path, 'rb')
1792 self.sf.append(sf)
1793 if self.verbose > 0:
1794 print(f'open_fishgrid("{path}")')
1795 # grid channels:
1796 self.grid_channels.append(grid_sizes[g])
1797 self.grid_offs.append(offs)
1798 offs += grid_sizes[g]
1799 # file size:
1800 sf.seek(0, os.SEEK_END)
1801 frames = sf.tell()//4//grid_sizes[g]
1802 if self.frames is None:
1803 self.frames = frames
1804 elif self.frames != frames:
1805 diff = self.frames - frames
1806 if diff > 1 or diff < -2:
1807 raise ValueError('number of frames of traces differ')
1808 elif diff >= 0:
1809 self.frames = frames
1810 sf.seek(0)
1811 self.shape = (self.frames, self.channels)
1812 self.size = self.frames * self.channels
1813 self.ndim = len(self.shape)
1814 self.format = 'FISHGRID'
1815 self.encoding = 'FLOAT'
1816 self.bufferframes = int(buffersize*self.rate)
1817 self.backframes = int(backsize*self.rate)
1818 self.init_buffer()
1819 self.offset = 0
1820 self.close = self._close_fishgrid
1821 self.load_audio_buffer = self._load_buffer_fishgrid
1822 self.basename = self._basename_fishgrid
1823 return self
1825 def _close_fishgrid(self):
1826 """Close the fishgrid data files.
1827 """
1828 for file in self.sf:
1829 file.close()
1830 self.sf = []
1832 def _load_buffer_fishgrid(self, r_offset, r_size, buffer):
1833 """Load new data from relacs data file.
1835 Parameters
1836 ----------
1837 r_offset: int
1838 First frame to be read from file.
1839 r_size: int
1840 Number of frames to be read from file.
1841 buffer: ndarray
1842 Buffer where to store the loaded data.
1843 """
1844 if len(self.sf) == 0 and len(self.trace_filepaths) > 0:
1845 for path in self.trace_filepaths:
1846 self.sf.append(open(path, 'rb'))
1847 for file, gchannels, goffset in zip(self.sf, self.grid_channels, self.grid_offs):
1848 file.seek(r_offset*4*gchannels)
1849 data = file.read(r_size*4*gchannels)
1850 buffer[:, goffset:goffset+gchannels] = np.frombuffer(data, dtype=np.float32).reshape((-1, gchannels))
1852 def _basename_fishgrid(self, path=None):
1853 """ Base name of the fishgrid data files.
1855 Parameters
1856 ----------
1857 path: str or Path or None
1858 Path of a fishgrid data file
1859 (*.raw, fishgrid.cfg, or just the directory).
1860 If `None`, use `self.filepath`.
1862 Returns
1863 -------
1864 s: str
1865 The base name, i.e. the name of the directory containing the
1866 fishgrid data files.
1868 """
1869 if path is None:
1870 path = self.filepath
1871 else:
1872 path = Path(path)
1873 if path.is_dir():
1874 return path.name
1875 else:
1876 return path.parent.name
1880 # container interface:
1881 def open_container(self, filepath, buffersize=10.0,
1882 backsize=0.0, verbose=0, datakey=None,
1883 samplekey=default_container_samplekeys,
1884 timekey=default_container_timekeys,
1885 amplkey=default_container_amplkeys,
1886 unitkey=default_container_unitkeys,
1887 metadatakey=default_container_metadatakeys,
1888 poskey=default_container_poskeys,
1889 spankey=default_container_spankeys,
1890 labelkey=default_container_labelkeys,
1891 descrkey=default_container_descrkeys,
1892 amax=1.0, unit='a.u.'):
1893 """Open generic container file.
1895 Supported file formats are:
1897 - python pickle files (.pkl)
1898 - numpy files (.npz)
1899 - matlab files (.mat)
1901 Parameters
1902 ----------
1903 filepath: str
1904 Path to a container file.
1905 buffersize: float
1906 Size of internal buffer in seconds.
1907 backsize: float
1908 Part of the buffer to be loaded before the requested start index in seconds.
1909 verbose: int
1910 If > 0 show detailed error/warning messages.
1911 datakey: None, str, or list of str
1912 Name of the variable holding the data. If `None` take the
1913 variable that is an 2D array and has the largest number of
1914 elements.
1915 samplekey: str or list of str
1916 Name of the variable holding the sampling rate.
1917 timekey: str or list of str
1918 Name of the variable holding sampling times.
1919 If no sampling rate is available, the sampling rate is retrieved
1920 from the sampling times.
1921 amplkey: str or list of str
1922 Name of the variable holding the amplitude range of the data.
1923 unitkey: str or list of str
1924 Name of the variable holding the unit of the data.
1925 metadatakey: str or list of str
1926 Name of the variable holding the metadata.
1927 poskey: str or list of str
1928 Name of the variable holding positions of markers.
1929 spankey: str or list of str
1930 Name of the variable holding spans of markers.
1931 labelkey: str or list of str
1932 Name of the variable holding labels of markers.
1933 descrkey: str or list of str
1934 Name of the variable holding descriptions of markers.
1935 amax: None or float
1936 If specified and no amplitude range has been found in the data
1937 container, then this is the amplitude range of the data.
1938 unit: None or str
1939 If specified and no unit has been found in the data container,
1940 then return this as the unit of the data.
1942 Raises
1943 ------
1944 ValueError:
1945 Invalid key requested.
1946 """
1947 self.verbose = verbose
1948 data_dict = {}
1949 filepath = Path(filepath)
1950 ext = filepath.suffix.lower()
1951 if ext == '.pkl':
1952 with open(filepath, 'rb') as f:
1953 data_dict = pickle.load(f)
1954 self.format = 'PKL'
1955 elif ext == '.npz':
1956 data_dict = np.load(filepath)
1957 self.format = 'NPZ'
1958 elif ext == '.mat':
1959 from scipy.io import loadmat
1960 data_dict = loadmat(filepath, squeeze_me=True)
1961 self.format = 'MAT'
1962 if self.verbose > 0:
1963 print(f'open_container("{filepath}")')
1964 self.buffer, self.rate, self.unit, amax = \
1965 extract_container_data(data_dict, datakey, samplekey,
1966 timekey, amplkey, unitkey, amax, unit)
1967 self.filepath = filepath
1968 self.file_paths = [self.filepath]
1969 self.file_indices = [0]
1970 self.channels = self.buffer.shape[1]
1971 self.frames = self.buffer.shape[0]
1972 self.shape = self.buffer.shape
1973 self.ndim = self.buffer.ndim
1974 self.size = self.buffer.size
1975 self.encoding = self.numpy_encodings[self.buffer.dtype]
1976 self.ampl_min = -amax
1977 self.ampl_max = +amax
1978 self.offset = 0
1979 self.buffer_changed = np.zeros(self.channels, dtype=bool)
1980 self.bufferframes = self.frames
1981 self.backsize = 0
1982 self.close = self._close_container
1983 self.load_audio_buffer = self._load_buffer_container
1984 self._metadata = extract_container_metadata(data_dict, metadatakey)
1985 self._load_metadata = None
1986 self._locs, self._labels = extract_container_markers(data_dict,
1987 poskey,
1988 spankey,
1989 labelkey,
1990 descrkey)
1991 self._load_markers = None
1993 def _close_container(self):
1994 """Close container. """
1995 pass
1997 def _load_buffer_container(self, r_offset, r_size, buffer):
1998 """Load new data from container."""
1999 buffer[:, :] = self.buffer[r_offset:r_offset + r_size, :]
2002 # raw data interface:
2003 def open_raw(self, filepath, buffersize=10.0, backsize=0.0,
2004 verbose=0, rate=44000, channels=1, encoding='FLOAT',
2005 amax=1.0, unit='a.u.'):
2006 """Load data from a raw file.
2008 Raw files just contain the data and absolutely no metadata, not
2009 even the smapling rate, number of channels, etc.
2010 Supported file formats are:
2012 - raw files (*.raw)
2013 - LabView scandata (*.scandat)
2015 Parameters
2016 ----------
2017 filepath: str or Path
2018 Path of the file to load.
2019 buffersize: float
2020 Size of internal buffer in seconds.
2021 backsize: float
2022 Part of the buffer to be loaded before the requested start index in seconds.
2023 verbose: int
2024 If > 0 show detailed error/warning messages.
2025 rate: float
2026 Sampling rate of the data in Hertz.
2027 channels: int
2028 Number of channels multiplexed in the data.
2029 encoding: str
2030 The encoding of the data stored in the file.
2031 Valid encodings are 'PCM_16', 'PCM_32', 'PCM_64', 'FLOAT', or
2032 'DOUBLE' or lower-case versions thereof.
2033 amax: float
2034 The amplitude range of the data.
2035 unit: str
2036 The unit of the data.
2037 """
2038 encodings = {'PCM_16': 'i2',
2039 'PCM_32': 'i4',
2040 'PCM_64': 'i8',
2041 'FLOAT': 'f',
2042 'DOUBLE': 'd'}
2043 encoding = encoding.upper()
2044 if not encoding in encodings:
2045 raise ValueError(f'invalid encoding {encoding} for raw file!')
2046 self.dtype = np.dtype(encodings[encoding])
2047 self.verbose = verbose
2048 self.filepath = Path(filepath)
2049 self.file_paths = [self.filepath]
2050 self.file_indices = [0]
2051 self.sf = open(self.filepath, 'rb')
2052 if self.verbose > 0:
2053 print(f'open_raw("{self.filepath}")')
2054 self.rate = float(rate)
2055 # file size:
2056 self.channels = int(channels)
2057 self.sf.seek(0, os.SEEK_END)
2058 self.frames = self.sf.tell()//self.dtype.itemsize//self.channels
2059 self.sf.seek(0)
2060 self.shape = (self.frames, self.channels)
2061 self.ndim = len(self.shape)
2062 self.size = self.frames*self.channels
2063 self.format = 'RAW'
2064 self.encoding = self.numpy_encodings.get(self.dtype, 'UNKNOWN')
2065 self.unit = unit
2066 self.ampl_max = float(amax)
2067 self.ampl_min = -self.ampl_max
2068 self.offset = 0
2069 self.bufferframes = int(buffersize*self.rate)
2070 self.backframes = int(backsize*self.rate)
2071 self.init_buffer()
2072 self.close = self._close_raw
2073 self.load_audio_buffer = self._load_buffer_raw
2074 self._metadata = None
2075 self._load_metadata = None
2076 self._locs = None
2077 self._labels = None
2078 self._load_markers = None
2080 def _close_raw(self):
2081 """Close raw file. """
2082 if self.sf is not None:
2083 self.sf.close()
2084 self.sf = None
2086 def _load_buffer_raw(self, r_offset, r_size, buffer):
2087 """Load new data from container."""
2088 if self.sf is None:
2089 self.sf = open(self.filepath, 'rb')
2090 self.sf.seek(r_offset*self.dtype.itemsize*self.channels)
2091 raw_data = self.sf.read(r_size*self.dtype.itemsize*self.channels)
2092 raw_data = np.frombuffer(raw_data, dtype=self.dtype)
2093 raw_data = raw_data.reshape(-1, self.channels)
2094 # recode:
2095 if self.dtype == np.dtype('int16'):
2096 data = raw_data.astype('float32')
2097 data *= self.ampl_max/2**15
2098 elif self.dtype == np.dtype('int32'):
2099 data = raw_data.astype(float)
2100 data *= self.ampl_max/2**31
2101 elif self.dtype == np.dtype('int64'):
2102 data = raw_data.astype(float)
2103 data *= self.ampl_max/2**63
2104 else:
2105 data = raw_data
2106 buffer[:, :] = data
2109 # audioio interface:
2110 def open_audioio(self, filepath, buffersize=10.0, backsize=0.0,
2111 verbose=0, gainkey=default_gain_keys, sep='.',
2112 amax=None, unit='a.u.'):
2113 """Open an audio file.
2115 See the [audioio](https://github.com/bendalab/audioio) package
2116 for details.
2118 Parameters
2119 ----------
2120 filepath: str
2121 Path to an audio file.
2122 buffersize: float
2123 Size of internal buffer in seconds.
2124 backsize: float
2125 Part of the buffer to be loaded before the requested start index
2126 in seconds.
2127 verbose: int
2128 If > 0 show detailed error/warning messages.
2129 gainkey: str or list of str
2130 Key in the file's metadata that holds some gain information.
2131 If found, the data will be multiplied with the gain,
2132 and if available, the corresponding unit is returned.
2133 See the [audioio.get_gain()](https://bendalab.github.io/audioio/api/audiometadata.html#audioio.audiometadata.get_gain) function for details.
2134 sep: str
2135 String that separates section names in `gainkey`.
2136 amax: None or float
2137 If specified and no gain has been found in the metadata,
2138 then use this as the amplitude range.
2139 unit: None or str
2140 If specified and no gain has been found in the metadata,
2141 then this is the unit of the data.
2143 """
2144 self.verbose = verbose
2145 super(DataLoader, self).open(filepath, buffersize, backsize, verbose)
2146 md = self.metadata()
2147 fac, unit = get_gain(md, gainkey, sep, amax, unit)
2148 if fac is None:
2149 self.gain_fac = 1.0
2150 else:
2151 self.gain_fac = fac
2152 self._load_buffer_audio_org = self.load_audio_buffer
2153 self.load_audio_buffer = self._load_buffer_audioio
2154 self.ampl_min *= self.gain_fac
2155 self.ampl_max *= self.gain_fac
2156 self.unit = unit
2157 return self
2159 def _load_buffer_audioio(self, r_offset, r_size, buffer):
2160 """Load and scale new data from an audio file.
2162 Parameters
2163 ----------
2164 r_offset: int
2165 First frame to be read from file.
2166 r_size: int
2167 Number of frames to be read from file.
2168 buffer: ndarray
2169 Buffer where to store the loaded data.
2170 """
2171 self._load_buffer_audio_org(r_offset, r_size, buffer)
2172 buffer *= self.gain_fac
2175 # open multiple files as one:
2176 def open_multiple(self, filepaths, buffersize=10.0, backsize=0.0,
2177 verbose=0, mode='strict', rate=None, channels=None,
2178 unit=None, amax=None, end_indices=None):
2179 """Open multiple files as a single concatenated array.
2181 Parameters
2182 ----------
2183 filepaths: list of str or Path
2184 List of file paths of audio files.
2185 buffersize: float
2186 Size of internal buffer in seconds.
2187 backsize: float
2188 Part of the buffer to be loaded before the requested start index in seconds.
2189 verbose: int
2190 If larger than zero show detailed error/warning messages.
2191 mode: 'relaxed' or 'strict'
2192 If 'strict', only concatenate files if they contain
2193 a start time in their meta data.
2194 rate: float
2195 If provided, do a minimal initialization (no checking)
2196 using the provided sampling rate (in Hertz), channels,
2197 unit, maximum amplitude, and end_indices.
2198 channels: int
2199 If provided, do a minimal initialization (no checking)
2200 using the provided rate, number of channels,
2201 unit, maximum amplitude, and end_indices.
2202 unit: str
2203 If provided, do a minimal initialization (no checking)
2204 using the provided rate, number of channels,
2205 unit, maximum amplitude, and end_indices.
2206 amax: float
2207 If provided, do a minimal initialization (no checking)
2208 using the provided rate, number of channels,
2209 unit, maximum amplitude amax, and end_indices.
2210 end_indices: sequence of int
2211 If provided, do a minimal initialization (no checking)
2212 using the provided rate, channels,
2213 unit, maximum amplitude, and end_indices.
2215 Raises
2216 ------
2217 TypeError
2218 `filepaths` must be a sequence.
2219 ValueError
2220 Empty `filepaths`.
2221 FileNotFoundError
2222 `filepaths` does not contain a single valid file.
2224 """
2225 if not isinstance(filepaths, (list, tuple, np.ndarray)):
2226 raise TypeError('input argument filepaths is not a sequence!')
2227 if len(filepaths) == 0:
2228 raise ValueError('input argument filepaths is empy sequence!')
2229 self.buffersize = buffersize
2230 self.backsize = backsize
2231 self.filepath = None
2232 self.file_paths = []
2233 self.open_files = []
2234 self.open_loaders = []
2235 self.data_files = []
2236 self.collect_counter = 0
2237 self.frames = 0
2238 self.start_indices = []
2239 self.end_indices = []
2240 self.start_time = None
2241 start_time = None
2242 self._metadata = {}
2243 self._locs = np.zeros((0, 2), dtype=int)
2244 self._labels = np.zeros((0, 2), dtype=object)
2245 if end_indices is not None:
2246 self.file_paths = [Path(fp) for fp in filepaths]
2247 self.filepath = self.file_paths[0]
2248 self.data_files = [None] * len(self.file_paths)
2249 self.frames = end_indices[-1]
2250 self.start_indices = [0] + list(end_indices[:-1])
2251 self.end_indices = end_indices
2252 self.format = None
2253 self.encoding = None
2254 self.rate = rate
2255 self.channels = channels
2256 self.unit = unit
2257 self.ampl_max = amax
2258 self.ampl_min = -amax
2259 else:
2260 for filepath in filepaths:
2261 try:
2262 a = DataLoader(filepath, buffersize, backsize, verbose)
2263 except Exception as e:
2264 if verbose > 0:
2265 print(e)
2266 continue
2267 # collect metadata:
2268 md = a.metadata()
2269 fmd = flatten_metadata(md, True)
2270 add_metadata(self._metadata, fmd)
2271 if self.filepath is None:
2272 # first file:
2273 self.filepath = a.filepath
2274 self.format = a.format
2275 self.encoding = a.encoding
2276 self.rate = a.rate
2277 self.channels = a.channels
2278 self.unit = a.unit
2279 self.ampl_max = a.ampl_max
2280 self.ampl_min = a.ampl_min
2281 self.start_time = get_datetime(md)
2282 start_time = self.start_time
2283 stime = self.start_time
2284 else:
2285 # check channels, rate, and amplitudes:
2286 error_str = None
2287 if a.channels != self.channels:
2288 error_str = f'number of channels differs: ' \
2289 f'{a.channels} in {a.filepath} versus ' \
2290 f'{self.channels} in {self.filepath}'
2291 if a.rate != self.rate:
2292 error_str = f'sampling rates differ: ' \
2293 f'{a.rate} in {a.filepath} versus ' \
2294 f'{self.rate} in {self.filepath}'
2295 if a.ampl_min != self.ampl_min:
2296 error_str = f'minimum amplitudes differ: ' \
2297 f'{a.ampl_min} in {a.filepath} versus ' \
2298 f'{self.ampl_min} in {self.filepath}'
2299 if a.ampl_max != self.ampl_max:
2300 error_Str = f'maximum amplitudes differ: ' \
2301 f'{a.ampl_max} in {a.filepath} versus ' \
2302 f'{self.ampl_max} in {self.filepath}'
2303 # check start time of recording:
2304 stime = get_datetime(md)
2305 if mode == 'strict' and (start_time is None or stime is None):
2306 error_str = 'file does not contain a start time in its meta data'
2307 if start_time is not None and stime is not None and \
2308 abs(start_time - stime) > timedelta(seconds=self._max_time_diff):
2309 error_str = f'start time does not indicate continuous recording: ' \
2310 f'expected {start_time} instead of ' \
2311 f'{stime} in {a.filepath}'
2312 if error_str is not None:
2313 if verbose > 0:
2314 print(error_str)
2315 a.close()
2316 del a
2317 break
2318 # markers:
2319 locs, labels = a.markers()
2320 locs[:,0] += self.frames
2321 self._locs = np.vstack((self._locs, locs))
2322 self._labels = np.vstack((self._labels, labels))
2323 # indices:
2324 self.start_indices.append(self.frames)
2325 self.frames += a.frames
2326 self.end_indices.append(self.frames)
2327 if stime is not None:
2328 start_time = stime + timedelta(seconds=a.frames/a.rate)
2329 # add file to lists:
2330 self.file_paths.append(a.filepath)
2331 if len(self.open_files) < AudioLoader.max_open_files:
2332 self.open_files.append(a)
2333 else:
2334 a.close()
2335 if len(self.open_loaders) < AudioLoader.max_open_loaders:
2336 self.data_files.append(a)
2337 self.open_loaders.append(a)
2338 else:
2339 a.close()
2340 del a
2341 self.data_files.append(None)
2342 if len(self.data_files) == 0:
2343 raise FileNotFoundError('input argument filepaths does not contain any valid audio file!')
2344 # set startime from first file:
2345 if self.start_time is not None:
2346 set_starttime(self._metadata, self.start_time)
2347 # setup infrastructure:
2348 self.file_indices = self.start_indices
2349 self.start_indices = np.array(self.start_indices)
2350 self.end_indices = np.array(self.end_indices)
2351 self.shape = (self.frames, self.channels)
2352 self.bufferframes = int(buffersize*self.rate)
2353 self.backframes = int(backsize*self.rate)
2354 self.init_buffer()
2355 self.close = self._close_multiple
2356 self.load_audio_buffer = self._load_buffer_multiple
2357 self._load_metadata = None
2358 self._load_markers = None
2359 return self
2361 def _close_multiple(self):
2362 """Close all the data files. """
2363 self.open_files = []
2364 self.open_loaders = []
2365 if hasattr(self, 'data_files'):
2366 for a in self.data_files:
2367 if a is not None:
2368 a.close()
2369 self.data_files = []
2370 self.filepath = None
2371 self.file_paths = []
2372 self.file_indices = []
2373 self.start_indices = []
2374 self.end_indices = []
2375 self.ampl_min = -1.0
2376 self.ampl_max = +1.0
2377 del self.data_files
2378 del self.open_files
2379 del self.open_loaders
2380 del self.start_indices
2381 del self.end_indices
2383 def _load_buffer_multiple(self, r_offset, r_size, buffer):
2384 """Load new data from the underlying files.
2386 Parameters
2387 ----------
2388 r_offset: int
2389 First frame to be read from file.
2390 r_size: int
2391 Number of frames to be read from file.
2392 buffer: ndarray
2393 Buffer where to store the loaded data.
2394 """
2395 offs = r_offset
2396 size = r_size
2397 boffs = 0
2398 ai = np.searchsorted(self.end_indices, offs, side='right')
2399 while size > 0:
2400 if self.data_files[ai] is None:
2401 a = DataLoader(self.file_paths[ai],
2402 self.buffersize, self.backsize, 0)
2403 self.data_files[ai] = a
2404 self.open_loaders.append(a)
2405 self.open_files.append(a)
2406 if len(self.open_files) > AudioLoader.max_open_files:
2407 a0 = self.open_files.pop(0)
2408 a0.close()
2409 if len(self.open_loaders) > AudioLoader.max_open_loaders:
2410 a0 = self.open_loaders.pop(0)
2411 self.data_files[self.data_files.index(a0)] = None
2412 a0.close()
2413 del a0
2414 self.collect_counter += 1
2415 if self.collect_counter > AudioLoader.max_open_loaders//2:
2416 gc.collect() # takes time!
2417 self.collect_counter = 0
2418 else:
2419 self.open_loaders.pop(self.open_loaders.index(self.data_files[ai]))
2420 self.open_loaders.append(self.data_files[ai])
2421 ai0 = offs - self.start_indices[ai]
2422 ai1 = offs + size
2423 if ai1 > self.end_indices[ai]:
2424 ai1 = self.end_indices[ai]
2425 ai1 -= self.start_indices[ai]
2426 n = ai1 - ai0
2427 self.data_files[ai].load_audio_buffer(ai0, n,
2428 buffer[boffs:boffs + n,:])
2429 if self.data_files[ai] in self.open_files:
2430 self.open_files.pop(self.open_files.index(self.data_files[ai]))
2431 self.open_files.append(self.data_files[ai])
2432 if len(self.open_files) > AudioLoader.max_open_files:
2433 self.open_files[0].close()
2434 self.open_files.pop(0)
2435 boffs += n
2436 offs += n
2437 size -= n
2438 ai += 1
2441 def open(self, filepath, buffersize=10.0, backsize=0.0,
2442 verbose=0, **kwargs):
2443 """Open file with time-series data for reading.
2445 Parameters
2446 ----------
2447 filepath: str or list of str
2448 Name of the file or list of many file names that should be
2449 made accessible as a single array.
2450 buffersize: float
2451 Size of internal buffer in seconds.
2452 backsize: float
2453 Part of the buffer to be loaded before the requested start index
2454 in seconds.
2455 verbose: int
2456 If > 0 show detailed error/warning messages.
2457 **kwargs: dict
2458 Further keyword arguments that are passed on to the
2459 format specific opening functions.
2460 For example:
2461 - `amax`: the amplitude range of the data.
2462 - 'unit': the unit of the data.
2464 Raises
2465 ------
2466 ValueError:
2467 `filepath` is empty string.
2468 """
2469 # list of implemented open functions:
2470 data_open_funcs = (
2471 ('relacs', check_relacs, self.open_relacs, 1),
2472 ('fishgrid', check_fishgrid, self.open_fishgrid, 1),
2473 ('container', check_container, self.open_container, 1),
2474 ('raw', check_raw, self.open_raw, 1),
2475 ('audioio', None, self.open_audioio, 0),
2476 )
2478 self.buffer = np.array([])
2479 self.rate = 0.0
2480 if not filepath:
2481 raise ValueError('input argument filepath is empty string.')
2482 if isinstance(filepath, (list, tuple, np.ndarray)):
2483 if len(filepath) > 1:
2484 self.open_multiple(filepath, buffersize, backsize,
2485 verbose, **kwargs)
2486 if len(self.file_paths) > 1:
2487 return self
2488 filepath = self.file_paths[0]
2489 self.close()
2490 else:
2491 filepath = filepath[0]
2492 # open data:
2493 for name, check_file, open_file, v in data_open_funcs:
2494 if check_file is None or check_file(filepath):
2495 open_file(filepath, buffersize, backsize, verbose, **kwargs)
2496 if v*verbose > 1:
2497 if self.format is not None:
2498 print(f' format : {self.format}')
2499 if self.encoding is not None:
2500 print(f' encoding : {self.encoding}')
2501 print(f' sampling rate: {self.rate} Hz')
2502 print(f' channels : {self.channels}')
2503 print(f' frames : {self.frames}')
2504 print(f' range : {self.ampl_max:g}{self.unit}')
2505 break
2506 return self
2509def demo(filepath, plot=False):
2510 print("try load_data:")
2511 data, rate, unit, amax = load_data(filepath, verbose=2)
2512 if plot:
2513 fig, ax = plt.subplots()
2514 time = np.arange(len(data))/rate
2515 for c in range(data.shape[1]):
2516 ax.plot(time, data[:,c])
2517 ax.set_xlabel('Time [s]')
2518 ax.set_ylabel(f'[{unit}]')
2519 if amax is not None and np.isfinite(amax):
2520 ax.set_ylim(-amax, +amax)
2521 plt.show()
2522 return
2524 print('')
2525 print("try DataLoader:")
2526 with DataLoader(filepath, 2.0, 1.0, 1) as data:
2527 print('sampling rate: %g' % data.rate)
2528 print('frames : %d %d' % (len(data), data.shape[0]))
2529 nframes = int(1.0 * data.rate)
2530 # forward:
2531 for i in range(0, len(data), nframes):
2532 print('forward %d-%d' % (i, i + nframes))
2533 x = data[i:i + nframes, 0]
2534 if plot:
2535 fig, ax = plt.subplots()
2536 ax.plot((i + np.arange(len(x)))/data.rate, x)
2537 ax.set_xlabel('Time [s]')
2538 ax.set_ylabel(f'[{data.unit}]')
2539 plt.show()
2540 # and backwards:
2541 for i in reversed(range(0, len(data), nframes)):
2542 print('backward %d-%d' % (i, i + nframes))
2543 x = data[i:i + nframes, 0]
2544 if plot:
2545 fig, ax = plt.subplots()
2546 ax.plot((i + np.arange(len(x)))/data.rate, x)
2547 ax.set_xlabel('Time [s]')
2548 ax.set_ylabel(f'[{data.unit}]')
2549 plt.show()
2552def main(*cargs):
2553 """Call demo with command line arguments.
2555 Parameters
2556 ----------
2557 cargs: list of str
2558 Command line arguments as provided by sys.argv[1:]
2559 """
2560 import argparse
2561 parser = argparse.ArgumentParser(description=
2562 'Checking thunderlab.dataloader module.')
2563 parser.add_argument('-p', dest='plot', action='store_true',
2564 help='plot loaded data')
2565 parser.add_argument('file', nargs=1, default='', type=str,
2566 help='name of data file')
2567 args = parser.parse_args(cargs)
2568 demo(args.file[0], args.plot)
2571if __name__ == "__main__":
2572 main(*sys.argv[1:])