Coverage for src / thunderfish / collectfish.py: 0%

448 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-01-15 17:50 +0000

1""" 

2Collect data generated by thunderfish in a wavefish and a pulsefish table. 

3""" 

4 

5import os 

6import glob 

7import io 

8import zipfile 

9import sys 

10import argparse 

11import numpy as np 

12 

13from thunderlab.configfile import ConfigFile 

14from thunderlab.tabledata import TableData, add_write_table_config, write_table_args 

15 

16from .version import __version__, __year__ 

17from .harmonics import add_harmonic_groups_config 

18from .eodanalysis import wave_similarity, pulse_similarity 

19from .eodanalysis import load_species_waveforms, add_species_config 

20from .eodanalysis import wave_quality, wave_quality_args, add_eod_quality_config 

21from .eodanalysis import pulse_quality, pulse_quality_args 

22from .eodanalysis import adjust_eodf 

23from .eodanalysis import parse_filename 

24 

25 

26def collect_fish(files, simplify_file=False, 

27 meta_data=None, meta_recordings=None, skip_recordings=False, 

28 temp_col=None, q10=1.62, max_fish=0, harmonics=None, 

29 phases0=None, phases1=None, cfg=None, verbose=0): 

30 """Combine all *-wavefish.* and/or *-pulsefish.* files into respective summary tables. 

31 

32 Data from the *-wavespectrum-*.* and the *-pulsephases-*.* files 

33 can be added as specified by `harmonics`, `phases0`, and 

34 `phases1`. 

35 

36 Meta data of the recordings can also be added via `meta_data` and 

37 `meta_recordings`. If `meta_data` contains a column with 

38 temperature, this column can be specified by the `temp_col` 

39 parameter. In this case, an 'T_adjust' and an 'EODf_adjust' column 

40 are inserted into the resulting tables containing the mean 

41 temperature and EOD frequencies adjusted to this temperature, 

42 respectively. For the temperature adjustment of EOD frequency 

43 a Q10 value can be supplied by the `q10` parameter. 

44 

45 Parameters 

46 ---------- 

47 files: list of strings 

48 Files to be combined. 

49 simplify_file: boolean 

50 Remove initial common directories from input files. 

51 meta_data: TableData or None 

52 Table with additional data for each of the recordings. 

53 The meta data are inserted into the summary table according to 

54 the name of the recording as specified in `meta_recordings`. 

55 meta_recordings: array of strings 

56 For each row in `meta_data` the name of the recording. 

57 This name is matched agains the basename of input `files`. 

58 skip_recordings: bool 

59 If True skip recordings that are not found in `meta_recordings`. 

60 temp_col: string or None 

61 A column in `meta_data` with temperatures to which EOD 

62 frequences should be adjusted. 

63 q10: float 

64 Q10 value describing temperature dependence of EOD 

65 frequencies. The default of 1.62 is from Dunlap, Smith, Yetka 

66 (2000) Brain Behav Evol, measured for Apteronotus 

67 lepthorhynchus in the lab. 

68 max_fish: int 

69 Maximum number of fish to be taken, if 0 take all. 

70 harmonics: int 

71 Number of harmonic to be added to the wave-type fish table 

72 (amplitude, relampl, phase). This data is read in from the 

73 corresponding *-wavespectrum-*.* files. 

74 phases0: int 

75 Index of the first phase of a EOD pulse to be added to the 

76 pulse-type fish table. This data is read in from the 

77 corresponding *-pulsephases-*.* files. 

78 phases1: int 

79 Index of the last phase of a EOD pulse to be added to the 

80 pulse-type fish table. This data is read in from the 

81 corresponding *-pulsephases-*.* files. 

82 cfg: ConfigFile 

83 Configuration parameter for EOD quality assessment and species 

84 assignment. 

85 verbose: int 

86 Verbose output: 

87  

88 1: print infos on meta data coverage. 

89 2: print additional infos on discarded recordings. 

90 

91 Returns 

92 ------- 

93 wave_table: TableData 

94 Summary table for all wave-type fish. 

95 pulse_table: TableData 

96 Summary table for all pulse-type fish. 

97 all_table: TableData 

98 Summary table for all wave-type and pulse-type fish. 

99 

100 """ 

101 def file_iter(files): 

102 """ Iterate over analysis files. 

103 

104 Parameters 

105 ---------- 

106 files: list of str 

107 Input files. 

108 

109 Yields 

110 ------ 

111 zf: ZipFile or None 

112 In case an input file is a zip archive, the open archive. 

113 file_path: str 

114 The full path of a single file to be processed. 

115 I.e. a '*-wavefish.*' or '*-pulsefish.*' file. 

116 fish_type: str 

117 Either 'wave' or 'pulse'. 

118 """ 

119 for file_path in files: 

120 _, _, _, _, ftype, _, ext = parse_filename(file_path) 

121 if ext == 'zip': 

122 zf = zipfile.ZipFile(file_path) 

123 file_pathes = sorted(zf.namelist()) 

124 for zfile in file_pathes: 

125 _, _, _, _, ftype, _, _ = parse_filename(zfile) 

126 if ftype in ['wavefish', 'pulsefish']: 

127 yield zf, zfile, ftype[:-4] 

128 elif ftype in ['wavefish', 'pulsefish']: 

129 yield None, file_path, ftype[:-4] 

130 else: 

131 continue 

132 

133 

134 def find_recording(recording, meta_recordings): 

135 """ Find row of a recording in meta data. 

136 

137 Parameters 

138 ---------- 

139 recording: string 

140 Path and base name of a recording. 

141 meta_recordings: list of string 

142 List of meta data recordings where to find `recording`. 

143 """ 

144 if meta_data is not None: 

145 rec = os.path.splitext(os.path.basename(recording))[0] 

146 for i in range(len(meta_recordings)): 

147 # TODO: strip extension! 

148 if rec == meta_recordings[i]: 

149 return i 

150 return -1 

151 

152 # prepare meta recodings names: 

153 meta_recordings_used = None 

154 if meta_recordings is not None: 

155 meta_recordings_used = np.zeros(len(meta_recordings), dtype=bool) 

156 for r in range(len(meta_recordings)): 

157 meta_recordings[r] = os.path.splitext(os.path.basename(meta_recordings[r]))[0] 

158 # prepare adjusted temperatures: 

159 if meta_data is not None and temp_col is not None: 

160 temp_idx = meta_data.index(temp_col) 

161 temp = meta_data[:,temp_idx] 

162 mean_tmp = np.round(np.nanmean(temp)/0.1)*0.1 

163 meta_data.insert(temp_idx+1, 'T_adjust', 'C', '%.1f') 

164 meta_data.append_data_column([mean_tmp]*meta_data.rows(), temp_idx+1) 

165 # prepare species distances: 

166 wave_names, wave_eods, pulse_names, pulse_eods = \ 

167 load_species_waveforms(cfg.value('speciesFile')) 

168 wave_max_rms = cfg.value('maximumWaveSpeciesRMS') 

169 pulse_max_rms = cfg.value('maximumPulseSpeciesRMS') 

170 # load data:  

171 wave_table = None 

172 pulse_table = None 

173 all_table = None 

174 file_pathes = [] 

175 for zf, file_name, fish_type in file_iter(files): 

176 # file name: 

177 table = None 

178 window_time = None 

179 recording, base_path, channel, start_time, _, _, file_ext = \ 

180 parse_filename(file_name) 

181 file_ext = os.extsep + file_ext 

182 file_pathes.append(os.path.normpath(recording).split(os.path.sep)) 

183 if verbose > 2: 

184 print('processing %s (%s):' % (file_name, recording)) 

185 # find row in meta_data: 

186 mr = -1 

187 if meta_data is not None: 

188 mr = find_recording(recording, meta_recordings) 

189 if mr < 0: 

190 if skip_recordings: 

191 if verbose > 0: 

192 print('skip recording %s: no metadata found' % recording) 

193 continue 

194 elif verbose > 0: 

195 print('no metadata found for recording %s' % recording) 

196 else: 

197 meta_recordings_used[mr] = True 

198 # data: 

199 if zf is not None: 

200 file_name = io.TextIOWrapper(zf.open(file_name, 'r')) 

201 data = TableData(file_name) 

202 if 'twin' in data: 

203 start_time = data[0, 'twin'] 

204 window_time = data[0, 'window'] 

205 data.remove(['twin', 'window']) 

206 table = wave_table if fish_type == 'wave' else pulse_table 

207 # prepare tables: 

208 if not table: 

209 df = TableData(data) 

210 df.clear_data() 

211 if meta_data is not None: 

212 if data.nsecs > 0: 

213 df.insert_section(0, 'metadata') 

214 for c in range(meta_data.columns()): 

215 df.insert(c, *meta_data.column_head(c)) 

216 df.insert(0, ['recording']*data.nsecs + ['file'], '', '%-s') 

217 if window_time is not None: 

218 df.insert(1, 'window', 's', '%.2f') 

219 if start_time is not None: 

220 df.insert(1, 'time', 's', '%.2f') 

221 if channel >= 0: 

222 df.insert(1, 'channel', '', '%d') 

223 if fish_type == 'wave': 

224 if harmonics is not None: 

225 fn = base_path + '-wavespectrum-0' + file_ext 

226 if zf is not None: 

227 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

228 wave_spec = TableData(fn) 

229 if data.nsecs > 0: 

230 df.append_section('harmonics') 

231 for h in range(min(harmonics, wave_spec.rows())+1): 

232 df.append('ampl%d' % h, wave_spec.unit('amplitude'), 

233 wave_spec.format('amplitude')) 

234 if h > 0: 

235 df.append('relampl%d' % h, '%', '%.2f') 

236 df.append('relpower%d' % h, '%', '%.2f') 

237 df.append('phase%d' % h, 'rad', '%.3f') 

238 if len(wave_names) > 0: 

239 if data.nsecs > 0: 

240 df.append_section('species') 

241 for species in wave_names: 

242 df.append(species, '%', '%.0f') 

243 df.append('species', '', '%-s') 

244 else: 

245 if phases0 is not None: 

246 fn = base_path + '-pulsephases-0' + file_ext 

247 if zf is not None: 

248 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

249 pulse_phases = TableData(fn) 

250 if data.nsecs > 0: 

251 df.append_section('phases') 

252 for p in range(phases0, phases1+1): 

253 if p != 1: 

254 df.append('P%dtime' % p, 'ms', '%.3f') 

255 df.append('P%dampl' % p, pulse_phases.unit('amplitude'), 

256 pulse_phases.format('amplitude')) 

257 if p != 1: 

258 df.append('P%drelampl' % p, '%', '%.2f') 

259 df.append('P%dwidth' % p, 'ms', '%.3f') 

260 if len(pulse_names) > 0: 

261 if data.nsecs > 0: 

262 df.append_section('species') 

263 for species in pulse_names: 

264 df.append(species, '%', '%.0f') 

265 df.append('species', '', '%-s') 

266 if fish_type == 'wave': 

267 wave_table = df 

268 table = wave_table 

269 else: 

270 pulse_table = df 

271 table = pulse_table 

272 if not all_table: 

273 df = TableData() 

274 df.append('file', '', '%-s') 

275 if channel >= 0: 

276 df.append('channel', '', '%d') 

277 if start_time is not None: 

278 df.append('time', 's', '%.1f') 

279 if window_time is not None: 

280 df.append('window', 's', '%.1f') 

281 if meta_data is not None: 

282 for c in range(meta_data.columns()): 

283 df.append(*meta_data.column_head(c)) 

284 df.append('index', '', '%d') 

285 df.append('EODf', 'Hz', '%.1f') 

286 df.append('type', '', '%-5s') 

287 if len(wave_names) + len(pulse_names) > 0: 

288 df.append('species', '', '%-s') 

289 all_table = df 

290 # fill tables: 

291 n = data.rows() if not max_fish or max_fish > data.rows() else max_fish 

292 for r in range(n): 

293 # fish index: 

294 idx = r 

295 if 'index' in data: 

296 idx = data[r,'index'] 

297 # check quality: 

298 skips = '' 

299 if fish_type == 'wave': 

300 fn = base_path + '-wavespectrum-%d'%idx + file_ext 

301 if zf is not None: 

302 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

303 wave_spec = TableData(fn) 

304 if cfg is not None: 

305 spec_data = wave_spec.array() 

306 props = data.row_dict(r) 

307 if 'clipped' in props: 

308 props['clipped'] *= 0.01 

309 if 'noise' in props: 

310 props['noise'] *= 0.01 

311 if 'rmserror' in props: 

312 props['rmserror'] *= 0.01 

313 if 'thd' in props: 

314 props['thd'] *= 0.01 

315 _, skips, msg = wave_quality(props, 0.01*spec_data[1:,3], 

316 **wave_quality_args(cfg)) 

317 else: 

318 if cfg is not None: 

319 props = data.row_dict(r) 

320 if 'clipped' in props: 

321 props['clipped'] *= 0.01 

322 if 'noise' in props: 

323 props['noise'] *= 0.01 

324 skips, msg, _ = pulse_quality(props, **pulse_quality_args(cfg)) 

325 if len(skips) > 0: 

326 if verbose > 1: 

327 print('skip fish %2d from %s: %s' % (idx, recording, skips)) 

328 continue 

329 # fill in data: 

330 data_col = 0 

331 table.append_data(recording, data_col) 

332 all_table.append_data(recording, data_col) 

333 data_col += 1 

334 if channel >= 0: 

335 table.append_data(channel, data_col) 

336 all_table.append_data(channel, data_col) 

337 data_col += 1 

338 if start_time is not None: 

339 table.append_data(start_time, data_col) 

340 all_table.append_data(start_time, data_col) 

341 data_col += 1 

342 if window_time is not None: 

343 table.append_data(window_time, data_col) 

344 all_table.append_data(window_time, data_col) 

345 data_col += 1 

346 # meta data: 

347 if mr >= 0: 

348 for c in range(meta_data.columns()): 

349 table.append_data(meta_data[mr,c], data_col) 

350 all_table.append_data(meta_data[mr,c], data_col) 

351 data_col += 1 

352 elif meta_data is not None: 

353 data_col += meta_data.columns() 

354 table.append_data(data[r,:].array(), data_col) 

355 eodf = data[r,'EODf'] 

356 all_table.append_data(data[r,'index'], data_col) 

357 all_table.append_data(eodf) 

358 all_table.append_data(fish_type) 

359 species_name = 'unknown' 

360 species_rms = 1.0e12 

361 if fish_type == 'wave': 

362 if harmonics is not None: 

363 for h in range(min(harmonics, wave_spec.rows())+1): 

364 table.append_data(wave_spec[h,'amplitude']) 

365 if h > 0: 

366 table.append_data(wave_spec[h,'relampl']) 

367 table.append_data(wave_spec[h,'relpower']) 

368 table.append_data(wave_spec[h,'phase']) 

369 if len(wave_names) > 0: 

370 fn = base_path + '-eodwaveform-%d'%idx + file_ext 

371 if zf is not None: 

372 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

373 wave_eod = TableData(fn).array() 

374 wave_eod[:,0] *= 0.001 

375 for species, eod in zip(wave_names, wave_eods): 

376 rms = wave_similarity(eod, wave_eod, 1.0, eodf) 

377 if rms < species_rms and rms < wave_max_rms: 

378 species_name = species 

379 species_rms = rms 

380 table.append_data(100.0*rms) 

381 table.append_data(species_name) 

382 else: 

383 if phases0 is not None: 

384 fn = base_path + '-pulsephases-%d'%idx + file_ext 

385 if zf is not None: 

386 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

387 pulse_phases = TableData(fn) 

388 for p in range(phases0, phases1+1): 

389 for pr in range(pulse_phases.rows()): 

390 if pulse_phases[pr,'P'] == p: 

391 break 

392 else: 

393 continue 

394 if p != 1: 

395 table.append_data(pulse_phases[pr,'time'], 'P%dtime' % p) 

396 table.append_data(pulse_phases[pr,'amplitude'], 'P%dampl' % p) 

397 if p != 1: 

398 table.append_data(pulse_phases[pr,'relampl'], 'P%drelampl' % p) 

399 table.append_data(pulse_phases[pr,'width'], 'P%dwidth' % p) 

400 if len(pulse_names) > 0: 

401 fn = base_path + '-eodwaveform-%d'%idx + file_ext 

402 if zf is not None: 

403 fn = io.TextIOWrapper(zf.open(fn, 'r')) 

404 pulse_eod = TableData(fn).array() 

405 pulse_eod[:,0] *= 0.001 

406 for species, eod in zip(pulse_names, pulse_eods): 

407 rms = pulse_similarity(eod, pulse_eod) 

408 if rms < species_rms and rms < pulse_max_rms: 

409 species_name = species 

410 species_rms = rms 

411 table.append_data(100.0*rms) 

412 table.append_data(species_name) 

413 #if len(wave_names) + len(pulse_names) > 0: 

414 # all_table.append_data(species_name) 

415 table.fill_data() 

416 all_table.fill_data() 

417 # check coverage of meta data: 

418 if meta_recordings_used is not None: 

419 if np.all(meta_recordings_used): 

420 if verbose > 0: 

421 print('found recordings for all meta data') 

422 else: 

423 if verbose > 0: 

424 print('no recordings found for:') 

425 for mr in range(len(meta_recordings)): 

426 recording = meta_recordings[mr] 

427 if not meta_recordings_used[mr]: 

428 if verbose > 0: 

429 print(recording) 

430 all_table.set_column(0) 

431 all_table.append_data(recording) 

432 for c in range(meta_data.columns()): 

433 all_table.append_data(meta_data[mr,c]) 

434 all_table.append_data(np.nan) # index 

435 all_table.append_data(np.nan) # EODf 

436 all_table.append_data('none') # type 

437 # adjust EODf to mean temperature: 

438 for table in [wave_table, pulse_table, all_table]: 

439 if table is not None and temp_col is not None: 

440 eodf_idx = table.index('EODf') 

441 table.insert(eodf_idx+1, 'EODf_adjust', 'Hz', '%.1f') 

442 table.fill_data() 

443 temp_idx = table.index(temp_col) 

444 tadjust_idx = table.index('T_adjust') 

445 for r in range(table.rows()): 

446 eodf = table[r,eodf_idx] 

447 if np.isfinite(table[r,temp_col]) and np.isfinite(table[r,tadjust_idx]): 

448 eodf = adjust_eodf(eodf, table[r,temp_col], table[r,tadjust_idx], q10) 

449 table[r,eodf_idx+1] = eodf 

450 # add wavefish species (experimental): 

451 # simplify pathes: 

452 if simplify_file and len(file_pathes) > 1: 

453 fp0 = file_pathes[0] 

454 for fi in range(len(fp0)): 

455 is_same = True 

456 for fp in file_pathes[1:]: 

457 if fi >= len(fp) or fp[fi] != fp0[fi]: 

458 is_same = False 

459 break 

460 if not is_same: 

461 break 

462 for table in [wave_table, pulse_table, all_table]: 

463 if table is not None: 

464 for k in range(table.rows()): 

465 idx = table.index('file') 

466 fps = os.path.normpath(table[k,idx]).split(os.path.sep) 

467 table[k,idx] = os.path.sep.join(fps[fi:]) 

468 return wave_table, pulse_table, all_table 

469 

470 

471def rangestr(string): 

472 """ 

473 Parse string of the form N:M . 

474 """ 

475 if string[0] == '=': 

476 string = '-' + string[1:] 

477 ss = string.split(':') 

478 v0 = v1 = None 

479 if len(ss) == 1: 

480 v0 = int(string) 

481 v1 = v0 

482 else: 

483 v0 = int(ss[0]) 

484 v1 = int(ss[1]) 

485 return (v0, v1) 

486 

487 

488def main(cargs=None): 

489 # command line arguments: 

490 if cargs is None: 

491 cargs = sys.argv[1:] 

492 parser = argparse.ArgumentParser(add_help=True, 

493 description='Collect data generated by thunderfish in a wavefish and a pulsefish table.', 

494 epilog='version %s by Benda-Lab (2019-%s)' % (__version__, __year__)) 

495 parser.add_argument('--version', action='version', version=__version__) 

496 parser.add_argument('-v', action='count', dest='verbose', default=0, 

497 help='verbosity level: -v for meta data coverage, -vv for additional info on discarded recordings.') 

498 parser.add_argument('-t', dest='table_type', default=None, choices=['wave', 'pulse'], 

499 help='wave-type or pulse-type fish') 

500 parser.add_argument('-c', dest='simplify_file', action='store_true', 

501 help='remove initial common directories from input files') 

502 parser.add_argument('-m', dest='max_fish', type=int, metavar='N', 

503 help='maximum number of fish to be taken from each recording') 

504 parser.add_argument('-p', dest='pulse_phases', type=rangestr, 

505 default=(0, 1), metavar='N:M', 

506 help='add properties of phase N to M of pulse-type EODs to the table') 

507 parser.add_argument('-w', dest='harmonics', type=int, default=3, metavar='N', 

508 help='add properties of first N harmonics of wave-type EODs to the table') 

509 parser.add_argument('-r', dest='remove_cols', action='append', default=[], metavar='COLUMN', 

510 help='columns to be removed from output table') 

511 parser.add_argument('-s', dest='statistics', action='store_true', 

512 help='also write table with statistics') 

513 parser.add_argument('-i', dest='meta_file', metavar='FILE:REC:TEMP', default='', type=str, 

514 help='insert rows from metadata table in FILE matching recording in colum REC. The optional TEMP specifies a column with temperatures to which EOD frequencies should be adjusted') 

515 parser.add_argument('-q', dest='q10', metavar='Q10', default=1.62, type=float, 

516 help='Q10 value for adjusting EOD frequencies to a common temperature') 

517 parser.add_argument('-S', dest='skip', action='store_true', 

518 help='skip recordings that are not contained in metadata table') 

519 parser.add_argument('-n', dest='file_suffix', metavar='NAME', default='', type=str, 

520 help='name for summary files that is appended to "wavefish" or "pulsefish"') 

521 parser.add_argument('-o', dest='out_path', metavar='PATH', default='.', type=str, 

522 help='path where to store summary tables') 

523 parser.add_argument('-f', dest='format', default='auto', type=str, 

524 choices=TableData.formats + ['same'], 

525 help='file format used for saving summary tables ("same" uses same format as input files)') 

526 parser.add_argument('file', nargs='+', default='', type=str, 

527 help='a *-wavefish.* or *-pulsefish.* file as generated by thunderfish') 

528 # fix minus sign issue: 

529 ca = [] 

530 pa = False 

531 for a in cargs: 

532 if pa and a[0] == '-': 

533 a = '=' + a[1:] 

534 pa = False 

535 if a == '-p': 

536 pa = True 

537 ca.append(a) 

538 # read in command line arguments:  

539 args = parser.parse_args(ca) 

540 verbose = args.verbose 

541 table_type = args.table_type 

542 remove_cols = args.remove_cols 

543 statistics = args.statistics 

544 meta_file = args.meta_file 

545 file_suffix = args.file_suffix 

546 out_path = args.out_path 

547 data_format = args.format 

548 

549 # expand wildcard patterns: 

550 files = [] 

551 if os.name == 'nt': 

552 for fn in args.file: 

553 files.extend(glob.glob(fn)) 

554 else: 

555 files = args.file 

556 

557 # read configuration: 

558 cfgfile = __package__ + '.cfg' 

559 cfg = ConfigFile() 

560 add_harmonic_groups_config(cfg) 

561 add_eod_quality_config(cfg) 

562 add_species_config(cfg) 

563 add_write_table_config(cfg, table_format='csv', unit_style='row', 

564 align_columns=True, shrink_width=False) 

565 cfg.load_files(cfgfile, files[0], 3) 

566 # output format: 

567 if data_format == 'same': 

568 ext = os.path.splitext(files[0])[1][1:] 

569 if ext in TableData.ext_formats: 

570 data_format = TableData.ext_formats[ext] 

571 else: 

572 data_format = 'dat' 

573 if data_format != 'auto': 

574 cfg.set('fileFormat', data_format) 

575 # create output folder: 

576 if not os.path.exists(out_path): 

577 os.makedirs(out_path) 

578 # read in meta file: 

579 md = None 

580 rec_data = None 

581 temp_col = None 

582 if len(meta_file) > 0: 

583 mds = meta_file.split(':') 

584 meta_data = mds[0] 

585 if not os.path.isfile(meta_data): 

586 print('meta data file "%s" not found.' % meta_data) 

587 exit() 

588 md = TableData(meta_data) 

589 if len(mds) < 2: 

590 print('no recording column specified for the table in %s. Choose one of' % meta_data) 

591 for k in md.keys(): 

592 print(' ', k) 

593 exit() 

594 rec_col = mds[1] 

595 if rec_col not in md: 

596 print('%s is not a valid key for the table in %s. Choose one of' % (rec_col, meta_data)) 

597 for k in md.keys(): 

598 print(' ', k) 

599 exit() 

600 else: 

601 rec_data = md[:,rec_col] 

602 del md[:,rec_col] 

603 if len(mds) > 2: 

604 temp_col = mds[2] 

605 if temp_col not in md: 

606 print('%s is not a valid key for the table in %s. Choose one of' % (temp_col, meta_data)) 

607 for k in md.keys(): 

608 print(' ', k) 

609 exit() 

610 # collect files: 

611 wave_table, pulse_table, all_table = collect_fish(files, args.simplify_file, 

612 md, rec_data, args.skip, 

613 temp_col, args.q10, 

614 args.max_fish, args.harmonics, 

615 args.pulse_phases[0], args.pulse_phases[1], 

616 cfg, verbose) 

617 # write tables: 

618 if len(file_suffix) > 0 and file_suffix[0] != '-': 

619 file_suffix = '-' + file_suffix 

620 tables = [] 

621 table_names = [] 

622 if pulse_table and (not table_type or table_type == 'pulse'): 

623 tables.append(pulse_table) 

624 table_names.append('pulse') 

625 if wave_table and (not table_type or table_type == 'wave'): 

626 tables.append(wave_table) 

627 table_names.append('wave') 

628 if all_table and not table_type: 

629 tables.append(all_table) 

630 table_names.append('all') 

631 for table, name in zip(tables, table_names): 

632 for rc in remove_cols: 

633 if rc in table: 

634 table.remove(rc) 

635 table.write(os.path.join(out_path, '%sfish%s' % (name, file_suffix)), 

636 **write_table_args(cfg)) 

637 if statistics: 

638 s = table.statistics() 

639 s.write(os.path.join(out_path, '%sfish%s-statistics' % (name, file_suffix)), 

640 **write_table_args(cfg)) 

641 

642 

643if __name__ == '__main__': 

644 main()