Coverage for src/thunderlab/tabledata.py: 87%
2168 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 15:01 +0000
1"""Tables with hierarchical headers and units
3## Classes
5- `class TableData`: tables with hierarchical header including units
6 and column-specific formats. Kind of similar to a pandas data frame,
7 but without index column and with intuitive numpy-style indexing and
8 nicely formatted output to csv, markdown, html, and latex.
11## Helper functions
13- `write()`: shortcut for constructing and writing a TableData.
14- `latex_unit()`: translate unit string into SIunit LaTeX code.
15- `index2aa()`: convert an integer into an alphabetical representation.
16- `aa2index()`: convert an alphabetical representation to an index.
19## Configuration
21- `add_write_table_config()`: add parameter specifying how to write a table to a file as a new section to a configuration.
22- `write_table_args()`: translates a configuration to the respective parameter names for writing a table to a file.
24"""
26import sys
27import re
28import math as m
29import numpy as np
30try:
31 import pandas as pd
32 has_pandas = True
33except ImportError:
34 has_pandas = False
36from pathlib import Path
37from itertools import product
38from io import StringIO
41__pdoc__ = {}
42__pdoc__['TableData.__contains__'] = True
43__pdoc__['TableData.__len__'] = True
44__pdoc__['TableData.__iter__'] = True
45__pdoc__['TableData.__next__'] = True
46__pdoc__['TableData.__setupkey__'] = True
47__pdoc__['TableData.__call__'] = True
48__pdoc__['TableData.__getitem__'] = True
49__pdoc__['TableData.__setitem__'] = True
50__pdoc__['TableData.__delitem__'] = True
51__pdoc__['TableData.__str__'] = True
54default_missing_str = '-'
55"""Default string indicating nan data elements when outputting data."""
57default_missing_inputs = ['na', 'NA', 'nan', 'NAN', '-']
58"""Default strings that are translated to nan when loading table data."""
61class TableData(object):
62 """Table with numpy-style indexing and hierarchical header including units and formats.
64 Parameters
65 ----------
66 data: str, stream, ndarray
67 - a filename: load table from file with name `data`.
68 - a stream/file handle: load table from that stream.
69 - 1-D or 2-D ndarray of data: the data of the table.
70 Requires als a specified `header`.
71 - pandas data frame.
72 header: TableData, dict, list of str, list of list of str
73 Header labels for each column.
74 See `set_labels()' for details.
75 units: None, TableData, dict, list of str, str
76 Optional unit strings for each column.
77 See `set_units()' for details.
78 formats: None, TableData, dict, list of str, str
79 Optional format strings for each column.
80 See `set_formats()' for details.
81 descriptions: None, TableData, dict, list of str, str
82 Optional description strings for each column.
83 See `set_descriptions()' for details.
84 missing: list of str
85 Missing data are indicated by one of these strings.
86 sep: str or None
87 If `data` is a file, force `sep` as column separator.
88 stop: str or None
89 If a line matches `stop`, stop reading the file. `stop`
90 can be an empty string to stop reading at the first empty
91 line.
93 Manipulate table header
94 -----------------------
96 Each column of the table has a label (the name of the column), a
97 unit, and a format specifier. Sections group columns into a hierarchy.
99 - `__init__()`: initialize a TableData from data or a file.
100 - `append()`: append column to the table.
101 - `insert()`: insert a table column at a given position.
102 - `remove()`: remove columns from the table.
103 - `section()`: the section name of a specified column.
104 - `set_section()`: set a section name.
105 - `append_section()`: add sections to the table header.
106 - `insert_section()`: insert a section at a given position of the table header.
107 - `label()`: the name of a column.
108 - `set_label()`: set the name of a column.
109 - `set_labels()`: set the labels of some columns.
110 - `unit()`: the unit of a column.
111 - `set_unit()`: set the unit of a column.
112 - `set_units()`: set the units of some columns.
113 - `format()`: the format string of the column.
114 - `set_format()`: set the format string of a column.
115 - `set_formats()`: set the format strings of some columns.
116 - `description()`: the description of a column.
117 - `set_description()`: set the description of a column.
118 - `set_descriptions()`: set the descriptions of some columns.
120 For example:
121 ```
122 tf = TableData('data.csv')
123 ```
124 loads a table directly from a file. See `load()` for details.
125 ```
126 tf = TableData(np.random.randn(4,3), header=['aaa', 'bbb', 'ccc'], units=['m', 's', 'g'], formats='%.2f')
127 ```
128 results in
129 ``` plain
130 aaa bbb ccc
131 m s g
132 1.45 0.01 0.16
133 -0.74 -0.58 -1.34
134 -2.06 0.08 1.47
135 -0.43 0.60 1.38
136 ```
138 A more elaborate way to construct a table is:
139 ```
140 df = TableData()
141 # first column with section names and 3 data values:
142 df.append(["data", "specimen", "size"], "m", "%6.2f",
143 [2.34, 56.7, 8.9])
144 # next columns with single data values:
145 df.append("full weight", "kg", "%.0f", 122.8)
146 df.append_section("all measures")
147 df.append("speed", "m/s", "%.3g", 98.7)
148 df.append("median jitter", "mm", "%.1f", 23)
149 df.append("size", "g", "%.2e", 1.234)
150 # add a missing value to the second column:
151 df.add(np.nan, 1)
152 # fill up the remaining columns of the row:
153 df.add((0.543, 45, 1.235e2))
154 # add data to the next row starting at the second column:
155 df.add([43.21, 6789.1, 3405, 1.235e-4], 1) # next row
156 ```
157 results in
158 ``` plain
159 data
160 specimen all measures
161 size full weight speed median jitter size
162 m kg m/s mm g
163 2.34 123 98.7 23.0 1.23e+00
164 56.70 - 0.543 45.0 1.24e+02
165 8.90 43 6.79e+03 3405.0 1.23e-04
166 ```
168 Table columns
169 -------------
171 Columns can be specified by an index or by the name of a column. In
172 table headers with sections the colum can be specified by the
173 section names and the column name separated by '>'.
175 - `index()`: the column index of a column specifier.
176 - `__contains__()`: check for existence of a column.
177 - `find_col()`: find the start and end index of a column specification.
178 - `column_spec()`: full specification of a column with all its section names.
179 - `column_head()`: the name, unit, and format of a column.
180 - `table_header()`: the header of the table without content.
182 For example:
183 ```
184 df.index('all measures>size) # returns 4
185 'speed' in df # is True
186 ```
188 Iterating over columns
189 ----------------------
191 A table behaves like an ordered dictionary with column names as
192 keys and the data of each column as values.
193 Iterating over a table goes over columns.
194 Note, however, that the len() of a table is the number of rows,
195 not the number of columns!
197 - `keys()`: list of unique column keys for all available columns.
198 - `values()`: list of column data corresponding to keys().
199 - `items()`: generator over column names and the corresponding data.
200 - `__iter__()`: initialize iteration over data columns.
201 - `__next__()`: return unique column key of next column.
202 - `data`: the table data as a list over columns each containing a list of data elements.
204 For example:
205 ```
206 print('column specifications:')
207 for c in range(df.columns()):
208 print(df.column_spec(c))
209 print('iterating over column specifications:')
210 for c, k in enumerate(df):
211 print(f'{c}: {k}')
212 print('keys():')
213 for c, k in enumerate(df.keys()):
214 print(f'{c}: {k}')
215 print('values():')
216 for c, v in enumerate(df.values()):
217 print(v)
218 print('iterating over the table:')
219 for v in df:
220 print(v)
221 ```
222 results in
223 ``` plain
224 column specifications:
225 data>specimen>size
226 data>specimen>full weight
227 data>all measures>speed
228 data>all measures>median jitter
229 data>all measures>size
230 iterating over column specifications:
231 0: data>specimen>size
232 1: data>specimen>full weight
233 2: data>all measures>speed
234 3: data>all measures>median jitter
235 4: data>all measures>size
236 keys():
237 0: data>specimen>size
238 1: data>specimen>full weight
239 2: data>all measures>speed
240 3: data>all measures>median jitter
241 4: data>all measures>size
242 values():
243 [2.34, 56.7, 8.9]
244 [122.8, nan, 43.21]
245 [98.7, 0.543, 6789.1]
246 [23, 45, 3405]
247 [1.234, 123.5, 0.0001235]
248 iterating over the table:
249 [2.34, 56.7, 8.9]
250 [122.8, nan, 43.21]
251 [98.7, 0.543, 6789.1]
252 [23, 45, 3405]
253 [1.234, 123.5, 0.0001235]
254 ```
256 Accessing data
257 --------------
259 In contrast to the iterator functions the [] operator treats the
260 table as a 2D-array where the first index indicates the row and
261 the second index the column.
263 Rows are indexed by integer row numbers or boolean arrays.
264 Columns are also indexed by integer column numbers, but in
265 addition can be index by their names.
267 A single index selects rows, unless it is specified by
268 strings. Since strings can only specify column names, this selects
269 whole columns.
271 Like a numpy array the table can be sliced, and logical indexing can
272 be used to select specific parts of the table.
274 As for any function, columns can be specified as indices or strings.
276 - `rows()`: the number of rows.
277 - `columns()`: the number of columns.
278 - `__len__()`: the number of rows.
279 - `ndim`: always 2.
280 - `size`: number of elements (sum of length of all data columns), can be smaller than `columns()*rows()`.
281 - `shape`: number of rows and columns.
282 - `row()`: a single row of the table as TableData.
283 - `row_list()`: a single row of the table as list.
284 - `row_data()`: a generator for iterating over rows of the table.
285 - `row_dict()`: a single row of the table as dictionary.
286 - `col()`: a single column of the table as TableData.
287 - `__call__()`: a single column of the table as ndarray.
288 - `__getitem__()`: data elements specified by slice.
289 - `__setitem__()`: assign values to data elements specified by slice.
290 - `__delitem__()`: delete data elements or whole columns or rows.
291 - `array()`: the table data as a ndarray.
292 - `data_frame()`: the table data as a pandas DataFrame.
293 - `dicts()`: the table as a list of dictionaries.
294 - `dict()`: the table as a dictionary.
295 - `add()`: add data elements row-wise.
296 - `append_data_column()`: append data elements to a column.
297 - `set_column()`: set the column where to add data.
298 - `fill_data()`: fill up all columns with missing data.
299 - `clear_data()`: clear content of the table but keep header.
300 - `clear()`: clear the table of any content and header information.
301 - `key_value()`: a data element returned as a key-value pair.
302 - `aggregate()`: apply functions to columns.
303 - `groupby()`: iterate through unique values of columns.
305 - `sort()`: sort the table rows in place.
306 - `statistics()`: descriptive statistics of each column.
308 For example:
309 ```
310 # single column:
311 df('size') # data of 'size' column as ndarray
312 df['size'] # data of 'size' column as ndarray
313 df[:, 'size'] # data of 'size' column as ndarray
314 df.col('size') # table with the single column 'size'
316 # single row:
317 df[2, :] # table with data of only the third row
318 df.row(2) # table with data of only the third row
320 # slices:
321 df[2:5,['size','jitter']] # sub-table
322 df[2:5,['size','jitter']].array() # ndarray with data only
324 # logical indexing:
325 df[df['speed'] > 100.0, 'size'] = 0.0 # set size to 0 if speed is > 100
327 # delete:
328 del df[3:6, 'weight'] # delete rows 3-6 from column 'weight'
329 del df[3:5, :] # delete rows 3-5 completeley
330 del df[:, 'speed'] # remove column 'speed' from table
331 del df['speed'] # remove column 'speed' from table
332 df.remove('weight') # remove column 'weigth' from table
334 # sort and statistics:
335 df.sort(['weight', 'jitter'])
336 df.statistics()
337 ```
338 statistics() returns a table with standard descriptive statistics:
339 ``` plain
340 statistics data
341 - specimen all measures
342 - size full weight speed median jitter size
343 - m kg m/s mm g
344 mean 22.65 83 2.3e+03 1157.7 4.16e+01
345 std 24.23 40 3.18e+03 1589.1 5.79e+01
346 min 2.34 43 0.543 23.0 1.23e-04
347 quartile1 5.62 83 49.6 34.0 6.17e-01
348 median 8.90 123 98.7 45.0 1.23e+00
349 quartile3 32.80 - 3.44e+03 1725.0 6.24e+01
350 max 56.70 123 6.79e+03 3405.0 1.24e+02
351 count 3.00 2 3 3.0 3.00e+00
352 ```
354 Write and load tables
355 ---------------------
357 Table data can be written to a variety of text-based formats
358 including comma separated values, latex and html files. Which
359 columns are written can be controlled by the hide() and show()
360 functions. TableData can be loaded from all the written file formats
361 (except html), also directly via the constructor.
363 - `hide()`: hide a column or a range of columns.
364 - `hide_all()`: hide all columns.
365 - `hide_empty_columns()`: hide all columns that do not contain data.
366 - `show()`: show a column or a range of columns.
367 - `write()`: write table to a file or stream.
368 - `write_file_stream()`: write table to file or stream and return appropriate file name.
369 - `__str__()`: write table to a string.
370 - `write_descriptions()`: write column descriptions of the table to a file or stream.
371 - `load()`: load table from file or stream.
372 - `formats`: list of supported file formats for writing.
373 - `descriptions`: dictionary with descriptions of the supported file formats.
374 - `extensions`: dictionary with default filename extensions for each of the file formats.
375 - `ext_formats`: dictionary mapping filename extensions to file formats.
377 See documentation of the `write()` function for examples of the supported file formats.
379 """
381 formats = ['dat', 'ascii', 'csv', 'rtai', 'md', 'tex', 'html']
382 """list of strings: Supported output formats."""
384 descriptions = {'dat': 'data text file', 'ascii': 'ascii-art table',
385 'csv': 'comma separated values', 'rtai': 'rtai-style table',
386 'md': 'markdown', 'tex': 'latex tabular',
387 'html': 'html markup'}
388 """dict: Decription of output formats corresponding to `formats`."""
390 extensions = {'dat': 'dat', 'ascii': 'txt', 'csv': 'csv', 'rtai': 'dat',
391 'md': 'md', 'tex': 'tex', 'html': 'html'}
392 """dict: Default file extensions for the output `formats`. """
394 ext_formats = {'dat': 'dat', 'DAT': 'dat', 'txt': 'dat', 'TXT': 'dat',
395 'csv': 'csv', 'CSV': 'csv', 'md': 'md', 'MD': 'md',
396 'tex': 'tex', 'TEX': 'tex', 'html': 'html', 'HTML': 'html'}
397 """dict: Mapping of file extensions to the output formats."""
399 stdev_labels = ['sd', 'std', 's.d.', 'stdev', 'error']
400 """list: column labels recognized as standard deviations."""
402 def __init__(self, data=None, header=None, units=None, formats=None,
403 descriptions=None, missing=default_missing_inputs,
404 sep=None, stop=None):
405 self.clear()
406 if data is None:
407 ncols = 0
408 for stuff in [header, units, formats, descriptions]:
409 if stuff is not None:
410 if isinstance(stuff, TableData):
411 n = stuff.columns()
412 elif isinstance(stuff, (list, tuple, np.ndarray)) and not \
413 (isinstance(stuff, np.ndarray) and \
414 len(stuff.shape) == 0):
415 n = len(stuff)
416 elif not isinstance(stuff, dict):
417 n = 1
418 if n > ncols:
419 ncols = n
420 # initialize empty table:
421 if ncols > 0:
422 for c in range(ncols):
423 self.data.append([])
424 else:
425 if isinstance(data, TableData):
426 self.ndim = data.ndim
427 self.size = data.size
428 self.shape = data.shape
429 self.nsecs = data.nsecs
430 self.setcol = data.setcol
431 self.addcol = data.addcol
432 for c in range(data.columns()):
433 self.header.append(list(data.header[c]))
434 self.units.append(data.units[c])
435 self.formats.append(data.formats[c])
436 self.descriptions.append(data.descriptions[c])
437 self.hidden.append(data.hidden[c])
438 self.data.append(list(data.data[c]))
439 elif has_pandas and isinstance(data, pd.DataFrame):
440 for c, key in enumerate(data.keys()):
441 new_key = key
442 new_unit = ''
443 if '/' in key:
444 p = key.split('/')
445 new_key = p[0].strip()
446 new_unit = '/'.join(p[1:])
447 formats = '%s' if isinstance(values[0], str) else '%g'
448 values = data[key].tolist()
449 self.append(new_key, new_unit, formats, value=values)
450 elif isinstance(data, (list, tuple, np.ndarray)) and not \
451 (isinstance(data, np.ndarray) and len(data.shape) == 0):
452 if len(data) > 0 and \
453 isinstance(data[0], (list, tuple, np.ndarray)) and not \
454 (isinstance(data[0], np.ndarray) and \
455 len(data[0].shape) == 0):
456 # 2D list, rows first:
457 for c in range(len(data[0])):
458 self.data.append([])
459 for row in data:
460 for c, val in enumerate(row):
461 self.data[c].append(val)
462 elif len(data) > 0 and isinstance(data[0], dict):
463 # list of dictionaries:
464 for d in data:
465 self._add_dict(d, True)
466 self.fill_data()
467 else:
468 # 1D list:
469 for val in data:
470 self.data.append([val])
471 elif isinstance(data, dict):
472 self._add_dict(data, True)
473 self.fill_data()
474 else:
475 self.load(data, missing, sep, stop)
476 self.set_labels(header)
477 self.set_units(units)
478 self.set_formats(formats)
479 self.set_descriptions(descriptions)
480 self.hidden = [False]*len(self.data)
481 self.addcol = len(self.data)
482 self.__recompute_shape()
484 def __recompute_shape(self):
485 self.size = sum(map(len, self.data))
486 self.shape = (self.rows(), self.columns())
488 def append(self, label, unit=None, formats=None, description=None,
489 value=None, fac=None, key=None):
490 """Append column to the table.
492 Parameters
493 ----------
494 label: str or list of str
495 Optional section titles and the name of the column.
496 unit: str or None
497 The unit of the column contents.
498 formats: str or None
499 The C-style format string used for printing out the column content, e.g.
500 '%g', '%.2f', '%s', etc.
501 If None, the format is set to '%g'.
502 description: str or None
503 The description of the column contents.
504 value: None, float, int, str, etc. or list thereof, or list of dict
505 If not None, data for the column.
506 If list of dictionaries, extract from each dictionary in the list
507 the value specified by `key`. If `key` is `None` use `label` as
508 the key.
509 fac: float
510 If not None, multiply the data values by this number.
511 key: None or key of a dictionary
512 If not None and `value` is a list of dictionaries,
513 extract from each dictionary in the list the value specified
514 by `key` and assign the resulting list as data to the column.
516 Returns
517 -------
518 self: TableData
519 This TableData
520 """
521 if self.addcol >= len(self.data):
522 if isinstance(label, (list, tuple, np.ndarray)):
523 label = list(reversed(label))
524 # number of sections larger than what we have so far:
525 n = max(0, len(label) - 1 - self.nsecs)
526 # find matching sections:
527 found = False
528 for s in range(1, len(label)):
529 for c in range(len(self.header) - 1, -1, -1):
530 if len(self.header[c]) > s - n:
531 if s - n >= 0 and \
532 self.header[c][s - n] == label[s]:
533 # remove matching sections:
534 label = label[:s]
535 found = True
536 break
537 if found:
538 break
539 # add label and unique sections:
540 self.header.append(label)
541 label = label[0]
542 if n > 0:
543 # lift previous header label and sections:
544 for c in range(len(self.header) - 1):
545 self.header[c] = ['-']*n + self.header[c]
546 else:
547 self.header.append([label])
548 self.units.append(unit or '')
549 self.formats.append(formats or '%g')
550 self.descriptions.append(description or '')
551 self.hidden.append(False)
552 self.data.append([])
553 self.nsecs = max(map(len, self.header)) - 1
554 else:
555 if isinstance(label, (list, tuple, np.ndarray)):
556 self.header[self.addcol] = list(reversed(label)) + self.header[self.addcol]
557 label = label[-1]
558 else:
559 self.header[self.addcol] = [label] + self.header[self.addcol]
560 self.units[self.addcol] = unit or ''
561 self.formats[self.addcol] = formats or '%g'
562 self.descriptions[self.addcol] = description or ''
563 if self.nsecs < len(self.header[self.addcol]) - 1:
564 self.nsecs = len(self.header[self.addcol]) - 1
565 if not key:
566 key = label
567 if value is not None:
568 if isinstance(value, (list, tuple, np.ndarray)):
569 if key and len(value) > 0 and isinstance(value[0], dict):
570 value = [d[key] if key in d else float('nan') for d in value]
571 self.data[-1].extend(value)
572 else:
573 self.data[-1].append(value)
574 if fac:
575 for k in range(len(self.data[-1])):
576 self.data[-1][k] *= fac
577 self.addcol = len(self.data)
578 self.__recompute_shape()
579 return self
581 def insert(self, column, label, unit=None, formats=None, description=None,
582 value=None, fac=None, key=None):
583 """Insert a table column at a given position.
585 .. WARNING::
586 If no `value` is given, the inserted column is an empty list.
588 Parameters
589 ----------
590 columns int or str
591 Column before which to insert the new column.
592 Column can be specified by index or name,
593 see `index()` for details.
594 label: str or list of str
595 Optional section titles and the name of the column.
596 unit: str or None
597 The unit of the column contents.
598 formats: str or None
599 The C-style format string used for printing out the column content, e.g.
600 '%g', '%.2f', '%s', etc.
601 If None, the format is set to '%g'.
602 description: str or None
603 The description of the column contents.
604 value: None, float, int, str, etc. or list thereof, or list of dict
605 If not None, data for the column.
606 If list of dictionaries, extract from each dictionary in the list
607 the value specified by `key`. If `key` is `None` use `label` as
608 the key.
609 fac: float
610 If not None, multiply the data values by this number.
611 key: None or key of a dictionary
612 If not None and `value` is a list of dictionaries,
613 extract from each dictionary in the list the value specified
614 by `key` and assign the resulting list as data to the column.
616 Returns
617 -------
618 index: int
619 The index of the inserted column.
621 Raises
622 ------
623 self: TableData
624 This TableData
625 """
626 col = self.index(column)
627 if col is None:
628 raise IndexError(f'Cannot insert before non-existing column "{column}"')
629 if isinstance(label, (list, tuple, np.ndarray)):
630 self.header.insert(col, list(reversed(label)))
631 else:
632 self.header.insert(col, [label])
633 self.units.insert(col, unit or '')
634 self.formats.insert(col, formats or '%g')
635 self.descriptions.insert(col, description or '')
636 self.hidden.insert(col, False)
637 self.data.insert(col, [])
638 if self.nsecs < len(self.header[col]) - 1:
639 self.nsecs = len(self.header[col]) - 1
640 if not key:
641 key = label
642 if value is not None:
643 if isinstance(value, (list, tuple, np.ndarray)):
644 if key and len(value) > 0 and isinstance(value[0], dict):
645 value = [d[key] if key in d else float('nan') for d in value]
646 self.data[col].extend(value)
647 else:
648 self.data[col].append(value)
649 if fac:
650 for k in range(len(self.data[col])):
651 self.data[col][k] *= fac
652 self.addcol = len(self.data)
653 self.__recompute_shape()
654 return self
656 def remove(self, columns):
657 """Remove columns from the table.
659 Parameters
660 -----------
661 columns: int or str or list of int or str
662 Columns can be specified by index or name,
663 see `index()` for details.
665 Raises
666 ------
667 IndexError:
668 If an invalid column was specified.
669 """
670 # fix columns:
671 if not isinstance(columns, (list, tuple, np.ndarray)):
672 columns = [ columns ]
673 if not columns:
674 return
675 # remove:
676 for col in columns:
677 c = self.index(col)
678 if c is None:
679 if isinstance(col, (np.integer, int)):
680 col = '%d' % col
681 raise IndexError('Cannot remove non-existing column ' + col)
682 continue
683 if c+1 < len(self.header):
684 self.header[c+1].extend(self.header[c][len(self.header[c+1]):])
685 del self.header[c]
686 del self.units[c]
687 del self.formats[c]
688 del self.descriptions[c]
689 del self.hidden[c]
690 del self.data[c]
691 if self.setcol >= len(self.data):
692 self.setcol = 0
693 self.__recompute_shape()
695 def section(self, column, level):
696 """The section name of a specified column.
698 Parameters
699 ----------
700 column: None, int, or str
701 A specification of a column.
702 See self.index() for more information on how to specify a column.
703 level: int
704 The level of the section to be returned. The column label itself is level=0.
706 Returns
707 -------
708 name: str
709 The name of the section at the specified level containing
710 the column.
711 index: int
712 The column index that contains this section
713 (equal or smaller thant `column`).
715 Raises
716 ------
717 IndexError:
718 If `level` exceeds the maximum possible level.
719 """
720 if level < 0 or level > self.nsecs:
721 raise IndexError('Invalid section level')
722 column = self.index(column)
723 while len(self.header[column]) <= level:
724 column -= 1
725 return self.header[column][level], column
727 def set_section(self, column, label, level):
728 """Set a section name.
730 Parameters
731 ----------
732 column: None, int, or str
733 A specification of a column.
734 See self.index() for more information on how to specify a column.
735 label: str
736 The new name to be used for the section.
737 level: int
738 The level of the section to be set. The column label itself is level=0.
739 """
740 column = self.index(column)
741 if not column is None:
742 self.header[column][level] = label
743 return self
745 def append_section(self, label):
746 """Add sections to the table header.
748 Each column of the table has a header label. Columns can be
749 grouped into sections. Sections can be nested arbitrarily.
751 Parameters
752 ----------
753 label: stri or list of str
754 The name(s) of the section(s).
756 Returns
757 -------
758 index: int
759 The column index where the section was appended.
760 """
761 if self.addcol >= len(self.data):
762 if isinstance(label, (list, tuple, np.ndarray)):
763 self.header.append(list(reversed(label)))
764 else:
765 self.header.append([label])
766 self.units.append('')
767 self.formats.append('')
768 self.descriptions.append('')
769 self.hidden.append(False)
770 self.data.append([])
771 else:
772 if isinstance(label, (list, tuple, np.ndarray)):
773 self.header[self.addcol] = list(reversed(label)) + self.header[self.addcol]
774 else:
775 self.header[self.addcol] = [label] + self.header[self.addcol]
776 if self.nsecs < len(self.header[self.addcol]):
777 self.nsecs = len(self.header[self.addcol])
778 self.addcol = len(self.data) - 1
779 self.__recompute_shape()
780 return self.addcol
782 def insert_section(self, column, section):
783 """Insert a section at a given position of the table header.
785 Parameters
786 ----------
787 columns int or str
788 Column before which to insert the new section.
789 Column can be specified by index or name,
790 see `index()` for details.
791 section: str
792 The name of the section.
794 Returns
795 -------
796 index: int
797 The index of the column where the section was inserted.
799 Raises
800 ------
801 IndexError:
802 If an invalid column was specified.
803 """
804 col = self.index(column)
805 if col is None:
806 if isinstance(column, (np.integer, int)):
807 column = '%d' % column
808 raise IndexError('Cannot insert at non-existing column ' + column)
809 self.header[col].append(section)
810 if self.nsecs < len(self.header[col]) - 1:
811 self.nsecs = len(self.header[col]) - 1
812 return col
814 def label(self, column):
815 """The name of a column.
817 Parameters
818 ----------
819 column: None, int, or str
820 A specification of a column.
821 See self.index() for more information on how to specify a column.
823 Returns
824 -------
825 self: TableData
826 This TableData
827 """
828 column = self.index(column)
829 return self.header[column][0]
831 def set_label(self, column, label):
832 """Set the name of a column.
834 Parameters
835 ----------
836 column: None, int, or str
837 A specification of a column.
838 See self.index() for more information on how to specify a column.
839 label: str
840 The new name to be used for the column.
842 Returns
843 -------
844 self: TableData
845 This TableData
846 """
847 column = self.index(column)
848 if not column is None:
849 self.header[column][0] = label
850 return self
852 def set_labels(self, labels):
853 """Set the labels of some columns.
855 Parameters
856 ----------
857 labels: TableData, dict, list of str, list of list of str, None
858 The new labels to be used.
859 If TableData, take the labels of the respective column indices.
860 If dict, keys are column labels (see self.index() for more
861 information on how to specify a column), and values are
862 the new labels for the respective columns as str or list of str.
863 If list of str or list of list of str,
864 set labels of the first successive columns to the list elements.
865 If `None`, do nothing.
867 Returns
868 -------
869 self: TableData
870 This TableData
871 """
872 while len(self.header) < len(self.data):
873 self.header.append([f'C{len(self.header) + 1}'])
874 if isinstance(labels, TableData):
875 for c in range(min(self.columns(), labels.columns())):
876 self.header[c] = labels.header[c]
877 elif isinstance(labels, dict):
878 for c in labels:
879 i = self.index(c)
880 if i is None:
881 continue
882 l = labels[c]
883 if isinstance(l, (list, tuple)):
884 self.header[i] = l
885 else:
886 self.header[i] = [l]
887 elif isinstance(labels, (list, tuple, np.ndarray)) and not \
888 (isinstance(labels, np.ndarray) and len(labels.shape) == 0):
889 for c, l in enumerate(labels):
890 if isinstance(l, (list, tuple)):
891 self.header[c] = l
892 else:
893 self.header[c] = [l]
894 return self
896 def unit(self, column):
897 """The unit of a column.
899 Parameters
900 ----------
901 column: None, int, or str
902 A specification of a column.
903 See self.index() for more information on how to specify a column.
905 Returns
906 -------
907 unit: str
908 The unit.
909 """
910 column = self.index(column)
911 return self.units[column]
913 def set_unit(self, column, unit):
914 """Set the unit of a column.
916 Parameters
917 ----------
918 column: None, int, or str
919 A specification of a column.
920 See self.index() for more information on how to specify a column.
921 unit: str
922 The new unit to be used for the column.
924 Returns
925 -------
926 self: TableData
927 This TableData
928 """
929 column = self.index(column)
930 if not column is None:
931 self.units[column] = unit
932 return self
934 def set_units(self, units):
935 """Set the units of some columns.
937 Parameters
938 ----------
939 units: TableData, dict, list of str, str, None
940 The new units to be used.
941 If TableData, take the units of matching column labels.
942 If dict, keys are column labels (see self.index() for more
943 information on how to specify a column), and values are
944 units for the respective columns as str.
945 If list of str, set units of the first successive columns to
946 the list elements.
947 If `None`, do nothing.
948 Otherwise, set units of all columns to `units`.
950 Returns
951 -------
952 self: TableData
953 This TableData
954 """
955 while len(self.units) < len(self.data):
956 self.units.append('')
957 if isinstance(units, TableData):
958 for c in units:
959 i = self.index(c)
960 if i is None:
961 continue
962 self.units[i] = units.unit(c)
963 elif isinstance(units, dict):
964 for c in units:
965 i = self.index(c)
966 if i is None:
967 continue
968 self.units[i] = units[c]
969 elif isinstance(units, (list, tuple, np.ndarray)) and not \
970 (isinstance(units, np.ndarray) and len(units.shape) == 0):
971 for c, u in enumerate(units):
972 self.units[c] = u
973 elif units is not None:
974 for c in range(len(self.units)):
975 self.units[c] = units
976 return self
978 def format(self, column):
979 """The format string of the column.
981 Parameters
982 ----------
983 column: None, int, or str
984 A specification of a column.
985 See self.index() for more information on how to specify a column.
987 Returns
988 -------
989 format: str
990 The format string.
991 """
992 column = self.index(column)
993 return self.formats[column]
995 def set_format(self, column, format):
996 """Set the format string of a column.
998 Parameters
999 ----------
1000 column: None, int, or str
1001 A specification of a column.
1002 See self.index() for more information on how to specify a column.
1003 format: str
1004 The new format string to be used for the column.
1006 Returns
1007 -------
1008 self: TableData
1009 This TableData
1010 """
1011 column = self.index(column)
1012 if not column is None:
1013 self.formats[column] = format
1014 return self
1016 def set_formats(self, formats):
1017 """Set the format strings of all columns.
1019 Parameters
1020 ----------
1021 formats: TableData, dict, list of str, str, None
1022 The new format strings to be used.
1023 If TableData, take the format strings of matching column labels.
1024 If dict, keys are column labels (see self.index() for more
1025 information on how to specify a column), and values are
1026 format strings for the respective columns as str.
1027 If list of str, set format strings of the first successive
1028 columns to the list elements.
1029 If `None`, do nothing.
1030 Otherwise, set format strings of all columns to `formats`.
1032 Returns
1033 -------
1034 self: TableData
1035 This TableData
1036 """
1037 while len(self.formats) < len(self.data):
1038 self.formats.append('%g')
1039 if isinstance(formats, TableData):
1040 for c in formats:
1041 i = self.index(c)
1042 if i is None:
1043 continue
1044 self.formats[i] = formats.format(c)
1045 elif isinstance(formats, dict):
1046 for c in formats:
1047 i = self.index(c)
1048 if i is None:
1049 continue
1050 self.formats[i] = formats[c] or '%g'
1051 elif isinstance(formats, (list, tuple, np.ndarray)) and not \
1052 (isinstance(formats, np.ndarray) and len(formats.shape) == 0):
1053 for c, f in enumerate(formats):
1054 self.formats[c] = f or '%g'
1055 elif formats is not None:
1056 for c in range(len(self.formats)):
1057 self.formats[c] = formats or '%g'
1058 return self
1060 def description(self, column):
1061 """The description of a column.
1063 Parameters
1064 ----------
1065 column: None, int, or str
1066 A specification of a column.
1067 See self.index() for more information on how to specify a column.
1069 Returns
1070 -------
1071 description: str
1072 The description.
1073 """
1074 column = self.index(column)
1075 return self.descriptions[column]
1077 def set_description(self, column, description):
1078 """Set the description of a column.
1080 Parameters
1081 ----------
1082 column: None, int, or str
1083 A specification of a column.
1084 See self.index() for more information on how to specify a column.
1085 description: str
1086 The new description to be used for the column.
1088 Returns
1089 -------
1090 self: TableData
1091 This TableData
1092 """
1093 column = self.index(column)
1094 if not column is None:
1095 self.descriptions[column] = description
1096 return self
1098 def set_descriptions(self, descriptions):
1099 """Set the descriptions of some columns.
1101 Parameters
1102 ----------
1103 descriptions: TableData, dict, list of str, str, None
1104 The new descriptions to be used.
1105 If TableData, take the descriptions of matching column labels.
1106 If dict, keys are column labels (see self.index() for more
1107 information on how to specify a column), and values are
1108 descriptions for the respective columns as str.
1109 If list of str, set descriptions of the first successive columns to
1110 the list elements.
1111 If `None`, do nothing.
1113 Returns
1114 -------
1115 self: TableData
1116 This TableData
1117 """
1118 while len(self.descriptions) < len(self.data):
1119 self.descriptions.append('')
1120 if isinstance(descriptions, TableData):
1121 for c in descriptions:
1122 i = self.index(c)
1123 if i is None:
1124 continue
1125 self.descriptions[i] = descriptions.description(c)
1126 elif isinstance(descriptions, dict):
1127 for c in descriptions:
1128 i = self.index(c)
1129 if i is None:
1130 continue
1131 self.descriptions[i] = descriptions[c]
1132 elif isinstance(descriptions, (list, tuple, np.ndarray)) and not \
1133 (isinstance(descriptions, np.ndarray) and len(descriptions.shape) == 0):
1134 for c, d in enumerate(descriptions):
1135 self.descriptions[c] = d
1136 return self
1138 def table_header(self):
1139 """The header of the table without content.
1141 Returns
1142 -------
1143 data: TableData
1144 A TableData object with the same header but empty data.
1145 """
1146 data = TableData()
1147 sec_indices = [-1] * self.nsecs
1148 for c in range(self.columns()):
1149 data.append(*self.column_head(c))
1150 for l in range(self.nsecs):
1151 s, i = self.section(c, l+1)
1152 if i != sec_indices[l]:
1153 data.header[-1].append(s)
1154 sec_indices[l] = i
1155 data.nsecs = self.nsecs
1156 return data
1158 def column_head(self, column, secs=False):
1159 """The name, unit, format, and description of a column.
1161 Parameters
1162 ----------
1163 column: None, int, or str
1164 A specification of a column.
1165 See self.index() for more information on how to specify a column.
1166 secs: bool
1167 If True, return all section names in addition to the column label.
1169 Returns
1170 -------
1171 name: str or list of str
1172 The column label or the label with all its sections.
1173 unit: str
1174 The unit.
1175 format: str
1176 The format string.
1177 description: str
1178 The description of the data column.
1179 """
1180 column = self.index(column)
1181 if secs:
1182 header = self.header[column]
1183 c = column - 1
1184 while len(header) < self.nsecs + 1 and c >= 0:
1185 if len(self.header[c]) > len(header):
1186 header.extend(self.header[c][len(header):])
1187 c -= 1
1188 return list(reversed(header)), self.units[column], self.formats[column], self.descriptions[column]
1189 else:
1190 return self.header[column][0], self.units[column], self.formats[column], self.descriptions[column]
1192 def column_spec(self, column):
1193 """Full specification of a column with all its section names.
1195 Parameters
1196 ----------
1197 column: int or str
1198 Specifies the column.
1199 See self.index() for more information on how to specify a column.
1201 Returns
1202 -------
1203 s: str
1204 Full specification of the column by all its section names and its header label.
1205 """
1206 c = self.index(column)
1207 fh = [self.header[c][0]]
1208 for l in range(self.nsecs):
1209 fh.append(self.section(c, l+1)[0])
1210 return '>'.join(reversed(fh))
1212 def find_col(self, column):
1213 """Find the start and end index of a column specification.
1215 Parameters
1216 ----------
1217 column: None, int, or str
1218 A specification of a column.
1219 See self.index() for more information on how to specify a column.
1221 Returns
1222 -------
1223 c0: int or None
1224 A valid column index or None that is specified by `column`.
1225 c1: int or None
1226 A valid column index or None of the column following the range specified
1227 by `column`.
1228 """
1230 def find_column_indices(ss, si, minns, maxns, c0, strict=True):
1231 if si >= len(ss):
1232 return None, None, None, None
1233 ns0 = 0
1234 for ns in range(minns, maxns+1):
1235 nsec = maxns - ns
1236 if ss[si] == '':
1237 si += 1
1238 continue
1239 for c in range(c0, len(self.header)):
1240 if nsec < len(self.header[c]) and \
1241 ((strict and self.header[c][nsec] == ss[si]) or
1242 (not strict and ss[si] in self.header[c][nsec])):
1243 ns0 = ns
1244 c0 = c
1245 si += 1
1246 if si >= len(ss):
1247 c1 = len(self.header)
1248 for c in range(c0+1, len(self.header)):
1249 if nsec < len(self.header[c]):
1250 c1 = c
1251 break
1252 return c0, c1, ns0, None
1253 elif nsec > 0:
1254 break
1255 return None, c0, ns0, si
1257 if column is None:
1258 return None, None
1259 if not isinstance(column, (np.integer, int)) and column.isdigit():
1260 column = int(column)
1261 if isinstance(column, (np.integer, int)):
1262 if column >= 0 and column < len(self.header):
1263 return column, column + 1
1264 else:
1265 return None, None
1266 # find column by header:
1267 ss = column.rstrip('>').split('>')
1268 maxns = self.nsecs
1269 si0 = 0
1270 while si0 < len(ss) and ss[si0] == '':
1271 maxns -= 1
1272 si0 += 1
1273 if maxns < 0:
1274 maxns = 0
1275 c0, c1, ns, si = find_column_indices(ss, si0, 0, maxns, 0, True)
1276 if c0 is None and c1 is not None:
1277 c0, c1, ns, si = find_column_indices(ss, si, ns, maxns, c1, False)
1278 return c0, c1
1280 def index(self, column):
1281 """The column index of a column specifier.
1283 Parameters
1284 ----------
1285 column: None, int, or str
1286 A specification of a column.
1287 - None: no column is specified
1288 - int: the index of the column (first column is zero), e.g. `index(2)`.
1289 - a string representing an integer is converted into the column index,
1290 e.g. `index('2')`
1291 - a string specifying a column by its header.
1292 Header names of descending hierarchy are separated by '>'.
1294 Returns
1295 -------
1296 index: int or None
1297 A valid column index or None.
1298 """
1299 c0, c1 = self.find_col(column)
1300 return c0
1302 def __contains__(self, column):
1303 """Check for existence of a column.
1305 Parameters
1306 ----------
1307 column: None, int, or str
1308 The column to be checked.
1309 See self.index() for more information on how to specify a column.
1311 Returns
1312 -------
1313 contains: bool
1314 True if `column` specifies an existing column key.
1315 """
1316 return self.index(column) is not None
1318 def keys(self):
1319 """List of unique column keys for all available columns.
1321 Returns
1322 -------
1323 keys: list of str
1324 List of unique column specifications.
1325 """
1326 return [self.column_spec(c) for c in range(self.columns())]
1328 def values(self):
1329 """List of column data corresponding to keys(). Same as `self.data`.
1331 Returns
1332 -------
1333 data: list of list of values
1334 The data of the table. First index is columns!
1335 """
1336 return self.data
1338 def items(self):
1339 """Generator over column names and corresponding data.
1341 Yields
1342 ------
1343 item: tuple
1344 Unique column specifications and the corresponding data.
1345 """
1346 for c in range(self.columns()):
1347 yield self.column_spec(c), self.data[c]
1349 def __len__(self):
1350 """The number of rows.
1352 Returns
1353 -------
1354 rows: int
1355 The number of rows contained in the table.
1356 """
1357 return self.rows()
1359 def __iter__(self):
1360 """Initialize iteration over data columns.
1361 """
1362 self.iter_counter = -1
1363 return self
1365 def __next__(self):
1366 """Next unique column key.
1368 Returns
1369 -------
1370 s: str
1371 Full specification of the column by all its section names and its header label.
1372 """
1373 self.iter_counter += 1
1374 if self.iter_counter >= self.columns():
1375 raise StopIteration
1376 else:
1377 return self.column_spec(self.iter_counter)
1379 def rows(self):
1380 """The number of rows.
1382 Returns
1383 -------
1384 rows: int
1385 The number of rows contained in the table.
1386 """
1387 return max(map(len, self.data)) if self.data else 0
1389 def columns(self):
1390 """The number of columns.
1392 Returns
1393 -------
1394 columns: int
1395 The number of columns contained in the table.
1396 """
1397 return len(self.header)
1399 def row(self, index):
1400 """A single row of the table as TableData.
1402 Parameters
1403 ----------
1404 index: int
1405 The index of the row to be returned.
1407 Returns
1408 -------
1409 data: TableData
1410 A TableData object with a single row.
1411 """
1412 data = TableData()
1413 sec_indices = [-1] * self.nsecs
1414 for c in range(self.columns()):
1415 data.append(*self.column_head(c))
1416 for l in range(self.nsecs):
1417 s, i = self.section(c, l+1)
1418 if i != sec_indices[l]:
1419 data.header[-1].append(s)
1420 sec_indices[l] = i
1421 data.data[-1] = [self.data[c][index]]
1422 data.nsecs = self.nsecs
1423 return data
1425 def row_list(self, index):
1426 """A single row of the table as list.
1428 Parameters
1429 ----------
1430 index: int
1431 The index of the row to be returned.
1433 Returns
1434 -------
1435 data: list
1436 A list with data values of each column of row `index`.
1437 """
1438 data = []
1439 for c in range(self.columns()):
1440 data.append(self.data[c][index])
1441 return data
1443 def row_data(self):
1444 """A generator for iterating over rows of the table.
1446 Yields
1447 ------
1448 data: list
1449 A list with data values of each column.
1450 """
1451 for r in range(self.rows()):
1452 yield self.row_list(r)
1454 def row_dict(self, index):
1455 """A single row of the table as dictionary.
1457 Parameters
1458 ----------
1459 index: int
1460 The index of the row to be returned.
1462 Returns
1463 -------
1464 data: dict
1465 A dictionary with column header as key and corresponding data value of row `index`
1466 as value.
1467 """
1468 data = {}
1469 for c in range(self.columns()):
1470 data[self.column_spec(c)] = self.data[c][index]
1471 return data
1473 def column(self, col):
1474 """A single column of the table.
1476 Parameters
1477 ----------
1478 col: None, int, or str
1479 The column to be returned.
1480 See self.index() for more information on how to specify a column.
1482 Returns
1483 -------
1484 table: TableData
1485 A TableData object with a single column.
1486 """
1487 data = TableData()
1488 c = self.index(col)
1489 data.append(*self.column_head(c))
1490 data.data = [self.data[c]]
1491 data.nsecs = 0
1492 return data
1494 def __call__(self, column):
1495 """A single column of the table as a ndarray.
1497 Parameters
1498 ----------
1499 column: None, int, or str
1500 The column to be returned.
1501 See self.index() for more information on how to specify a column.
1503 Returns
1504 -------
1505 data: 1-D ndarray
1506 Content of the specified column as a ndarray.
1507 """
1508 c = self.index(column)
1509 return np.asarray(self.data[c])
1511 def __setupkey(self, key):
1512 """Helper function that turns a key into row and column indices.
1514 Returns
1515 -------
1516 rows: list of int, slice, None
1517 Indices of selected rows.
1518 cols: list of int
1519 Indices of selected columns.
1521 Raises
1522 ------
1523 IndexError:
1524 If an invalid column was specified.
1525 """
1526 if type(key) is not tuple:
1527 if isinstance(key, str):
1528 cols = key
1529 rows = slice(0, self.rows(), 1)
1530 elif isinstance(key, slice) and isinstance(key.start, str) and isinstance(key.stop, str):
1531 cols = key
1532 rows = slice(0, self.rows(), 1)
1533 else:
1534 rows = key
1535 cols = range(self.columns())
1536 else:
1537 rows = key[0]
1538 cols = key[1]
1539 if isinstance(cols, slice):
1540 start = cols.start
1541 if start is not None:
1542 start = self.index(start)
1543 if start is None:
1544 raise IndexError('"%s" is not a valid column index' % cols.start)
1545 stop = cols.stop
1546 if stop is not None:
1547 stop_str = isinstance(stop, str)
1548 stop = self.index(stop)
1549 if stop is None:
1550 raise IndexError('"%s" is not a valid column index' % cols.stop)
1551 if stop_str:
1552 stop += 1
1553 cols = slice(start, stop, cols.step)
1554 cols = range(self.columns())[cols]
1555 else:
1556 if not isinstance(cols, (list, tuple, np.ndarray, range)):
1557 cols = [cols]
1558 c = [self.index(inx) for inx in cols]
1559 if None in c:
1560 raise IndexError('"%s" is not a valid column index' % cols[c.index(None)])
1561 cols = c
1562 if isinstance(rows, np.ndarray) and rows.dtype == np.dtype(bool):
1563 rows = np.where(rows)[0]
1564 if len(rows) == 0:
1565 rows = None
1566 return rows, cols
1568 def __getitem__(self, key):
1569 """Data elements specified by slice.
1571 Parameters
1572 -----------
1573 key:
1574 First key specifies row, (optional) second key the column.
1575 Columns can be specified by index or name,
1576 see `index()` for details.
1577 A single key of strings selects columns by their names: `td[:, 'col'] == td['col']`
1578 If a stop column is specified by name,
1579 it is inclusively!
1581 Returns
1582 -------
1583 data:
1584 - A single data value if a single row and a single column is specified.
1585 - A ndarray of data elements if a single column is specified.
1586 - A TableData object for multiple columns.
1587 - None if no row is selected (e.g. by a logical index that nowhere is True)
1589 Raises
1590 ------
1591 IndexError:
1592 If an invalid column was specified.
1593 """
1594 rows, cols = self.__setupkey(key)
1595 if len(cols) == 1:
1596 if cols[0] >= self.columns():
1597 return None
1598 if rows is None:
1599 return None
1600 elif isinstance(rows, slice):
1601 return np.asarray(self.data[cols[0]][rows])
1602 elif isinstance(rows, (list, tuple, np.ndarray)):
1603 return np.asarray([self.data[cols[0]][r] for r in rows if r < len(self.data[cols[0]])])
1604 elif rows < len(self.data[cols[0]]):
1605 return self.data[cols[0]][rows]
1606 else:
1607 return None
1608 else:
1609 data = TableData()
1610 sec_indices = [-1] * self.nsecs
1611 for c in cols:
1612 data.append(*self.column_head(c, secs=True))
1613 if rows is None:
1614 continue
1615 if isinstance(rows, (list, tuple, np.ndarray)):
1616 for r in rows:
1617 data.data[-1].append(self.data[c][r])
1618 else:
1619 try:
1620 if isinstance(self.data[c][rows],
1621 (list, tuple, np.ndarray)):
1622 data.data[-1].extend(self.data[c][rows])
1623 else:
1624 data.data[-1].append(self.data[c][rows])
1625 except IndexError:
1626 data.data[-1].append(np.nan)
1627 data.nsecs = self.nsecs
1628 return data
1630 def __setitem__(self, key, value):
1631 """Assign values to data elements specified by slice.
1633 Parameters
1634 -----------
1635 key:
1636 First key specifies row, (optional) second one the column.
1637 Columns can be specified by index or name,
1638 see `index()` for details.
1639 A single key of strings selects columns by their names: `td[:, 'col'] == td['col']`
1640 If a stop column is specified by name,
1641 it is inclusively!
1642 value: TableData, list, ndarray, float, ...
1643 Value(s) used to assing to the table elements as specified by `key`.
1645 Raises
1646 ------
1647 IndexError:
1648 If an invalid column was specified.
1649 """
1650 rows, cols = self.__setupkey(key)
1651 if rows is None:
1652 return
1653 if isinstance(value, TableData):
1654 if isinstance(self.data[cols[0]][rows], (list, tuple, np.ndarray)):
1655 for k, c in enumerate(cols):
1656 self.data[c][rows] = value.data[k]
1657 else:
1658 for k, c in enumerate(cols):
1659 self.data[c][rows] = value.data[k][0]
1660 else:
1661 if len(cols) == 1:
1662 if isinstance(rows, (list, tuple, np.ndarray)):
1663 if len(rows) == 1:
1664 self.data[cols[0]][rows[0]] = value
1665 elif isinstance(value, (list, tuple, np.ndarray)):
1666 for k, r in enumerate(rows):
1667 self.data[cols[0]][r] = value[k]
1668 else:
1669 for r in rows:
1670 self.data[cols[0]][r] = value
1671 elif isinstance(value, (list, tuple, np.ndarray)):
1672 self.data[cols[0]][rows] = value
1673 elif isinstance(rows, (np.integer, int)):
1674 self.data[cols[0]][rows] = value
1675 else:
1676 n = len(self.data[cols[0]][rows])
1677 if n > 1:
1678 self.data[cols[0]][rows] = [value]*n
1679 else:
1680 self.data[cols[0]][rows] = value
1681 else:
1682 if isinstance(self.data[0][rows], (list, tuple, np.ndarray)):
1683 for k, c in enumerate(cols):
1684 self.data[c][rows] = value[:,k]
1685 elif isinstance(value, (list, tuple, np.ndarray)):
1686 for k, c in enumerate(cols):
1687 self.data[c][rows] = value[k]
1688 else:
1689 for k, c in enumerate(cols):
1690 self.data[c][rows] = value
1692 def __delitem__(self, key):
1693 """Delete data elements or whole columns or rows.
1695 Parameters
1696 -----------
1697 key:
1698 First key specifies row, (optional) second one the column.
1699 Columns can be specified by index or name,
1700 see `index()` for details.
1701 A single key of strings selects columns by their names: `td[:, 'col'] == td['col']`
1702 If a stop column is specified by name,
1703 it is inclusively!
1704 If all rows are selected, then the specified columns are removed from the table.
1705 Otherwise only data values are removed.
1706 If all columns are selected than entire rows of data values are removed.
1707 Otherwise only data values in the specified rows are removed.
1709 Raises
1710 ------
1711 IndexError:
1712 If an invalid column was specified.
1713 """
1714 rows, cols = self.__setupkey(key)
1715 if rows is None:
1716 return
1717 row_indices = np.arange(self.rows(), dtype=int)[rows]
1718 if isinstance(row_indices, np.ndarray):
1719 if len(row_indices) == self.rows():
1720 # delete whole columns:
1721 self.remove(cols)
1722 elif len(row_indices) > 0:
1723 for r in reversed(sorted(row_indices)):
1724 for c in cols:
1725 if r < len(self.data[c]):
1726 del self.data[c][r]
1727 self.__recompute_shape()
1728 else:
1729 for c in cols:
1730 del self.data[c][row_indices]
1731 self.__recompute_shape()
1733 def array(self, row=None):
1734 """The table data as a ndarray.
1736 Parameters
1737 ----------
1738 row: int or None
1739 If specified, a 1D ndarray of that row will be returned.
1741 Returns
1742 -------
1743 data: 2D or 1D ndarray
1744 If no row is specified, the data content of the entire table
1745 as a 2D ndarray (rows first).
1746 If a row is specified, a 1D ndarray of that row.
1747 """
1748 if row is None:
1749 return np.array(self.data).T
1750 else:
1751 return np.array([d[row] for d in self.data])
1753 def data_frame(self):
1754 """The table data as a pandas DataFrame.
1756 Returns
1757 -------
1758 data: pandas.DataFrame
1759 A pandas DataFrame of the whole table.
1760 """
1761 return pd.DataFrame(self.dict())
1763 def dicts(self, raw_values=True, missing=default_missing_str):
1764 """The table as a list of dictionaries.
1766 Parameters
1767 ----------
1768 raw_values: bool
1769 If True, use raw table values as values,
1770 else format the values and add unit string.
1771 missing: str
1772 String indicating non-existing data elements.
1774 Returns
1775 -------
1776 table: list of dict
1777 For each row of the table a dictionary with header as key.
1778 """
1779 table = []
1780 for row in range(self.rows()):
1781 data = {}
1782 for col in range(len(self.header)):
1783 if raw_values:
1784 v = self.data[col][row];
1785 else:
1786 if isinstance(self.data[col][row], (float, np.floating)) and m.isnan(self.data[col][row]):
1787 v = missing
1788 else:
1789 u = ''
1790 if not self.units[col] in '1-' and self.units[col] != 'a.u.':
1791 u = self.units[col]
1792 v = (self.formats[col] % self.data[col][row]) + u
1793 data[self.header[col][0]] = v
1794 table.append(data)
1795 return table
1797 def dict(self):
1798 """The table as a dictionary.
1800 Returns
1801 -------
1802 table: dict
1803 A dictionary with keys being the column headers and
1804 values the list of data elements of the corresponding column.
1805 """
1806 table = {k: v for k, v in self.items()}
1807 return table
1809 def _add_table_data(self, data, add_all):
1810 """Add data of a TableData.
1812 Parameters
1813 ----------
1814 data: TableData
1815 Table with the data to be added.
1816 add_all: bool
1817 If False, then only data of columns that already exist in
1818 the table are added to the table. If the table is empty or
1819 `add_all` is set to `True` then all data is added and if
1820 necessary new columns are appended to the table.
1821 """
1822 empty = False
1823 if self.shape[1] == 0:
1824 add_all = True
1825 empty = True
1826 maxr = self.rows()
1827 for k in data.keys():
1828 col = self.index(k)
1829 if empty or col is None:
1830 if not add_all:
1831 continue
1832 self.append(*data.column_head(k, secs=True),
1833 value=[np.nan]*maxr)
1834 col = len(self.data) - 1
1835 c = data.index(k)
1836 self.data[col].extend(data.data[c])
1837 self.__recompute_shape()
1839 def _add_dict(self, data, add_all):
1840 """Add data of a TableData.
1842 Parameters
1843 ----------
1844 data: dict
1845 Keys are column labels and values are single values or
1846 lists of values to be added to the corresponding table columns.
1847 add_all: bool
1848 If False, then only data of columns that already exist in
1849 the table are added to the table. If the table is empty or
1850 `add_all` is set to `True` then all data is added and if
1851 necessary new columns are appended to the table.
1853 """
1854 empty = False
1855 if self.shape[1] == 0:
1856 add_all = True
1857 empty = True
1858 maxr = self.rows()
1859 for key in data:
1860 new_key = key
1861 new_unit = ''
1862 if '/' in key:
1863 p = key.split('/')
1864 new_key = p[0].strip()
1865 new_unit = '/'.join(p[1:])
1866 col = self.index(new_key)
1867 if empty or col is None:
1868 if not add_all:
1869 continue
1870 self.append(new_key, new_unit,
1871 value=[np.nan]*maxr)
1872 col = len(self.data) - 1
1873 if isinstance(data[key], (list, tuple, np.ndarray)):
1874 self.data[col].extend(data[key])
1875 else:
1876 self.data[col].append(data[key])
1877 self.__recompute_shape()
1879 def add(self, data, column=None, add_all=False):
1880 """Add data elements to successive columns.
1882 The current column is set behid the added columns.
1884 Parameters
1885 ----------
1886 data: float, int, str, etc. or list thereof or list of list thereof or dict or list of dict or TableData
1887 Data values to be appended to successive columns:
1888 - A single value is simply appended to the specified
1889 column of the table.
1890 - A 1D-list of values is appended to successive columns of the table
1891 starting with the specified column.
1892 - The columns (second index) of a 2D-list of values are
1893 appended to successive columns of the table starting
1894 with the specified column.
1895 - Values of a dictionary or of a list of dictionaries are
1896 added to the columns specified by the keys. Dictionary
1897 values can also be lists of values. Their values are
1898 added to successive rows of the columns specified by the
1899 dictionary keys. Does not affect the current column.
1900 - All elements of a TableData are added to matching columns.
1901 Does not affect the current column.
1902 column: None, int, or str
1903 The first column to which the data should be appended,
1904 if `data` does not specify columns.
1905 If None, append to the current column.
1906 See self.index() for more information on how to specify a column.
1907 add_all: bool
1908 If the data are given as dictionaries or TableData, then
1909 only data of columns that already exist in the table are
1910 added to the table. If the table is empty or `add_all` is
1911 set to `True` then all data is added and if necessary new
1912 columns are appended to the table.
1913 """
1914 if self.shape[1] == 0:
1915 add_all = True
1916 column = self.index(column)
1917 if column is None:
1918 column = self.setcol
1919 if isinstance(data, TableData):
1920 self._add_table_data(data, add_all)
1921 elif isinstance(data, (list, tuple, np.ndarray)) and not \
1922 (isinstance(data, np.ndarray) and len(data.shape) == 0):
1923 if len(data) > 0 and \
1924 isinstance(data[0], (list, tuple, np.ndarray)) and not \
1925 (isinstance(data[0], np.ndarray) and len(data[0].shape) == 0):
1926 # 2D list, rows first:
1927 for row in data:
1928 for i, val in enumerate(row):
1929 self.data[column + i].append(val)
1930 self.setcol = column + len(data[0])
1931 elif len(data) > 0 and isinstance(data[0], dict):
1932 # list of dictionaries:
1933 for d in data:
1934 self._add_dict(d, add_all)
1935 else:
1936 # 1D list:
1937 for val in data:
1938 self.data[column].append(val)
1939 column += 1
1940 self.setcol = column
1941 elif isinstance(data, dict):
1942 # dictionary with values:
1943 self._add_dict(data, add_all)
1944 else:
1945 # single value:
1946 self.data[column].append(data)
1947 self.setcol = column + 1
1948 if self.setcol >= len(self.data):
1949 self.setcol = 0
1950 self.__recompute_shape()
1952 def append_data_column(self, data, column=None):
1953 """Append data elements to a column.
1955 The current column is incremented by one.
1957 Parameters
1958 ----------
1959 data: float, int, str, etc. or list thereof
1960 Data values to be appended to a column.
1961 column: None, int, or str
1962 The column to which the data should be appended.
1963 If None, append to the current column.
1964 See self.index() for more information on how to specify a column.
1965 """
1966 column = self.index(column)
1967 if column is None:
1968 column = self.setcol
1969 if isinstance(data, (list, tuple, np.ndarray)):
1970 self.data[column].extend(data)
1971 column += 1
1972 self.setcol = column
1973 else:
1974 self.data[column].append(data)
1975 self.setcol = column+1
1976 if self.setcol >= len(self.data):
1977 self.setcol = 0
1978 self.__recompute_shape()
1980 def set_column(self, column):
1981 """Set the column where to add data.
1983 Parameters
1984 ----------
1985 column: int or str
1986 The column to which data elements should be appended.
1987 See self.index() for more information on how to specify a column.
1989 Raises
1990 ------
1991 IndexError:
1992 If an invalid column was specified.
1993 """
1994 col = self.index(column)
1995 if col is None:
1996 if isinstance(column, (np.integer, int)):
1997 column = '%d' % column
1998 raise IndexError('column ' + column + ' not found or invalid')
1999 self.setcol = col
2000 return col
2002 def fill_data(self):
2003 """Fill up all columns with missing data to have the same number of
2004 data elements.
2005 """
2006 # maximum rows:
2007 maxr = self.rows()
2008 # fill up:
2009 for c in range(len(self.data)):
2010 while len(self.data[c]) < maxr:
2011 self.data[c].append(np.nan)
2012 self.setcol = 0
2013 self.__recompute_shape()
2015 def clear_data(self):
2016 """Clear content of the table but keep header.
2017 """
2018 for c in range(len(self.data)):
2019 self.data[c] = []
2020 self.setcol = 0
2021 self.__recompute_shape()
2023 def clear(self):
2024 """Clear the table of any content and header information.
2025 """
2026 self.ndim = 2
2027 self.size = 0
2028 self.shape = (0, 0)
2029 self.nsecs = 0
2030 self.header = []
2031 self.units = []
2032 self.formats = []
2033 self.descriptions = []
2034 self.data = []
2035 self.hidden = []
2036 self.setcol = 0
2037 self.addcol = 0
2039 def sort(self, columns, reverse=False):
2040 """Sort the table rows in place.
2042 Parameters
2043 ----------
2044 columns: int or str or list of int or str
2045 A column specifier or a list of column specifiers of the columns
2046 to be sorted.
2047 reverse: boolean
2048 If `True` sort in descending order.
2050 Raises
2051 ------
2052 IndexError:
2053 If an invalid column was specified.
2054 """
2055 # fix columns:
2056 if not isinstance(columns, (list, tuple, np.ndarray)):
2057 columns = [ columns ]
2058 if not columns:
2059 return
2060 cols = []
2061 for col in columns:
2062 c = self.index(col)
2063 if c is None:
2064 if isinstance(col, (np.integer, int)):
2065 col = '%d' % col
2066 raise IndexError('sort column ' + col + ' not found')
2067 continue
2068 cols.append(c)
2069 # get sorted row indices:
2070 row_inx = range(self.rows())
2071 row_inx = sorted(row_inx, key=lambda x : [float('-inf') if self.data[c][x] is np.nan \
2072 or self.data[c][x] != self.data[c][x] \
2073 else self.data[c][x] for c in cols], reverse=reverse)
2074 # sort table according to indices:
2075 for c in range(self.columns()):
2076 self.data[c] = [self.data[c][r] for r in row_inx]
2078 def key_value(self, row, col, missing=default_missing_str):
2079 """A data element returned as a key-value pair.
2081 Parameters
2082 ----------
2083 row: int
2084 Specifies the row from which the data element should be retrieved.
2085 col: None, int, or str
2086 A specification of a column.
2087 See self.index() for more information on how to specify a column.
2088 missing: str
2089 String indicating non-existing data elements.
2091 Returns
2092 -------
2093 key: str
2094 Header label of the column
2095 value: str
2096 A textual representation of the data element according to the format
2097 of the column, followed by the unit of the column.
2098 """
2099 col = self.index(col)
2100 if col is None:
2101 return ''
2102 if isinstance(self.data[col][row], (float, np.floating)) and m.isnan(self.data[col][row]):
2103 v = missing
2104 else:
2105 u = ''
2106 if not self.units[col] in '1-' and self.units[col] != 'a.u.':
2107 u = self.units[col]
2108 v = (self.formats[col] % self.data[col][row]) + u
2109 return self.header[col][0], v
2111 def _aggregate(self, funcs, columns=None, label=None,
2112 numbers_only=False, remove_nans=False, single_row=False,
2113 keep_columns=None):
2114 """Apply functions to columns.
2116 Parameter
2117 ---------
2118 funcs: function, list of function, dict
2119 Functions that are applied to columns of the table.
2120 - a single function that is applied to the `columns`.
2121 The results are named according to the function's `__name__`.
2122 - a list or tuple of functions.
2123 The results are named according to the functions' `__name__`.
2124 - a dictionary. The results are named after the provided keys,
2125 the functions are given by the values.
2126 If the function returns more than one value, then the
2127 corresponding key in the dictionary needs to be a tuple
2128 (not a list!) of names for each of the returned values.
2129 Functions in lists or dictionaries can be just a plain
2130 function, like `max` or `np.mean`. In case a function
2131 needs further arguments, then you need to supply a tuple
2132 with the first elements being the function, the second
2133 element being another tuple holding positional arguments,
2134 and an optional third argument holding a dictionary for
2135 key-word arguments.
2136 columns: None, int or str or list of int or str
2137 Columns of the table on which functions are applied.
2138 If None apply functions on all columns.
2139 label: str or list of str
2140 Column label and optional section names of the first
2141 column with the function labels (if `single_row` is `False`).
2142 numbers_only: bool
2143 If True, skip columns that do not contain numbers.
2144 remove_nans: bool
2145 If True, remove nans before passing column values to function.
2146 single_row: bool
2147 If False, add for each function a row to the table.
2148 If True, add function values in a single row.
2149 keep_columns: None, int or str or list of int or str
2150 Columns of the table from which to simply keep the first value.
2151 Only if single_row is True. Usefull for grouped tables.
2152 Order of columns and keep_columns are kept from the original table.
2154 Returns
2155 -------
2156 dest: TableData
2157 A new table with the column headers specified by `columns`.
2158 A first column is inserted with the function labels.
2159 The functions are applied to all columns specified by `columns`
2160 and their return values are written into the new table.
2162 """
2163 # standardize functions dictionary:
2164 if not isinstance(funcs, (list, tuple, dict)):
2165 funcs = [funcs]
2166 if isinstance(funcs, (list, tuple)):
2167 fs = {}
2168 for f in funcs:
2169 fs[f.__name__] = f
2170 funcs = fs
2171 fs = {}
2172 for k in funcs:
2173 kk = k
2174 if not isinstance(k, tuple):
2175 kk = (k,)
2176 v = funcs[k]
2177 if not isinstance(v, tuple):
2178 v = (funcs[k], (), {})
2179 elif len(v) < 3:
2180 v = (v[0], v[1], {})
2181 fs[kk] = v
2182 funcs = fs
2183 # standardize columns:
2184 if columns is None:
2185 columns = list(range(self.shape[1]))
2186 if not isinstance(columns, (list, tuple, np.ndarray)):
2187 columns = [columns]
2188 if numbers_only:
2189 cols = []
2190 for c in columns:
2191 c = self.index(c)
2192 if len(self.data[c]) > 0 and \
2193 isinstance(self.data[c][0], (float, int, np.floating, np.integer)):
2194 cols.append(c)
2195 columns = cols
2196 if label is None:
2197 label = 'property'
2198 dest = TableData()
2199 if single_row:
2200 if keep_columns is None:
2201 keep_columns = []
2202 elif not isinstance(keep_columns, (list, tuple)):
2203 keep_columns = [keep_columns]
2204 keep_columns = [self.index(c) for c in keep_columns]
2205 columns = [self.index(c) for c in columns]
2206 columns = [c for c in columns if not c in keep_columns]
2207 keep = np.zeros(len(keep_columns) + len(columns), dtype=bool)
2208 keep[:len(keep_columns)] = True
2209 columns = keep_columns + columns
2210 idx = np.argsort(columns)
2211 for i in idx:
2212 c = columns[i]
2213 if keep[i]:
2214 name, unit, format, descr = self.column_head(c, secs=True)
2215 dest.append(name + ['-'], unit, format, descr,
2216 value=self.data[c][0])
2217 else:
2218 name, unit, format, descr = self.column_head(c, secs=True)
2219 values = self[:, c]
2220 if remove_nans:
2221 values = values[np.isfinite(values)]
2222 for k in funcs:
2223 v = funcs[k][0](values, *funcs[k][1], **funcs[k][2])
2224 if len(k) == 1:
2225 dest.append(name + [k[0]], unit, format, descr,
2226 value=v)
2227 else:
2228 for j in range(len(k)):
2229 dest.append(name + [k[j]], unit, format, descr,
2230 value=v[j])
2231 dest.fill_data()
2232 else:
2233 dest.append(label, '', '%-s')
2234 for c in columns:
2235 dest.append(*self.column_head(c, secs=True))
2236 for k in funcs:
2237 for j in range(len(k)):
2238 dest.add(k[j], 0)
2239 for i, c in enumerate(columns):
2240 values = self[:, c]
2241 if remove_nans:
2242 values = values[np.isfinite(values)]
2243 v = funcs[k][0](values, *funcs[k][1], **funcs[k][2])
2244 if len(k) == 1:
2245 dest.add(v, i + 1)
2246 else:
2247 for j in range(len(k)):
2248 dest.add(v[j], i + 1)
2249 dest.fill_data()
2250 return dest
2252 def aggregate(self, funcs, columns=None, label=None,
2253 numbers_only=False, remove_nans=False,
2254 single_row=False, by=None):
2255 """Apply functions to columns.
2257 Parameter
2258 ---------
2259 funcs: function, list of function, dict
2260 Functions that are applied to columns of the table.
2261 - a single function that is applied to the `columns`.
2262 The results are named according to the function's `__name__`.
2263 - a list or tuple of functions.
2264 The results are named according to the functions' `__name__`.
2265 - a dictionary. The results are named after the provided keys,
2266 the functions are given by the values.
2267 If the function returns more than one value, then the
2268 corresponding key in the dictionary needs to be a tuple
2269 (not a list!) of names for each of the returned values.
2270 Functions in lists or dictionaries can be just a plain
2271 function, like `max` or `np.mean`. In case a function
2272 needs further arguments, then you need to supply a tuple
2273 with the first elements being the function, the second
2274 element being another tuple holding positional arguments,
2275 and an optional third argument holding a dictionary for
2276 key-word arguments.
2277 columns: None, int or str or list of int or str
2278 Columns of the table on which functions are applied.
2279 If None apply functions on all columns.
2280 label: str or list of str
2281 Column label and optional section names of the first
2282 column with the function labels (if `single_row` is `False`).
2283 numbers_only: bool
2284 If True, skip columns that do not contain numbers.
2285 remove_nans: bool
2286 If True, remove nans before passing column values to function.
2287 single_row: bool
2288 If False, add for each function a row to the table.
2289 If True, add function values in a single row.
2290 by: None, int or str or list of int or str
2291 Group the table by the specified columns and apply the functions
2292 to each resulting sub-table separately.
2294 Returns
2295 -------
2296 dest: TableData
2297 A new table with the column headers specified by `columns`.
2298 A first column is inserted with the function labels
2299 (if not `single_row`).
2300 The functions are applied to all columns specified by `columns`
2301 and their return values are written into the new table.
2302 """
2303 if by is not None:
2304 # aggregate on grouped table:
2305 if not isinstance(by, (list, tuple)):
2306 by = [by]
2307 if len(by) > 0:
2308 gd = TableData()
2309 for name, values in self.groupby(*by):
2310 ad = values._aggregate(funcs, columns, label,
2311 numbers_only=numbers_only,
2312 remove_nans=remove_nans,
2313 single_row=True, keep_columns=by)
2314 gd.add(ad)
2315 return gd
2316 # aggregate on whole table:
2317 return self._aggregate(funcs, columns, label,
2318 numbers_only=numbers_only,
2319 remove_nans=remove_nans,
2320 single_row=single_row,
2321 keep_columns=None)
2323 def statistics(self, columns=None, label=None,
2324 remove_nans=False, single_row=False, by=None):
2325 """Descriptive statistics of each column.
2327 Parameter
2328 ---------
2329 columns: None, int or str or list of int or str
2330 Columns of the table on which statistics should be computed.
2331 If None apply functions on all columns.
2332 label: str or list of str
2333 Column label and optional section names of the first
2334 column with the function labels (if `single_row` is `False`).
2335 remove_nans: bool
2336 If True, remove nans before passing column values to function.
2337 single_row: bool
2338 If False, add for each function a row to the table.
2339 If True, add function values in a single row.
2340 by: None, int or str or list of int or str
2341 Group the table by the specified columns and compute statistics
2342 to each resulting sub-table separately.
2344 Returns
2345 -------
2346 dest: TableData
2347 A new table with the column headers specified by `columns`.
2348 For each column that contains numbers some basic
2349 descriptive statistics is computed.
2350 """
2351 if label is None:
2352 label = 'statistics'
2353 funcs = {'mean': np.mean,
2354 'std': np.std,
2355 'min': np.min,
2356 ('quartile1', 'median', 'quartile2'):
2357 (np.quantile, ([0.25, 0.5, 0.75],)),
2358 'max': np.max,
2359 'count': len}
2360 ds = self.aggregate(funcs, columns, label,
2361 numbers_only=True,
2362 remove_nans=remove_nans,
2363 single_row=single_row, by=by)
2364 if by is not None:
2365 if not isinstance(by, (list, tuple)):
2366 by = [by]
2367 if len(by) > 0:
2368 single_row = True
2369 c0 = 0
2370 if not single_row:
2371 ds.set_format(0, '%-10s')
2372 c0 = 1
2373 for c in range(c0, ds.shape[1]):
2374 f = ds.formats[c]
2375 if single_row and ds.label(c) == 'count':
2376 ds.set_unit(c, '')
2377 ds.set_format(c, '%d')
2378 elif f[-1] in 'fge':
2379 i0 = f.find('.')
2380 if i0 > 0:
2381 p = int(f[i0 + 1:-1])
2382 f = f'{f[:i0 + 1]}{p + 1}{f[-1]}'
2383 ds.set_format(c, f)
2384 else:
2385 ds.set_format(c, '%.1f')
2386 return ds
2388 def groupby(self, *columns):
2389 """ Iterate through unique values of a column.
2391 Parameter
2392 ---------
2393 columns: int or str
2394 One or several columns used to group the data.
2395 See self.index() for more information on how to specify a column.
2397 Yields
2398 ------
2399 values: float or str or tuple of float or str
2400 The values of the specified columns.
2401 data: TableData
2402 The sub table where the specified columns equals `values`.
2403 """
2404 # check column indices and values:
2405 cols = []
2406 vals = []
2407 for col in columns:
2408 c = self.index(col)
2409 if c is None:
2410 raise StopIteration
2411 cols.append(c)
2412 vals.append(np.unique(self.data[c]))
2413 for values in product(*vals):
2414 mask = np.ones(len(self), dtype=bool)
2415 for c, v in zip(cols, values):
2416 mask &= self[:, c] == v
2417 if len(cols) == 1:
2418 yield values[0], self[mask]
2419 else:
2420 yield values, self[mask]
2422 def hide(self, column):
2423 """Hide a column or a range of columns.
2425 Hidden columns will not be printed out by the write() function.
2427 Parameters
2428 ----------
2429 column: int or str
2430 The column to be hidden.
2431 See self.index() for more information on how to specify a column.
2432 """
2433 c0, c1 = self.find_col(column)
2434 if c0 is not None:
2435 for c in range(c0, c1):
2436 self.hidden[c] = True
2438 def hide_all(self):
2439 """Hide all columns.
2441 Hidden columns will not be printed out by the write() function.
2442 """
2443 for c in range(len(self.hidden)):
2444 self.hidden[c] = True
2446 def hide_empty_columns(self, missing=default_missing_inputs):
2447 """Hide all columns that do not contain data.
2449 Hidden columns will not be printed out by the write() function.
2451 Parameters
2452 ----------
2453 missing: list of str
2454 Strings indicating missing data.
2455 """
2456 for c in range(len(self.data)):
2457 # check for empty column:
2458 isempty = True
2459 for v in self.data[c]:
2460 if isinstance(v, (float, np.floating)):
2461 if not m.isnan(v):
2462 isempty = False
2463 break
2464 else:
2465 if not v in missing:
2466 isempty = False
2467 break
2468 if isempty:
2469 self.hidden[c] = True
2471 def show(self, column):
2472 """Show a column or a range of columns.
2474 Undoes hiding of a column.
2476 Parameters
2477 ----------
2478 column: int or str
2479 The column to be shown.
2480 See self.index() for more information on how to specify a column.
2481 """
2482 c0, c1 = self.find_col(column)
2483 if c0 is not None:
2484 for c in range(c0, c1):
2485 self.hidden[c] = False
2487 def write(self, fh=sys.stdout, table_format=None, delimiter=None,
2488 unit_style=None, column_numbers=None, sections=None,
2489 align_columns=None, shrink_width=True,
2490 missing=default_missing_str, center_columns=False,
2491 latex_unit_package=None, latex_label_command='',
2492 latex_merge_std=False, descriptions_name='-description',
2493 section_headings=0, maxc=80):
2494 """Write the table to a file or stream.
2496 Parameters
2497 ----------
2498 fh: str or Path or file object
2499 If not a file object, the file with path `fh` is opened.
2500 If `fh` does not have an extension,
2501 the `table_format` is appended as an extension.
2502 Otherwise `fh` is used as a stream for writing.
2503 table_format: None or str
2504 The format to be used for output.
2505 One of 'out', 'dat', 'ascii', 'csv', 'rtai', 'md', 'tex', 'html'.
2506 If None or 'auto' then the format is set to the extension of the
2507 filename given by `fh`. If the filename does not have an extension
2508 `fh` is set to 'csv'. if `fh` is a stream the format is set
2509 to 'out'.
2510 delimiter: str
2511 String or character separating columns, if supported by the
2512 `table_format`.
2513 If None or 'auto' use the default for the specified `table_format`.
2514 unit_style: None or str
2515 - None or 'auto': use default of the specified `table_format`.
2516 - 'row': write an extra row to the table header specifying the
2517 units of the columns.
2518 - 'header': add the units to the column headers.
2519 - 'none': do not specify the units.
2520 column_numbers: str or None
2521 Add a row specifying the column index:
2522 - 'index': indices are integers, first column is 0.
2523 - 'num': indices are integers, first column is 1.
2524 - 'aa': use 'a', 'b', 'c', ..., 'z', 'aa', 'ab', ... for indexing
2525 - 'aa': use 'A', 'B', 'C', ..., 'Z', 'AA', 'AB', ... for indexing
2526 - None or 'none': do not add a row with column indices
2527 TableData.column_numbering is a list with the supported styles.
2528 sections: None or int
2529 Number of section levels to be printed.
2530 If `None` or 'auto' use default of selected `table_format`.
2531 align_columns: boolean
2532 - `True`: set width of column formats to make them align.
2533 - `False`: set width of column formats to 0 - no unnecessary spaces.
2534 - None or 'auto': Use default of the selected `table_format`.
2535 shrink_width: boolean
2536 If `True` disregard width specified by the format strings,
2537 such that columns can become narrower.
2538 missing: str
2539 Indicate missing data by this string.
2540 center_columns: boolean
2541 If True center all columns (markdown, html, and latex).
2542 latex_unit_package: None or 'siunitx' or 'SIunit'
2543 Translate units for the specified LaTeX package.
2544 If None set sub- and superscripts in text mode.
2545 If 'siunitx', also use `S` columns for numbers to align
2546 them on the decimal point.
2547 latex_label_command: str
2548 LaTeX command for formatting header labels.
2549 E.g. 'textbf' for making the header labels bold.
2550 latex_merge_std: str
2551 Merge header of columns with standard deviations with
2552 previous column (LaTeX tables only), but separate them
2553 with $\\pm$. Valid labels for standrad deviations are
2554 listed in `TableData.stdev_labels`.
2555 descriptions_name: None or str
2556 If not None and if `fh` is a file path, then write the column
2557 descriptions to a file with the same name as `fh`, but with
2558 `descriptions_name` appended.
2559 section_headings: None or int
2560 How to treat header sections in the column descriptions.
2561 If set, set header sections as headings with the top-level
2562 section at the level as specified. 0 is the top level.
2563 If None, just produce a nested list.
2564 maxc: int
2565 Maximum character count for each line in the column descriptions.
2567 Returns
2568 -------
2569 file_name: Path or None
2570 The full name of the file into which the data were written.
2572 Supported file formats
2573 ----------------------
2575 ## `dat`: data text file
2576 ``` plain
2577 # info reaction
2578 # size weight delay jitter
2579 # m kg ms mm
2580 2.34 123 98.7 23
2581 56.70 3457 54.3 45
2582 8.90 43 67.9 345
2583 ```
2585 ## `ascii`: ascii-art table
2586 ``` plain
2587 |---------------------------------|
2588 | info | reaction |
2589 | size | weight | delay | jitter |
2590 | m | kg | ms | mm |
2591 |-------|--------|-------|--------|
2592 | 2.34 | 123 | 98.7 | 23 |
2593 | 56.70 | 3457 | 54.3 | 45 |
2594 | 8.90 | 43 | 67.9 | 345 |
2595 |---------------------------------|
2596 ```
2598 ## `csv`: comma separated values
2599 ``` plain
2600 size/m,weight/kg,delay/ms,jitter/mm
2601 2.34,123,98.7,23
2602 56.70,3457,54.3,45
2603 8.90,43,67.9,345
2604 ```
2606 ## `rtai`: rtai-style table
2607 ``` plain
2608 RTH| info | reaction
2609 RTH| size | weight| delay| jitter
2610 RTH| m | kg | ms | mm
2611 RTD| 2.34| 123| 98.7| 23
2612 RTD| 56.70| 3457| 54.3| 45
2613 RTD| 8.90| 43| 67.9| 345
2614 ```
2616 ## `md`: markdown
2617 ``` plain
2618 | size/m | weight/kg | delay/ms | jitter/mm |
2619 |------:|-------:|------:|-------:|
2620 | 2.34 | 123 | 98.7 | 23 |
2621 | 56.70 | 3457 | 54.3 | 45 |
2622 | 8.90 | 43 | 67.9 | 345 |
2623 ```
2625 ## `tex`: latex tabular
2626 ``` tex
2627 \\begin{tabular}{rrrr}
2628 \\hline
2629 \\multicolumn{2}{l}{info} & \\multicolumn{2}{l}{reaction} \\
2630 \\multicolumn{1}{l}{size} & \\multicolumn{1}{l}{weight} & \\multicolumn{1}{l}{delay} & \\multicolumn{1}{l}{jitter} \\
2631 \\multicolumn{1}{l}{m} & \\multicolumn{1}{l}{kg} & \\multicolumn{1}{l}{ms} & \\multicolumn{1}{l}{mm} \\
2632 \\hline
2633 2.34 & 123 & 98.7 & 23 \\
2634 56.70 & 3457 & 54.3 & 45 \\
2635 8.90 & 43 & 67.9 & 345 \\
2636 \\hline
2637 \\end{tabular}
2638 ```
2640 ## `html`: html
2641 ``` html
2642 <table>
2643 <thead>
2644 <tr class="header">
2645 <th align="left" colspan="2">info</th>
2646 <th align="left" colspan="2">reaction</th>
2647 </tr>
2648 <tr class="header">
2649 <th align="left">size</th>
2650 <th align="left">weight</th>
2651 <th align="left">delay</th>
2652 <th align="left">jitter</th>
2653 </tr>
2654 <tr class="header">
2655 <th align="left">m</th>
2656 <th align="left">kg</th>
2657 <th align="left">ms</th>
2658 <th align="left">mm</th>
2659 </tr>
2660 </thead>
2661 <tbody>
2662 <tr class"odd">
2663 <td align="right">2.34</td>
2664 <td align="right">123</td>
2665 <td align="right">98.7</td>
2666 <td align="right">23</td>
2667 </tr>
2668 <tr class"even">
2669 <td align="right">56.70</td>
2670 <td align="right">3457</td>
2671 <td align="right">54.3</td>
2672 <td align="right">45</td>
2673 </tr>
2674 <tr class"odd">
2675 <td align="right">8.90</td>
2676 <td align="right">43</td>
2677 <td align="right">67.9</td>
2678 <td align="right">345</td>
2679 </tr>
2680 </tbody>
2681 </table>
2682 ```
2684 """
2685 # fix parameter:
2686 if table_format is not None:
2687 table_format = table_format.lower()
2688 if table_format == 'auto':
2689 table_format = None
2690 if delimiter == 'auto':
2691 delimiter = None
2692 if unit_style == 'auto':
2693 unit_style = None
2694 if column_numbers == 'none':
2695 column_numbers = None
2696 if sections == 'auto':
2697 sections = None
2698 if align_columns == 'auto':
2699 align_columns = None
2700 # open file:
2701 own_file = False
2702 file_name = None
2703 if not hasattr(fh, 'write'):
2704 fh = Path(fh)
2705 ext = fh.suffix
2706 if table_format is None:
2707 if len(ext) > 1 and ext[1:] in self.ext_formats:
2708 table_format = self.ext_formats[ext[1:]]
2709 else:
2710 table_format = 'csv'
2711 if not ext or not ext[1:].lower() in self.ext_formats:
2712 fh = fh.with_suffix('.' + self.extensions[table_format])
2713 file_name = fh
2714 fh = open(file_name, 'w')
2715 own_file = True
2716 if table_format is None:
2717 table_format = 'out'
2718 # set style:
2719 if table_format[0] == 'd':
2720 align_columns = True
2721 begin_str = ''
2722 end_str = ''
2723 header_start = '# '
2724 header_sep = ' '
2725 header_close = ''
2726 header_end = '\n'
2727 data_start = ' '
2728 data_sep = ' '
2729 data_close = ''
2730 data_end = '\n'
2731 top_line = False
2732 header_line = False
2733 bottom_line = False
2734 if delimiter is not None:
2735 header_sep = delimiter
2736 data_sep = delimiter
2737 if sections is None:
2738 sections = 1000
2739 elif table_format[0] == 'a':
2740 align_columns = True
2741 begin_str = ''
2742 end_str = ''
2743 header_start = '| '
2744 header_sep = ' | '
2745 header_close = ''
2746 header_end = ' |\n'
2747 data_start = '| '
2748 data_sep = ' | '
2749 data_close = ''
2750 data_end = ' |\n'
2751 top_line = True
2752 header_line = True
2753 bottom_line = True
2754 if delimiter is not None:
2755 header_sep = delimiter
2756 data_sep = delimiter
2757 if sections is None:
2758 sections = 1000
2759 elif table_format[0] == 'c':
2760 # csv according to http://www.ietf.org/rfc/rfc4180.txt :
2761 column_numbers=None
2762 if unit_style is None:
2763 unit_style = 'header'
2764 if align_columns is None:
2765 align_columns = False
2766 begin_str = ''
2767 end_str = ''
2768 header_start=''
2769 header_sep = ','
2770 header_close = ''
2771 header_end='\n'
2772 data_start=''
2773 data_sep = ','
2774 data_close = ''
2775 data_end='\n'
2776 top_line = False
2777 header_line = False
2778 bottom_line = False
2779 if delimiter is not None:
2780 header_sep = delimiter
2781 data_sep = delimiter
2782 if sections is None:
2783 sections = 0
2784 elif table_format[0] == 'r':
2785 align_columns = True
2786 begin_str = ''
2787 end_str = ''
2788 header_start = 'RTH| '
2789 header_sep = '| '
2790 header_close = ''
2791 header_end = '\n'
2792 data_start = 'RTD| '
2793 data_sep = '| '
2794 data_close = ''
2795 data_end = '\n'
2796 top_line = False
2797 header_line = False
2798 bottom_line = False
2799 if sections is None:
2800 sections = 1000
2801 elif table_format[0] == 'm':
2802 if unit_style is None or unit_style == 'row':
2803 unit_style = 'header'
2804 align_columns = True
2805 begin_str = ''
2806 end_str = ''
2807 header_start='| '
2808 header_sep = ' | '
2809 header_close = ''
2810 header_end=' |\n'
2811 data_start='| '
2812 data_sep = ' | '
2813 data_close = ''
2814 data_end=' |\n'
2815 top_line = False
2816 header_line = True
2817 bottom_line = False
2818 if sections is None:
2819 sections = 0
2820 elif table_format[0] == 'h':
2821 align_columns = False
2822 begin_str = '<table>\n<thead>\n'
2823 end_str = '</tbody>\n</table>\n'
2824 if center_columns:
2825 header_start=' <tr>\n <th align="center"'
2826 header_sep = '</th>\n <th align="center"'
2827 else:
2828 header_start=' <tr>\n <th align="left"'
2829 header_sep = '</th>\n <th align="left"'
2830 header_close = '>'
2831 header_end='</th>\n </tr>\n'
2832 data_start=' <tr>\n <td'
2833 data_sep = '</td>\n <td'
2834 data_close = '>'
2835 data_end='</td>\n </tr>\n'
2836 top_line = False
2837 header_line = False
2838 bottom_line = False
2839 if sections is None:
2840 sections = 1000
2841 elif table_format[0] == 't':
2842 if align_columns is None:
2843 align_columns = False
2844 begin_str = '\\begin{tabular}'
2845 end_str = '\\end{tabular}\n'
2846 header_start=' '
2847 header_sep = ' & '
2848 header_close = ''
2849 header_end=' \\\\\n'
2850 data_start=' '
2851 data_sep = ' & '
2852 data_close = ''
2853 data_end=' \\\\\n'
2854 top_line = True
2855 header_line = True
2856 bottom_line = True
2857 if sections is None:
2858 sections = 1000
2859 else:
2860 if align_columns is None:
2861 align_columns = True
2862 begin_str = ''
2863 end_str = ''
2864 header_start = ''
2865 header_sep = ' '
2866 header_close = ''
2867 header_end = '\n'
2868 data_start = ''
2869 data_sep = ' '
2870 data_close = ''
2871 data_end = '\n'
2872 top_line = False
2873 header_line = False
2874 bottom_line = False
2875 if sections is None:
2876 sections = 1000
2877 # check units:
2878 if unit_style is None:
2879 unit_style = 'row'
2880 have_units = False
2881 for u in self.units:
2882 if u and u != '1' and u != '-':
2883 have_units = True
2884 break
2885 if not have_units:
2886 unit_style = 'none'
2887 # find std columns:
2888 stdev_col = np.zeros(len(self.header), dtype=bool)
2889 for c in range(len(self.header) - 1):
2890 if self.header[c + 1][0].lower() in self.stdev_labels and \
2891 not self.hidden[c+1]:
2892 stdev_col[c] = True
2893 # begin table:
2894 fh.write(begin_str)
2895 if table_format[0] == 't':
2896 fh.write('{')
2897 merged = False
2898 for h, f, s in zip(self.hidden, self.formats, stdev_col):
2899 if merged:
2900 fh.write('l')
2901 merged = False
2902 continue
2903 if h:
2904 continue
2905 if latex_merge_std and s:
2906 fh.write('r@{$\\,\\pm\\,$}')
2907 merged = True
2908 elif center_columns:
2909 fh.write('c')
2910 elif f[1] == '-':
2911 fh.write('l')
2912 else:
2913 if latex_unit_package is not None and \
2914 latex_unit_package.lower() == 'siunitx':
2915 fh.write('S')
2916 else:
2917 fh.write('r')
2918 fh.write('}\n')
2919 # retrieve column formats and widths:
2920 widths = [] # width from format string
2921 widths_pos = [] # start and end index of width specifyer in format
2922 for c, f in enumerate(self.formats):
2923 w = 0
2924 # position of width specification:
2925 i0 = 1
2926 if len(f) > 1 and f[1] == '-' :
2927 i0 = 2
2928 i1 = f.find('.')
2929 if i1 < 0:
2930 i1 = i0
2931 while i1 < len(f) and f[i1].isdigit():
2932 i1 += 1
2933 if not shrink_width and i1 > i0:
2934 if f[i0:i1]:
2935 w = int(f[i0:i1])
2936 widths_pos.append((i0, i1))
2937 # adapt width to header label:
2938 hw = len(self.header[c][0])
2939 if unit_style == 'header' and self.units[c] and \
2940 self.units[c] != '1' and self.units[c] != '-':
2941 hw += 1 + len(self.units[c])
2942 if w < hw:
2943 w = hw
2944 # adapt width to data:
2945 if f[-1] == 's':
2946 for v in self.data[c]:
2947 if isinstance(v, str) and w < len(v):
2948 w = len(v)
2949 else:
2950 fs = f'{f[:i0]}0{f[i1:]}'
2951 for v in self.data[c]:
2952 if v is None or (isinstance(v, (float, np.floating)) and
2953 m.isnan(v)):
2954 s = missing
2955 else:
2956 try:
2957 s = fs % v
2958 except ValueError:
2959 s = missing
2960 except TypeError:
2961 s = str(v)
2962 if w < len(s):
2963 w = len(s)
2964 widths.append(w)
2965 # adapt width to sections:
2966 sec_indices = [0] * self.nsecs # previous column with this level
2967 sec_widths = [0] * self.nsecs # total width of section level
2968 sec_columns = [0] * self.nsecs # number of columns in section level
2969 for c in range(len(self.header)):
2970 w = widths[c]
2971 for l in range(min(self.nsecs, sections)):
2972 if 1 + l < len(self.header[c]):
2973 if c > 0 and sec_columns[l] > 0 and \
2974 1 + l < len(self.header[sec_indices[l]]) and \
2975 len(self.header[sec_indices[l]][1 + l]) > sec_widths[l]:
2976 dw = len(self.header[sec_indices[l]][1 + l]) - sec_widths[l]
2977 nc = sec_columns[l]
2978 ddw = np.zeros(nc, dtype=int) + dw // nc
2979 ddw[:dw % nc] += 1
2980 wk = 0
2981 for ck in range(sec_indices[l], c):
2982 if not self.hidden[ck]:
2983 widths[ck] += ddw[wk]
2984 wk += 1
2985 sec_indices[l] = c
2986 sec_widths[l] = 0
2987 sec_columns[l] = 0
2988 if not self.hidden[c]:
2989 if sec_widths[l] > 0:
2990 sec_widths[l] += len(header_sep)
2991 sec_widths[l] += w
2992 sec_columns[l] += 1
2993 # set width of format string:
2994 formats = []
2995 for c, (f, w) in enumerate(zip(self.formats, widths)):
2996 formats.append(f'{f[:widths_pos[c][0]]}{w}{f[widths_pos[c][1]:]}')
2997 # top line:
2998 if top_line:
2999 if table_format[0] == 't':
3000 fh.write(' \\hline \\\\[-2ex]\n')
3001 else:
3002 first = True
3003 fh.write(header_start.replace(' ', '-'))
3004 for c in range(len(self.header)):
3005 if self.hidden[c]:
3006 continue
3007 if not first:
3008 fh.write('-'*len(header_sep))
3009 first = False
3010 fh.write(header_close)
3011 w = widths[c]
3012 fh.write(w*'-')
3013 fh.write(header_end.replace(' ', '-'))
3014 # section and column headers:
3015 nsec0 = self.nsecs - sections
3016 if nsec0 < 0:
3017 nsec0 = 0
3018 for ns in range(nsec0, self.nsecs+1):
3019 nsec = self.nsecs - ns
3020 first = True
3021 last = False
3022 merged = False
3023 fh.write(header_start)
3024 for c in range(len(self.header)):
3025 if nsec < len(self.header[c]):
3026 # section width and column count:
3027 sw = -len(header_sep)
3028 columns = 0
3029 if not self.hidden[c]:
3030 sw = widths[c]
3031 columns = 1
3032 for k in range(c+1, len(self.header)):
3033 if nsec < len(self.header[k]):
3034 break
3035 if self.hidden[k]:
3036 continue
3037 sw += len(header_sep) + widths[k]
3038 columns += 1
3039 else:
3040 last = True
3041 if len(header_end.strip()) == 0:
3042 sw = 0 # last entry needs no width
3043 if columns == 0:
3044 continue
3045 if not first and not merged:
3046 fh.write(header_sep)
3047 first = False
3048 if table_format[0] == 'c':
3049 sw -= len(header_sep)*(columns - 1)
3050 elif table_format[0] == 'h':
3051 if columns>1:
3052 fh.write(' colspan="%d"' % columns)
3053 elif table_format[0] == 't':
3054 if merged:
3055 merged = False
3056 continue
3057 if latex_merge_std and nsec == 0 and stdev_col[c]:
3058 merged = True
3059 fh.write('\\multicolumn{%d}{c}{' % (columns+1))
3060 elif center_columns:
3061 fh.write('\\multicolumn{%d}{c}{' % columns)
3062 else:
3063 fh.write('\\multicolumn{%d}{l}{' % columns)
3064 if latex_label_command:
3065 fh.write('\\%s{' % latex_label_command)
3066 fh.write(header_close)
3067 hs = self.header[c][nsec]
3068 if nsec == 0 and unit_style == 'header':
3069 if self.units[c] and self.units[c] != '1' and self.units[c] != '-':
3070 hs += '/' + self.units[c]
3071 if align_columns and not table_format[0] in 'th':
3072 f = '%%-%ds' % sw
3073 fh.write(f % hs)
3074 else:
3075 fh.write(hs)
3076 if table_format[0] == 'c':
3077 if not last:
3078 fh.write(header_sep*(columns - 1))
3079 elif table_format[0] == 't':
3080 if latex_label_command:
3081 fh.write('}')
3082 fh.write('}')
3083 fh.write(header_end)
3084 # units:
3085 if unit_style == 'row':
3086 first = True
3087 merged = False
3088 fh.write(header_start)
3089 for c in range(len(self.header)):
3090 if self.hidden[c] or merged:
3091 merged = False
3092 continue
3093 if not first:
3094 fh.write(header_sep)
3095 first = False
3096 fh.write(header_close)
3097 unit = self.units[c]
3098 if not unit:
3099 unit = '-'
3100 if table_format[0] == 't':
3101 if latex_merge_std and stdev_col[c]:
3102 merged = True
3103 fh.write('\\multicolumn{2}{c}{%s}' % latex_unit(unit, latex_unit_package))
3104 elif center_columns:
3105 fh.write('\\multicolumn{1}{c}{%s}' % latex_unit(unit, latex_unit_package))
3106 else:
3107 fh.write('\\multicolumn{1}{l}{%s}' % latex_unit(unit, latex_unit_package))
3108 else:
3109 if align_columns and not table_format[0] in 'h':
3110 f = '%%-%ds' % widths[c]
3111 fh.write(f % unit)
3112 else:
3113 fh.write(unit)
3114 fh.write(header_end)
3115 # column numbers:
3116 if column_numbers is not None:
3117 first = True
3118 fh.write(header_start)
3119 for c in range(len(self.header)):
3120 if self.hidden[c]:
3121 continue
3122 if not first:
3123 fh.write(header_sep)
3124 first = False
3125 fh.write(header_close)
3126 i = c
3127 if column_numbers == 'num':
3128 i = c+1
3129 aa = index2aa(c, 'a')
3130 if column_numbers == 'AA':
3131 aa = index2aa(c, 'A')
3132 if table_format[0] == 't':
3133 if column_numbers == 'num' or column_numbers == 'index':
3134 fh.write('\\multicolumn{1}{l}{%d}' % i)
3135 else:
3136 fh.write('\\multicolumn{1}{l}{%s}' % aa)
3137 else:
3138 if column_numbers == 'num' or column_numbers == 'index':
3139 if align_columns:
3140 f = '%%%dd' % widths[c]
3141 fh.write(f % i)
3142 else:
3143 fh.write('%d' % i)
3144 else:
3145 if align_columns:
3146 f = '%%-%ds' % widths[c]
3147 fh.write(f % aa)
3148 else:
3149 fh.write(aa)
3150 fh.write(header_end)
3151 # header line:
3152 if header_line:
3153 if table_format[0] == 'm':
3154 fh.write('|')
3155 for c in range(len(self.header)):
3156 if self.hidden[c]:
3157 continue
3158 w = widths[c]+2
3159 if center_columns:
3160 fh.write(':' + (w-2)*'-' + ':|')
3161 elif formats[c][1] == '-':
3162 fh.write(w*'-' + '|')
3163 else:
3164 fh.write((w - 1)*'-' + ':|')
3165 fh.write('\n')
3166 elif table_format[0] == 't':
3167 fh.write(' \\hline \\\\[-2ex]\n')
3168 else:
3169 first = True
3170 fh.write(header_start.replace(' ', '-'))
3171 for c in range(len(self.header)):
3172 if self.hidden[c]:
3173 continue
3174 if not first:
3175 fh.write(header_sep.replace(' ', '-'))
3176 first = False
3177 fh.write(header_close)
3178 w = widths[c]
3179 fh.write(w*'-')
3180 fh.write(header_end.replace(' ', '-'))
3181 # start table data:
3182 if table_format[0] == 'h':
3183 fh.write('</thead>\n<tbody>\n')
3184 # data:
3185 for k in range(self.rows()):
3186 first = True
3187 merged = False
3188 fh.write(data_start)
3189 for c, f in enumerate(formats):
3190 if self.hidden[c] or merged:
3191 merged = False
3192 continue
3193 if not first:
3194 fh.write(data_sep)
3195 first = False
3196 if table_format[0] == 'h':
3197 if center_columns:
3198 fh.write(' align="center"')
3199 elif f[1] == '-':
3200 fh.write(' align="left"')
3201 else:
3202 fh.write(' align="right"')
3203 fh.write(data_close)
3204 if k >= len(self.data[c]) or self.data[c][k] is None or \
3205 (isinstance(self.data[c][k], (float, np.floating)) and m.isnan(self.data[c][k])):
3206 # missing data:
3207 if table_format[0] == 't' and latex_merge_std and stdev_col[c]:
3208 merged = True
3209 fh.write('\\multicolumn{2}{c}{%s}' % missing)
3210 elif align_columns:
3211 if f[1] == '-':
3212 fn = '%%-%ds' % widths[c]
3213 else:
3214 fn = '%%%ds' % widths[c]
3215 fh.write(fn % missing)
3216 else:
3217 fh.write(missing)
3218 else:
3219 # data value:
3220 try:
3221 ds = f % self.data[c][k]
3222 except ValueError:
3223 ds = missing
3224 except TypeError:
3225 ds = str(self.data[c][k])
3226 if not align_columns:
3227 ds = ds.strip()
3228 fh.write(ds)
3229 fh.write(data_end)
3230 # bottom line:
3231 if bottom_line:
3232 if table_format[0] == 't':
3233 fh.write(' \\hline\n')
3234 else:
3235 first = True
3236 fh.write(header_start.replace(' ', '-'))
3237 for c in range(len(self.header)):
3238 if self.hidden[c]:
3239 continue
3240 if not first:
3241 fh.write('-'*len(header_sep))
3242 first = False
3243 fh.write(header_close)
3244 w = widths[c]
3245 fh.write(w*'-')
3246 fh.write(header_end.replace(' ', '-'))
3247 # end table:
3248 fh.write(end_str)
3249 # close file:
3250 if own_file:
3251 fh.close()
3252 # write descriptions:
3253 if file_name is not None and descriptions_name:
3254 write_descriptions = False
3255 for c in range(len(self.descriptions)):
3256 if self.descriptions[c]:
3257 write_descriptions = True
3258 if write_descriptions:
3259 descr_path = file_name.with_name(file_name.stem +
3260 descriptions_name)
3261 if table_format[0] not in 'th': # neither tex nore html
3262 table_format = 'md'
3263 if len(descr_path.suffix) <= 1:
3264 descr_path = descr_path.with_suffix('.' + self.extensions[table_format])
3265 self.write_descriptions(descr_path, table_format=table_format,
3266 sections=sections,
3267 section_headings=section_headings,
3268 latex_unit_package=latex_unit_package,
3269 maxc=maxc)
3270 # return file name:
3271 return file_name
3273 def write_file_stream(self, base_name, file_name, **kwargs):
3274 """Write table to file or stream and return appropriate file name.
3276 Parameters
3277 ----------
3278 base_name: str, Path, or stream
3279 If str, path and basename of file.
3280 `file_name` and an extension are appended.
3281 If stream, write table data into this stream.
3282 file_name: str
3283 Name of file that is appended to `base_name`.
3284 kwargs:
3285 Arguments passed on to `TableData.write()`.
3286 In particular, 'table_format' is used to set the file extension
3287 that is appended to the returned `file_name`.
3289 Returns
3290 -------
3291 file_name: Path
3292 Path and full name of the written file in case of `base_name`
3293 being a string. Otherwise, the file name and extension that
3294 should be appended to a base name.
3295 """
3296 if hasattr(base_name, 'write'):
3297 table_format = kwargs.get('table_format', None)
3298 if table_format is None or table_format == 'auto':
3299 table_format = 'csv'
3300 file_name = Path(file_name)
3301 file_name = file_name.with_suffix('.' + TableData.extensions[table_format])
3302 self.write(base_name, **kwargs)
3303 return file_name
3304 else:
3305 base_name = Path(base_name)
3306 base_name = base_name.with_name(base_name.stem + file_name)
3307 file_name = self.write(base_name, **kwargs)
3308 return file_name
3310 def __str__(self):
3311 """Write table to a string.
3312 """
3313 stream = StringIO()
3314 self.write(stream, table_format='out')
3315 return stream.getvalue()
3317 def write_descriptions(self, fh=sys.stdout, table_format=None,
3318 sections=None, section_headings=0,
3319 latex_unit_package=None, maxc=80):
3320 """Write column descriptions of the table to a file or stream.
3322 Parameters
3323 ----------
3324 fh: str or Path or file object
3325 If not a stream, the file with path `fh` is opened.
3326 If `fh` does not have an extension,
3327 the `table_format` is appended as an extension.
3328 Otherwise `fh` is used as a stream for writing.
3329 table_format: None or str
3330 The format to be used for output.
3331 One of 'md', 'tex', or 'html'.
3332 If None or 'auto' then the format is set to the extension
3333 of the filename given by `fh`.
3334 If `fh` is a stream the format is set to 'md'.
3335 sections: None or int
3336 Number of section levels to be printed.
3337 If `None` or 'auto' use default of selected `table_format`.
3338 section_headings: None or int
3339 If set, set header sections as headings with the top-level
3340 section at the level as specified. 0 is the top level.
3341 If False, just produce a nested list.
3342 latex_unit_package: None or 'siunitx' or 'SIunit'
3343 Translate units for the specified LaTeX package.
3344 If None set sub- and superscripts in text mode.
3345 If 'siunitx', also use `S` columns for numbers to align
3346 them on the decimal point.
3347 maxc: int
3348 Maximum character count for each line.
3349 """
3350 # fix parameter:
3351 if table_format == 'auto':
3352 table_format = None
3353 if sections is None:
3354 sections = 1000
3355 nsecs = min(self.nsecs, sections)
3356 # open file:
3357 own_file = False
3358 file_name = None
3359 if not hasattr(fh, 'write'):
3360 fh = Path(fh)
3361 ext = fh.suffix
3362 if table_format is None:
3363 if len(ext) > 1 and ext[1:] in self.ext_formats:
3364 table_format = self.ext_formats[ext[1:]]
3365 else:
3366 fh = fh.with_suffix('.' + self.extensions[table_format])
3367 file_name = fh
3368 fh = open(fh, 'w')
3369 own_file = True
3370 if table_format is None:
3371 table_format = 'md'
3372 # write descriptions:
3373 headers = ['']*(1 + nsecs)
3374 prev_headers = ['']*(1 + nsecs)
3375 if table_format == 'md':
3376 for c in range(len(self.header)):
3377 headers[:len(self.header[c])] = self.header[c]
3378 if not self.hidden[c]:
3379 changed = False
3380 for k in reversed(range(nsecs)):
3381 if changed or prev_headers[k + 1] != headers[k + 1]:
3382 changed = True
3383 if section_headings is None:
3384 fh.write(f'{" "*2*(nsecs - k - 1)}- '
3385 f'{headers[k + 1]}\n')
3386 else:
3387 level = nsecs - k - 1 + section_headings + 1
3388 fh.write(f'\n{"#"*level} {headers[k + 1]}\n')
3389 prev_headers[k + 1] = headers[k + 1]
3390 indent = 2*nsecs if section_headings is None else 0
3391 fh.write(f'{" "*indent}- **{headers[0]}**')
3392 if self.units[c]:
3393 fh.write(f' [{self.units[c]}]')
3394 fh.write(' \n')
3395 break_text(fh, self.descriptions[c], maxc,
3396 indent=indent + 2)
3397 prev_headers[0] = headers[0]
3398 elif table_format == 'html':
3399 level = -1
3400 for c in range(len(self.header)):
3401 headers[:len(self.header[c])] = self.header[c]
3402 if not self.hidden[c]:
3403 changed = False
3404 for k in reversed(range(nsecs)):
3405 if changed or prev_headers[k + 1] != headers[k + 1]:
3406 new_level = nsecs - k - 1
3407 if not changed:
3408 if section_headings is None:
3409 while level > new_level:
3410 fh.write(f'{" "*2*level}</ul>\n')
3411 level -= 1
3412 elif level >= 0:
3413 fh.write(f'{" "*2*level}</ul>\n')
3414 level -= 1
3415 changed = True
3416 if section_headings is None:
3417 while level < new_level:
3418 level += 1
3419 fh.write(f'{" "*2*level}<ul>\n')
3420 fh.write(f'{" "*2*(level + 1)}<li><b>{headers[k + 1]}</b></li>\n')
3421 else:
3422 fh.write(f'\n<h{new_level + section_headings + 1}>{headers[k + 1]}</h{new_level + section_headings + 1}>\n')
3423 prev_headers[k + 1] = headers[k + 1]
3424 if changed:
3425 level += 1
3426 fh.write(f'{" "*2*level}<ul>\n')
3428 fh.write(f'{" "*2*(level + 1)}<li><b>{headers[0]}</b>')
3429 if self.units[c]:
3430 fh.write(f'[{self.units[c]}]')
3431 fh.write('<br>\n')
3432 break_text(fh, self.descriptions[c], maxc,
3433 indent=2*(level + 1))
3434 fh.write(f'{" "*2*(level + 1)}</li>\n')
3435 prev_headers[0] = headers[0]
3436 while level >= 0:
3437 fh.write(f'{" "*2*level}</ul>\n')
3438 level -= 1
3439 elif table_format == 'tex':
3440 headings = [r'\section', r'\subsection', r'\subsubsection',
3441 r'\paragraph', r'\subparagraph']
3442 level = -1
3443 for c in range(len(self.header)):
3444 headers[:len(self.header[c])] = self.header[c]
3445 if not self.hidden[c]:
3446 changed = False
3447 for k in reversed(range(nsecs)):
3448 if changed or prev_headers[k + 1] != headers[k + 1]:
3449 new_level = nsecs - k - 1
3450 if not changed:
3451 if section_headings is None:
3452 while level > new_level:
3453 fh.write(f'{" "*2*level}\\end{{enumerate}}\n')
3454 level -= 1
3455 elif level >= 0:
3456 fh.write(f'{" "*2*level}\\end{{enumerate}}\n')
3457 level -= 1
3458 changed = True
3459 if section_headings is None:
3460 while level < new_level:
3461 level += 1
3462 fh.write(f'{" "*2*level}\\begin{{enumerate}}\n')
3463 fh.write(f'{" "*2*(level + 1)}\\item \\textbf{{{headers[k + 1]}}}\n')
3464 else:
3465 fh.write(f'\n{headings[new_level + section_headings]}{{{headers[k + 1]}}}\n')
3466 prev_headers[k + 1] = headers[k + 1]
3467 if changed:
3468 level += 1
3469 fh.write(f'{" "*2*level}\\begin{{enumerate}}\n')
3470 fh.write(f'{" "*2*(level + 1)}\\item \\textbf{{{headers[0]}}}')
3471 if self.units[c]:
3472 fh.write(f' [{latex_unit(self.units[c], latex_unit_package)}]')
3473 fh.write('\n')
3474 break_text(fh, self.descriptions[c], maxc,
3475 indent=2*(level + 1))
3476 prev_headers[0] = headers[0]
3477 while level >= 0:
3478 fh.write(f'{" "*2*level}\\end{{enumerate}}\n')
3479 level -= 1
3480 else:
3481 raise ValueError(f'File format "{table_format}" not supported for writing column descriptions')
3482 # close file:
3483 if own_file:
3484 fh.close()
3485 # return file name:
3486 return file_name
3488 def load_descriptions(self, fh):
3489 """Load column descriptions from file or stream.
3491 Parameters
3492 ----------
3493 fh: str, Path, or stream
3494 If not a stream, the file with path `fh` is opened for reading.
3495 """
3496 def set_descr_unit(sections, label, unit, description):
3497 secs = '>'.join(sections)
3498 if len(secs) > 0:
3499 secs += '>'
3500 secs += label
3501 col = self.index(secs)
3502 self.set_unit(col, unit)
3503 self.set_description(col, description)
3505 # open file:
3506 own_file = False
3507 if not hasattr(fh, 'readline'):
3508 fh = open(fh, 'r')
3509 own_file = True
3510 # read first non-empty line:
3511 for line in fh:
3512 line = line.strip()
3513 if len(line) > 0:
3514 break
3515 if len(line) > 0 and line[0] in '#-':
3516 # markdown:
3517 description = None
3518 nested = line[0] == '-' and not '**' in line
3519 if nested:
3520 sections = [line[1:].strip()]
3521 elif line[0] == '#':
3522 top_level = len(line.split()[0])
3523 sections = [line[top_level + 1:].strip()]
3524 else:
3525 top_level = 0
3526 sections = []
3527 if '**' in line:
3528 lp = line.split('**')
3529 label = lp[1]
3530 unit = None
3531 if len(lp) > 1 and lp[-1].strip()[0] == '[':
3532 unit = lp[-1].strip().lstrip('[').rstrip(']')
3533 description = ''
3534 for line in fh:
3535 sline = line.strip()
3536 if (len(sline) == 0 or sline[0] in '#-') and description:
3537 set_descr_unit(sections, label, unit, description)
3538 description = None
3539 if len(sline) == 0:
3540 continue
3541 elif sline[0] == '#':
3542 heading_level = len(line.split()[0])
3543 section_name = line[heading_level + 1:].strip()
3544 heading_level -= top_level
3545 while len(sections) > heading_level:
3546 sections.pop()
3547 sections.append(section_name)
3548 elif sline[0] == '-':
3549 if '**' in sline:
3550 lp = sline.split('**')
3551 label = lp[1]
3552 unit = None
3553 if len(lp) > 1 and lp[-1].strip()[0] == '[':
3554 unit = lp[-1].strip().lstrip('[').rstrip(']')
3555 description = ''
3556 elif nested:
3557 heading_level = line.find('-')//2
3558 while len(sections) > heading_level:
3559 sections.pop()
3560 sections.append(sline.lstrip('-').strip())
3561 elif description is not None:
3562 if len(description) > 0:
3563 description += ' '
3564 description += sline
3565 if description:
3566 set_descr_unit(sections, label, unit, description)
3567 else:
3568 print('unsupported file format for reading descriptions')
3569 # close file:
3570 if own_file:
3571 fh.close()
3573 def load(self, fh, missing=default_missing_inputs, sep=None, stop=None):
3574 """Load table from file or stream.
3576 File type and properties are automatically inferred.
3578 Parameters
3579 ----------
3580 fh: str, Path, or stream
3581 If not a stream, the file with path `fh` is opened for reading.
3582 missing: str or list of str
3583 Missing data are indicated by this string and
3584 are translated to np.nan.
3585 sep: str or None
3586 Column separator.
3587 stop: str or None
3588 If a line matches `stop`, stop reading the file. `stop`
3589 can be an empty string to stop reading at the first empty
3590 line.
3592 Raises
3593 ------
3594 FileNotFoundError:
3595 If `fh` is a path that does not exist.
3597 """
3599 def read_key_line(line, sep, table_format):
3600 if sep is None:
3601 cols, indices = zip(*[(m.group(0), m.start()) for m in re.finditer(r'( ?[\S]+)+(?=[ ][ ]+|\Z)', line.strip())])
3602 elif table_format == 'csv':
3603 cols, indices = zip(*[(c.strip(), i) for i, c in enumerate(line.strip().split(sep)) if c.strip()])
3604 else:
3605 seps = r'[^'+re.escape(sep)+']+'
3606 cols, indices = zip(*[(m.group(0), m.start()) for m in re.finditer(seps, line.strip())])
3607 colss = []
3608 indicess = []
3609 if table_format == 'tex':
3610 i = 0
3611 for c in cols:
3612 if 'multicolumn' in c:
3613 fields = c.split('{')
3614 n = int(fields[1].strip().rstrip('}').rstrip())
3615 colss.append(fields[3].strip().rstrip('}').rstrip())
3616 indicess.append(i)
3617 i += n
3618 else:
3619 colss.append(c.strip())
3620 indicess.append(i)
3621 i += 1
3622 else:
3623 for k, (c, i) in enumerate(zip(cols, indices)):
3624 if table_format != 'csv':
3625 if k == 0:
3626 c = c.lstrip('|')
3627 if k == len(cols) - 1:
3628 c = c.rstrip('|')
3629 cs = c.strip()
3630 if len(cs) >= 2 and cs[0] == '"' and cs[-1] == '"':
3631 cs = cs.strip('"')
3632 colss.append(cs)
3633 indicess.append(i)
3634 return colss, indicess
3636 def read_data_line(line, sep, post, precd, alld, numc, exped,
3637 fixed, strf, missing, nans):
3638 # read line:
3639 cols = []
3640 if sep is None:
3641 cols = [m.group(0) for m in re.finditer(r'\S+', line.strip())]
3642 else:
3643 if sep.isspace():
3644 seps = r'[^'+re.escape(sep)+']+'
3645 cols = [m.group(0) for m in re.finditer(seps, line.strip())]
3646 else:
3647 cols = line.split(sep)
3648 if len(cols) > 0 and len(cols[0]) == 0:
3649 cols = cols[1:]
3650 if len(cols) > 0 and len(cols[-1]) == 0:
3651 cols = cols[:-1]
3652 if len(cols) > 0:
3653 cols[0] = cols[0].lstrip('|').lstrip()
3654 cols[-1] = cols[-1].rstrip('|').rstrip()
3655 cols = [c.strip() for c in cols if c != '|']
3656 # read columns:
3657 for k, c in enumerate(cols):
3658 try:
3659 v = float(c)
3660 ad = 0
3661 ve = c.split('e')
3662 if len(ve) <= 1:
3663 exped[k] = False
3664 else:
3665 ad = len(ve[1])+1
3666 vc = ve[0].split('.')
3667 ad += len(vc[0])
3668 prec = len(vc[0].lstrip('-').lstrip('+').lstrip('0'))
3669 if len(vc) == 2:
3670 if numc[k] and post[k] != len(vc[1]):
3671 fixed[k] = False
3672 if post[k] < len(vc[1]):
3673 post[k] = len(vc[1])
3674 ad += len(vc[1])+1
3675 prec += len(vc[1].rstrip('0'))
3676 if precd[k] < prec:
3677 precd[k] = prec
3678 if alld[k] < ad:
3679 alld[k] = ad
3680 numc[k] = True
3681 except ValueError:
3682 if c in missing:
3683 v = np.nan
3684 nans[k] = c
3685 elif len(c) == 0 and not strf[k]:
3686 v = np.nan
3687 else:
3688 strf[k] = True
3689 if alld[k] < len(c):
3690 alld[k] = len(c)
3691 if len(c) >= 2 and c[0] == '"' and c[-1] == '"':
3692 v = c.strip('"')
3693 else:
3694 v = c
3695 self.add(v, k)
3696 self.fill_data()
3698 # initialize:
3699 if isinstance(missing, str):
3700 missing = [missing]
3701 self.data = []
3702 self.ndim = 2
3703 self.shape = (0, 0)
3704 self.header = []
3705 self.nsecs = 0
3706 self.units = []
3707 self.formats = []
3708 self.descriptions = []
3709 self.hidden = []
3710 self.setcol = 0
3711 self.addcol = 0
3712 # open file:
3713 file_name = None
3714 own_file = False
3715 if not hasattr(fh, 'readline'):
3716 file_name = Path(fh)
3717 fh = open(fh, 'r')
3718 own_file = True
3719 # read inital lines of file:
3720 key = []
3721 data = []
3722 target = data
3723 comment = False
3724 table_format='dat'
3725 for line in fh:
3726 line = line.rstrip()
3727 if line == stop:
3728 break;
3729 if line:
3730 if r'\begin{tabular' in line:
3731 table_format='tex'
3732 target = key
3733 continue
3734 if table_format == 'tex':
3735 if r'\end{tabular' in line:
3736 break
3737 if r'\hline' in line:
3738 if key:
3739 target = data
3740 continue
3741 line = line.rstrip(r'\\')
3742 if line[0] == '#':
3743 comment = True
3744 table_format='dat'
3745 target = key
3746 line = line.lstrip('#')
3747 elif comment:
3748 target = data
3749 if line[0:3] == 'RTH':
3750 target = key
3751 line = line[3:]
3752 table_format='rtai'
3753 elif line[0:3] == 'RTD':
3754 target = data
3755 line = line[3:]
3756 table_format='rtai'
3757 if (line[0:3] == '|--' or line[0:3] == '|:-') and \
3758 (line[-3:] == '--|' or line[-3:] == '-:|'):
3759 if not data and not key:
3760 table_format='ascii'
3761 target = key
3762 continue
3763 elif not key:
3764 table_format='md'
3765 key = data
3766 data = []
3767 target = data
3768 continue
3769 elif not data:
3770 target = data
3771 continue
3772 else:
3773 break
3774 target.append(line)
3775 else:
3776 break
3777 if len(data) > 5:
3778 break
3779 # find column separator of data and number of columns:
3780 col_seps = ['|', ',', ';', ':', '\t', '&', None]
3781 if sep is not None:
3782 col_seps = [sep]
3783 colstd = np.zeros(len(col_seps))
3784 colnum = np.zeros(len(col_seps), dtype=int)
3785 for k, sep in enumerate(col_seps):
3786 cols = []
3787 s = 5 if len(data) >= 8 else len(data) - 3
3788 if s < 0 or key:
3789 s = 0
3790 for line in data[s:]:
3791 cs = line.strip().split(sep)
3792 if not cs[0]:
3793 cs = cs[1:]
3794 if cs and not cs[-1]:
3795 cs = cs[:-1]
3796 cols.append(len(cs))
3797 colstd[k] = np.std(cols)
3798 colnum[k] = np.median(cols)
3799 if np.max(colnum) < 2:
3800 sep = None
3801 colnum = 1
3802 else:
3803 ci = np.where(np.array(colnum) > 1.5)[0]
3804 ci = ci[np.argmin(colstd[ci])]
3805 sep = col_seps[ci]
3806 colnum = int(colnum[ci])
3807 # fix key:
3808 if not key and sep is not None and sep in ',;:\t|':
3809 table_format = 'csv'
3810 # read key:
3811 key_cols = []
3812 key_indices = []
3813 for line in key:
3814 cols, indices = read_key_line(line, sep, table_format)
3815 key_cols.append(cols)
3816 key_indices.append(indices)
3817 if not key_cols:
3818 # no obviously marked table key:
3819 key_num = 0
3820 for line in data:
3821 cols, indices = read_key_line(line, sep, table_format)
3822 numbers = 0
3823 for c in cols:
3824 try:
3825 v = float(c)
3826 numbers += 1
3827 except ValueError:
3828 pass
3829 if numbers == 0:
3830 key_cols.append(cols)
3831 key_indices.append(indices)
3832 key_num += 1
3833 else:
3834 break
3835 if len(key_cols) == len(data):
3836 key_num = 1
3837 key_cols = key_cols[:1]
3838 key_indices = key_indices[:1]
3839 colnum = len(key_cols[0])
3840 data = data[key_num:]
3841 kr = len(key_cols) - 1
3842 # check for key with column indices:
3843 if kr >= 0:
3844 cols = key_cols[kr]
3845 numrow = True
3846 try:
3847 pv = int(cols[0])
3848 for c in cols[1:]:
3849 v = int(c)
3850 if v != pv+1:
3851 numrow = False
3852 break
3853 pv = v
3854 except ValueError:
3855 try:
3856 pv = aa2index(cols[0])
3857 for c in cols[1:]:
3858 v = aa2index(c)
3859 if v != pv+1:
3860 numrow = False
3861 break
3862 pv = v
3863 except ValueError:
3864 numrow = False
3865 if numrow:
3866 kr -= 1
3867 # check for unit line:
3868 units = None
3869 if kr > 0 and len(key_cols[kr]) == len(key_cols[kr - 1]):
3870 units = key_cols[kr]
3871 kr -= 1
3872 # column labels:
3873 if kr >= 0:
3874 if units is None:
3875 # units may be part of the label:
3876 labels = []
3877 units = []
3878 for c in key_cols[kr]:
3879 if len(c) > 0 and c[-1] == ')':
3880 lu = c[:-1].split('(')
3881 if len(lu) >= 2:
3882 labels.append(lu[0].strip())
3883 units.append('('.join(lu[1:]).strip())
3884 continue
3885 lu = c.split('/')
3886 if len(lu) >= 2:
3887 labels.append(lu[0].strip())
3888 units.append('/'.join(lu[1:]).strip())
3889 else:
3890 labels.append(c)
3891 units.append('')
3892 else:
3893 labels = key_cols[kr]
3894 indices = key_indices[kr]
3895 # init table columns:
3896 for k in range(colnum):
3897 self.append(labels[k], units[k], '%g')
3898 # read in sections:
3899 while kr > 0:
3900 kr -= 1
3901 for sec_label, sec_inx in zip(key_cols[kr], key_indices[kr]):
3902 col_inx = indices.index(sec_inx)
3903 self.header[col_inx].append(sec_label)
3904 if self.nsecs < len(self.header[col_inx]) - 1:
3905 self.nsecs = len(self.header[col_inx]) - 1
3906 # read data:
3907 post = np.zeros(colnum, dtype=int)
3908 precd = np.zeros(colnum, dtype=int)
3909 alld = np.zeros(colnum, dtype=int)
3910 numc = [False] * colnum
3911 exped = [True] * colnum
3912 fixed = [True] * colnum
3913 strf = [False] * colnum
3914 nans = [None] * colnum # for each column the missing string that was encountered.
3915 for line in data:
3916 read_data_line(line, sep, post, precd, alld, numc, exped, fixed,
3917 strf, missing, nans)
3918 # read remaining data:
3919 for line in fh:
3920 line = line.rstrip()
3921 if line == stop:
3922 break;
3923 if table_format == 'tex':
3924 if r'\end{tabular' in line or r'\hline' in line:
3925 break
3926 line = line.rstrip(r'\\')
3927 if (line[0:3] == '|--' or line[0:3] == '|:-') and \
3928 (line[-3:] == '--|' or line[-3:] == '-:|'):
3929 break
3930 if line[0:3] == 'RTD':
3931 line = line[3:]
3932 read_data_line(line, sep, post, precd, alld, numc, exped, fixed,
3933 strf, missing, nans)
3934 # set formats:
3935 for k in range(len(alld)):
3936 if strf[k]:
3937 self.set_format(k, '%%-%ds' % alld[k])
3938 # make sure all elements are strings:
3939 for i in range(len(self.data[k])):
3940 if self.data[k][i] is np.nan:
3941 self.data[k][i] = nans[k]
3942 else:
3943 self.data[k][i] = str(self.data[k][i])
3944 elif exped[k]:
3945 self.set_format(k, '%%%d.%de' % (alld[k], post[k]))
3946 elif fixed[k]:
3947 self.set_format(k, '%%%d.%df' % (alld[k], post[k]))
3948 else:
3949 self.set_format(k, '%%%d.%dg' % (alld[k], precd[k]))
3950 # close file:
3951 if own_file:
3952 fh.close()
3953 # load descriptions:
3954 print(file_name)
3955 if file_name is not None:
3956 descr_path = file_name.parent
3957 descr_files = descr_path.glob(file_name.stem + '-*.md')
3958 for path in descr_files:
3959 if path.is_file():
3960 self.load_descriptions(path)
3961 break
3964def add_write_table_config(cfg, table_format=None, delimiter=None,
3965 unit_style=None, column_numbers=None,
3966 sections=None, align_columns=None,
3967 shrink_width=True, missing='-',
3968 center_columns=False,
3969 latex_label_command='',
3970 latex_merge_std=False):
3971 """Add parameter specifying how to write a table to a file as a new
3972section to a configuration.
3974 Parameters
3975 ----------
3976 cfg: ConfigFile
3977 The configuration.
3978 """
3980 cfg.add_section('File format for storing analysis results:')
3981 cfg.add('fileFormat', table_format or 'auto', '', 'Default file format used to store analysis results.\nOne of %s.' % ', '.join(TableData.formats))
3982 cfg.add('fileDelimiter', delimiter or 'auto', '', 'String used to separate columns or "auto".')
3983 cfg.add('fileUnitStyle', unit_style or 'auto', '', 'Add units as extra row ("row"), add units to header label separated by "/" ("header"), do not print out units ("none"), or "auto".')
3984 cfg.add('fileColumnNumbers', column_numbers or 'none', '', 'Add line with column indices ("index", "num", "aa", "AA", or "none")')
3985 cfg.add('fileSections', sections or 'auto', '', 'Maximum number of section levels or "auto"')
3986 cfg.add('fileAlignColumns', align_columns or 'auto', '', 'If True, write all data of a column using the same width, if False write the data without any white space, or "auto".')
3987 cfg.add('fileShrinkColumnWidth', shrink_width, '', 'Allow to make columns narrower than specified by the corresponding format strings.')
3988 cfg.add('fileMissing', missing, '', 'String used to indicate missing data values.')
3989 cfg.add('fileCenterColumns', center_columns, '', 'Center content of all columns instead of left align columns of strings and right align numbers (markdown, html, and latex).')
3990 cfg.add('fileLaTeXLabelCommand', latex_label_command, '', 'LaTeX command name for formatting column labels of the table header.')
3991 cfg.add('fileLaTeXMergeStd', latex_merge_std, '', 'Merge header of columns with standard deviations with previous column (LaTeX tables only).')
3994def write_table_args(cfg):
3995 """Translates a configuration to the respective parameter names for
3996writing a table to a file.
3998 The return value can then be passed as key-word arguments to TableData.write().
4000 Parameters
4001 ----------
4002 cfg: ConfigFile
4003 The configuration.
4005 Returns
4006 -------
4007 a: dict
4008 Dictionary with names of arguments of the `TableData.write` function
4009 and their values as supplied by `cfg`.
4010 """
4011 d = cfg.map(table_format='fileFormat',
4012 delimiter='fileDelimiter',
4013 unit_style='fileUnitStyle',
4014 column_numbers='fileColumnNumbers',
4015 sections='fileSections',
4016 align_columns='fileAlignColumns',
4017 shrink_width='fileShrinkColumnWidth',
4018 missing='fileMissing',
4019 center_columns='fileCenterColumns',
4020 latex_label_command='fileLaTeXLabelCommand',
4021 latex_merge_std='fileLaTeXMergeStd')
4022 if 'sections' in d:
4023 if d['sections'] != 'auto':
4024 d['sections'] = int(d['sections'])
4025 return d
4028def latex_unit(unit, unit_package=None):
4029 """Translate unit string into LaTeX code.
4031 Parameters
4032 ----------
4033 unit: str
4034 String denoting a unit.
4035 unit_package: None or 'siunitx' or 'SIunit'
4036 Translate unit string for the specified LaTeX package.
4037 If None set sub- and superscripts in text mode.
4039 Returns
4040 -------
4041 unit: str
4042 Unit string as valid LaTeX code.
4043 """
4044 si_prefixes = {'y': '\\yocto',
4045 'z': '\\zepto',
4046 'a': '\\atto',
4047 'f': '\\femto',
4048 'p': '\\pico',
4049 'n': '\\nano',
4050 'u': '\\micro',
4051 'm': '\\milli',
4052 'c': '\\centi',
4053 'd': '\\deci',
4054 'h': '\\hecto',
4055 'k': '\\kilo',
4056 'M': '\\mega',
4057 'G': '\\giga',
4058 'T': '\\tera',
4059 'P': '\\peta',
4060 'E': '\\exa',
4061 'Z': '\\zetta',
4062 'Y': '\\yotta' }
4063 si_units = {'m': '\\metre',
4064 'g': '\\gram',
4065 's': '\\second',
4066 'A': '\\ampere',
4067 'K': '\\kelvin',
4068 'mol': '\\mole',
4069 'M': '\\mole',
4070 'cd': '\\candela',
4071 'Hz': '\\hertz',
4072 'N': '\\newton',
4073 'Pa': '\\pascal',
4074 'J': '\\joule',
4075 'W': '\\watt',
4076 'C': '\\coulomb',
4077 'V': '\\volt',
4078 'F': '\\farad',
4079 'O': '\\ohm',
4080 'S': '\\siemens',
4081 'Wb': '\\weber',
4082 'T': '\\tesla',
4083 'H': '\\henry',
4084 'C': '\\celsius',
4085 'lm': '\\lumen',
4086 'lx': '\\lux',
4087 'Bq': '\\becquerel',
4088 'Gv': '\\gray',
4089 'Sv': '\\sievert'}
4090 other_units = {"'": '\\arcminute',
4091 "''": '\\arcsecond',
4092 'a': '\\are',
4093 'd': '\\dday',
4094 'eV': '\\electronvolt',
4095 'ha': '\\hectare',
4096 'h': '\\hour',
4097 'L': '\\liter',
4098 'l': '\\litre',
4099 'min': '\\minute',
4100 'Np': '\\neper',
4101 'rad': '\\rad',
4102 't': '\\ton',
4103 '%': '\\%'}
4104 unit_powers = {'^2': '\\squared',
4105 '^3': '\\cubed',
4106 '/': '\\per',
4107 '^-1': '\\power{}{-1}',
4108 '^-2': '\\rpsquared',
4109 '^-3': '\\rpcubed'}
4110 if unit_package is None:
4111 # without any unit package:
4112 units = ''
4113 k = 0
4114 while k < len(unit):
4115 if unit[k] == '^':
4116 j = k + 1
4117 while j < len(unit) and (unit[j] == '-' or unit[j].isdigit()):
4118 j += 1
4119 units = units + '$^{\\text{' + unit[k + 1:j] + '}}$'
4120 k = j
4121 elif unit[k] == '_':
4122 j = k + 1
4123 while j < len(unit) and not unit[j].isspace():
4124 j += 1
4125 units = units + '$_{\\text{' + unit[k + 1:j] + '}}$'
4126 k = j
4127 else:
4128 units = units + unit[k]
4129 k += 1
4130 elif unit_package.lower() in ['siunit', 'siunitx']:
4131 # use SIunit package:
4132 if '\\' in unit: # this string is already translated!
4133 return unit
4134 units = ''
4135 j = len(unit)
4136 while j >= 0:
4137 for k in range(-3, 0):
4138 if j+k < 0:
4139 continue
4140 uss = unit[j+k:j]
4141 if uss in unit_powers:
4142 units = unit_powers[uss] + units
4143 break
4144 elif uss in other_units:
4145 units = other_units[uss] + units
4146 break
4147 elif uss in si_units:
4148 units = si_units[uss] + units
4149 j = j+k
4150 k = 0
4151 if j - 1 >= 0:
4152 uss = unit[j - 1:j]
4153 if uss in si_prefixes:
4154 units = si_prefixes[uss] + units
4155 k = -1
4156 break
4157 else:
4158 k = -1
4159 units = unit[j+k:j] + units
4160 j = j + k
4161 if unit_package.lower() == 'siunitx':
4162 units = '\\unit{' + units + '}'
4163 else:
4164 raise ValueError(f'latex_unit(): invalid unit_package={unit_package}!')
4165 return units
4168def break_text(stream, text, maxc=80, indent=0):
4169 """Write text to stream and break lines at maximum character count.
4171 Parameters
4172 ----------
4173 stream: io
4174 Stream into which the text is written.
4175 text: str
4176 The text to be written to the stream.
4177 maxc: int
4178 Maximum character count for each line.
4179 indent: int
4180 Number of characters each line is indented.
4181 """
4182 nc = 0
4183 nw = 0
4184 stream.write(' '*indent)
4185 nc += indent
4186 for word in text.split():
4187 if nc + len(word) > maxc:
4188 stream.write('\n')
4189 nc = 0
4190 nw = 0
4191 stream.write(' '*indent)
4192 nc += indent
4193 if nw > 0:
4194 stream.write(' ')
4195 nc += 1
4196 stream.write(word)
4197 nc += len(word)
4198 nw += 1
4199 stream.write('\n')
4202def index2aa(n, a='a'):
4203 """Convert an integer into an alphabetical representation.
4205 The integer number is converted into 'a', 'b', 'c', ..., 'z',
4206 'aa', 'ab', 'ac', ..., 'az', 'ba', 'bb', ...
4208 Inspired by https://stackoverflow.com/a/37604105
4210 Parameters
4211 ----------
4212 n: int
4213 An integer to be converted into alphabetical representation.
4214 a: str ('a' or 'A')
4215 Use upper or lower case characters.
4217 Returns
4218 -------
4219 ns: str
4220 Alphabetical represtnation of an integer.
4221 """
4222 d, m = divmod(n, 26)
4223 bm = chr(ord(a)+m)
4224 return index2aa(d - 1, a) + bm if d else bm
4227def aa2index(s):
4228 """Convert an alphabetical representation to an index.
4230 The alphabetical representation 'a', 'b', 'c', ..., 'z',
4231 'aa', 'ab', 'ac', ..., 'az', 'ba', 'bb', ...
4232 is converted to an index starting with 0.
4234 Parameters
4235 ----------
4236 s: str
4237 Alphabetical representation of an index.
4239 Returns
4240 -------
4241 index: int
4242 The corresponding index.
4244 Raises
4245 ------
4246 ValueError:
4247 Invalid character in input string.
4248 """
4249 index = 0
4250 maxc = ord('z') - ord('a') + 1
4251 for c in s.lower():
4252 index *= maxc
4253 if ord(c) < ord('a') or ord(c) > ord('z'):
4254 raise ValueError('invalid character "%s" in string.' % c)
4255 index += ord(c) - ord('a') + 1
4256 return index - 1
4259class IndentStream(object):
4260 """Filter an output stream and start each newline with a number of
4261 spaces.
4262 """
4263 def __init__(self, stream, indent=4):
4264 self.stream = stream
4265 self.indent = indent
4266 self.pending = True
4268 def __getattr__(self, attr_name):
4269 return getattr(self.stream, attr_name)
4271 def write(self, data):
4272 if not data:
4273 return
4274 if self.pending:
4275 self.stream.write(' '*self.indent)
4276 self.pending = False
4277 substr = data.rstrip('\n')
4278 rn = len(data) - len(substr)
4279 if len(substr) > 0:
4280 self.stream.write(substr.replace('\n', '\n'+' '*self.indent))
4281 if rn > 0:
4282 self.stream.write('\n'*rn)
4283 self.pending = True
4285 def flush(self):
4286 self.stream.flush()
4289def main():
4290 # setup a table:
4291 df = TableData()
4292 df.append(["data", "specimen", "ID"], "", "%-s", value=list('ABCBAACB'))
4293 df.append("size", "m", "%6.2f", value=[2.34, 56.7, 8.9])
4294 df.append("full weight", "kg", "%.0f", value=122.8)
4295 df.append_section("all measures")
4296 df.append("speed", "m/s", "%.3g", value=98.7)
4297 df.append("median jitter", "mm", "%.1f", value=23)
4298 df.append("size", "g", "%.2e", value=1.234)
4299 df.set_descriptions({'ID': 'A unique identifier of a snake.',
4300 'size': 'The total length of each snake.',
4301 'full weight': 'Weight of each snake',
4302 'speed': 'Maximum speed the snake can climb a tree.',
4303 'median jitter': 'The jitter around a given path the snake should follow.',
4304 'all measures>size': 'Weight of mouse the snake has eaten before.',
4305 })
4306 df.add(np.nan, 2) # single value
4307 df.add([0.543, 45, 1.235e2]) # remaining row
4308 df.add([43.21, 6789.1, 3405, 1.235e-4], 2) # next row
4309 a = 0.5*np.arange(1, 6)*np.random.randn(5, 5) + 10.0 + np.arange(5)
4310 df.add(a.T, 1) # rest of table
4311 #df[3:6,'weight'] = [11.0]*3
4312 df.insert('median jitter', 's.d.', 'm/s', '%.3g',
4313 'Standard deviation of all speeds',
4314 value=2*np.random.rand(df.rows()))
4316 # write out in all formats:
4317 for tf in TableData.formats:
4318 print(' - `%s`: %s' % (tf, TableData.descriptions[tf]))
4319 print(' ```')
4320 iout = IndentStream(sys.stdout, 4+2)
4321 df.write(iout, table_format=tf, latex_unit_package='siunitx',
4322 latex_merge_std=True)
4323 print(' ```')
4324 print()
4326 # write descriptions:
4327 for tf in ['md', 'html', 'tex']:
4328 df.write_descriptions(table_format=tf, maxc=40)
4329 print()
4331 # aggregate demos:
4332 print(df)
4333 print(df.aggregate(np.mean, numbers_only=True))
4334 print(df.aggregate(dict(count=len, maximum=np.max), numbers_only=True))
4335 print(df.aggregate([np.mean, len, max],
4336 ['size', 'full weight', 'speed'], 'statistics',
4337 remove_nans=True, single_row=False))
4338 print(df.aggregate({('25%', '50%', '75%'):
4339 (np.quantile, ([0.25, 0.6, 0.75],))},
4340 numbers_only=True))
4342 print(df.statistics(single_row=False))
4343 print(df.statistics(single_row=True, remove_nans=True))
4344 print(df.statistics(remove_nans=True, by='ID'))
4346 # groupby demo:
4347 for name, values in df.groupby('ID'):
4348 print(name)
4349 print(values)
4350 print()
4352 # aggregrate on groups demo:
4353 print(df.aggregate(np.mean, by='ID'))
4354 print()
4357if __name__ == "__main__":
4358 main()