Commit a40a4318 authored by Antoine Laudrain's avatar Antoine Laudrain
Browse files

Merge branch 'master' into merging

parents 09cbdde6 f9d0e707
Pipeline #142079 passed with stages
in 3 minutes and 53 seconds
......@@ -84,6 +84,7 @@ pylint_TandD_XML:
test_MSOS:
stage: test
script:
- python MSOS254A/ut_helpers.py
- python MSOS254A/ut_helpers_wav.py
test_TandD_XML:
......
......@@ -429,12 +429,12 @@ def parse_args(argv):
parser.add_argument("--print", dest="do_print", action="store_true",
help="Print status before applying other commands.")
parser.add_argument("--off",
dest="channelOFF", metavar="CHANNEL", type=int, nargs="*", default=None,
parser.add_argument("--off", dest="channelOFF",
metavar="CHANNEL", type=int, nargs="*", default=None,
help="Set given channels off. If option is provided without arguments, "
"all channels are turned OFF.")
parser.add_argument("--on",
dest="channelON", metavar="CHANNEL", type=int, nargs="+",
parser.add_argument("--on", dest="channelON",
metavar="CHANNEL", type=int, nargs="+",
help="Set given channels ON.")
parser.add_argument("--trackOFF",
......@@ -445,10 +445,11 @@ def parse_args(argv):
metavar="CHANNEL", type=int, nargs="+",
help="List of channels to set tracking ON.")
parser.add_argument("-c", "--channel",
metavar="CHANNEL", type=int, nargs="+",
choices=set(INSTALLED_CHANNELS), dest="target_channels",
help="Channel to apply the following settings (default: none).")
parser.add_argument("-c", "--channel", dest="target_channels",
metavar="CHANNEL", type=int, nargs="+", default=None,
choices=set(INSTALLED_CHANNELS),
help="Channel to apply the following settings. "
"If not provided, channel-related settings will not be applied.")
timing = parser.add_mutually_exclusive_group()
timing.add_argument("--freq",
......@@ -462,8 +463,8 @@ def parse_args(argv):
help="Set output phase start point (in degrees, regularised in 0-360).")
voltage = parser.add_argument_group("Voltage control (indpt of function)")
voltage.add_argument("--imp", "--impedance",
dest="impedance", type=int, default=None, metavar="OHMS",
voltage.add_argument("--imp", "--impedance", dest="impedance",
type=int, default=None, metavar="OHMS",
help="Set output impedance (integer, in Ohms).")
voltage.add_argument("--vlow", type=float, default=None, metavar="VOLT",
help="Set output low voltage (in Volts).")
......
......@@ -20,10 +20,10 @@ The first argument (only positional one) is the path prefix to the output file.
It should not contain the extension, as the timestamp, channel number, and extension are automatically added.
For example if `cosmics` is given, the following output files will be created:
- `data/cosmics.yymmdd_HHMMSS.CHAN<N>.raw`: a binary file containing the raw waveforms (byte-string flux from the scope).
- `data/cosmics.yymmdd_HHMMSS.CHAN4.raw`: a binary file containing the raw waveforms (byte-string flux from the scope).
The first two (`'#0'`) and the last characters (`'\n'`) are not part of the data.
All the waveform are concatenated (no event separation), the event length is deduced from the preamble.
- `data/cosmics.yymmdd_HHMMSS.CHAN<N>.dat`, plain text file containing the preamble on the first line, providing all necessary information to reconstruct the waveform length.
- `data/cosmics.yymmdd_HHMMSS.CHAN4.dat`, plain text file containing the preamble on the first line, providing all necessary information to reconstruct the waveform length.
This argument can be a path containing folders (eg. `data/200909/cosmics`).
The last directory in the tree is created if not existing (if the above one don't exist, the code will crash).
......@@ -72,20 +72,87 @@ This log file concatenates the information from all runs.
The log files are in all cases appended, so that no information can be lost.
Moreover, the name of the 'single' log file can be explicitely specified using the `--logfile <path/to/logfile.log>` option.
Moreover, the name of the 'single' log file can be explicitly specified using the `--logfile <path/to/logfile.log>` option.
### Screen dump
The screen of the oscilloscope can exported using the [`dump_screen.py`](./dump_screen.py) script.
One can dump the full screen using the `-f <name>` option (`--fullscreen`), or only the graticule with the `-g <name>` option (`--graticule`).
Both options can be provided simultaneously, and require an argument corresponding to the name of the plot.
The name is automatically appended with the date and time, and with the file extension.
Dump formats can be specified with the `--exts` option.
The list of possible dump formats is `BMP, JPG, GIF, TIF, PNG`.
Example: `python dump_screen.py --exts png bmp -f fulldump -g gratonly` will write files
- `fulldump.yymmdd_HHMMSS.png`
- `gratonly.yymmdd_HHMMSS.png`
- `fulldump.yymmdd_HHMMSS.bmp`
- `gratonly.yymmdd_HHMMSS.bmp`
Reading previously recorded data
--------------------------------
### Time series and waveforms
If using python, one can simply
```python
from helpers_wav import read_data
time_series, waveforms = read_data("path/to/data.*.CHAN1.raw")
# or explicitly
time_series, waveforms = read_data([
"path/to/data.yymmdd_123456.CHAN1.raw",
"path/to/data.yymmdd_234567.CHAN1.raw",
...
])
```
The function assumes each `.raw` data file has a corresponding `.dat` preamble file sitting next to it with same basename.
By default the data are flatten and aligned, ie `time_series[0]` contains the time series corresponding to `waveforms[0]`, `time_series[1]` to `waveform[1]` etc...
This is the simplest format to use, but if `waveform[0]` and `waveform[1]` come from the same file their time series is identical (it was duplicated by the reading function).
To use a more memory-efficient format, pass the option `flatten=False` to `read_data`.
In this case `time_series` has a different meaning and `waveforms` one more nested level.
The `time_series[0]` will give the time series corresponding to all waveform from the first file read, ie `waveforms[0][0]`, `waveforms[0][1]`, `waveforms[0][2]`, ...,
`time_series[1]` to all waveforms from the second file read, ie `waveforms[1][0]`, `waveforms[1][1]`, `waveforms[1][2]`, ...
### Data streaming
When reading several files, the above method will dump in memory all the files' content, making it hardly sustainable.
The `iter_data` function returns a generator that will iterate over all waveforms of all files, yielding one waveform at a time with its corresponding time points.
This generator is provided as convenience for data analysis pipeline.
The snippets below should give equivalent results:
```python
time_series, waveforms = read_data("path/to/data.*.CHAN1.raw")
for idx in range(len(waveform)):
time_points = time_series[idx]
waveform = waveforms[idx]
...
for time_points, waveform in zip(*read_data("path/to/data.*.CHAN1.raw")):
...
for time_points, waveform in iter_data("path/to/data.*.CHAN1.raw"):
...
```
### Step-by-step example on a single file
```python
from helpers_wav import process_preamble, get_time_points, process_sequence
# First read and process preamble for metadata.
with open("path/to/data.yymmdd_HHMMSS.dat") as infile:
with open("path/to/data.yymmdd_HHMMSS.CHAN1.dat") as infile:
preamble = infile.read()
wav_format, wav_type, npoints, count, \
......@@ -98,7 +165,7 @@ xunit, yunit, bw_max, bw_min, segcount = process_preamble(preamble)
time_points = get_time_points(npoints, xinc, xorigin)
# Read the raw data.
with open("path/to/data.yymmdd_HHMMSS.raw", 'rb') as infile:
with open("path/to/data.yymmdd_HHMMSS.CHAN1.raw", 'rb') as infile:
raw_data = infile.read()
events = process_sequence(raw_data, npoints, yinc, yorigin)
......@@ -106,6 +173,9 @@ events = process_sequence(raw_data, npoints, yinc, yorigin)
# At this point, events is a list of np.array.
```
### Reading the events time stamps
Since the time tags file is in plain text (one tag per line), retrieving the ttags is super easy:
```python
......@@ -114,3 +184,41 @@ ttags = np.loadtxt("path/to/data.yymmdd_HHMMSS.ttags.txt", dtype=np.float64)
# The default dtype should be float64 anyway, so this is optional.
```
### Plot various waveforms
To quickly explore data files, the following utility is provided: [`plot_waveforms.py`](./plot_waveforms.py).
An example minimal usage is as follows:
```bash
python plot_waveforms.py --out test_superimpose \
--data /localscratch/data/cosmics_autoTrig230mV.210424_130855.CHAN1.raw 0 1 2 \
--data /localscratch/data/cosmics_autoTrig230mV.210424_204052.CHAN1.raw 5 10 15
```
Its main argument in the `--data` flag, which expects at least one token.
The first token is an input file name (`.raw` file, the corresponding preamble `.dat` file is automatically read),
and all the following are understood as event number indices within this file.
As shown in the example the `--data` flag can be repeated to get events from as many files as needed.
The other mandatory option is the `--output` flag, which should be provided with the name of the output file
(with directories if needed, but those won't be created), **without extension**.
By default, the plot is created in `pdf` and `png` format.
This can be changed by passing the desired extension to the `--exts` option.
The default behaviour is to process the plot without displaying it on screen (only writing output files).
If the `--display` option is passed, matplotlib will display the plot and the program will not exit before the plot is closed.
#### Plotting options
The following additional option are provided:
- `xmin`, `xmax`, `ymin`, `ymax`: to manually set the plot boundaries.
- `--shift <value in ns>`: each successive waveform will be shifted by this amount.
Can be used to improve the readability.
The following two options can be used for very basic processing and are mutually exclusive:
- `--every <N>`: use only one every `N` points for the plot.
- `--average <N>`: average over groups of `N` points (not rolling: the number or plotted points will be decreased by this amount).
......@@ -9,13 +9,15 @@
import sys
import os.path
from collections.abc import Iterable
from time import strftime
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from textwrap import dedent
import visa
from helpers import get_exts_list
from logger_cfg import getLogger, setup_logging
logger = getLogger(__name__)
INSTRUMENT_ADDRESS = 'TCPIP::192.168.0.2::INSTR'
ALLOWED_EXT = set(["BMP", "JPG", "GIF", "TIF", "PNG"])
......@@ -29,7 +31,7 @@ def filter_data(data):
return data[ 2 + int(chr(data[1])) : -1 ]
def main(fullscreen="screen_dump", graticule="", exts="png"):
def main(fullscreen="screen_dump", graticule="", exts=None):
"""Main executable
@param[in] fullscreen: string
......@@ -38,47 +40,55 @@ def main(fullscreen="screen_dump", graticule="", exts="png"):
@param[in] graticule: string
Output file path without extension for graticule-only dump.
If empty, graticule-only is not dumped is not dumped.
@param[in] exts: string or list of string
@param[in] exts: string or list of string or None, default: None
List of file extensions for screen dump.
If None, assume it's 'png'.
@param[in] debug: flag to enable debug output.
"""
setup_logging()
if not fullscreen and not graticule:
print("None of fullscreen nor graticule requested, stopping here.")
logger.error("None of fullscreen nor graticule requested, "
"stopping here.")
return
# Check that output dir exists.
if fullscreen:
base = os.path.normpath(os.path.dirname(fullscreen))
if not os.path.exists(base):
print(f"Fullscreen output directory ({base}) does not exist, aborting.")
outdir = os.path.normpath(os.path.dirname(fullscreen))
if not os.path.exists(outdir):
logger.error("Fullscreen outdir (%s) does not exist, aborting.",
outdir)
return
if graticule:
base = os.path.normpath(os.path.dirname(graticule))
if not os.path.exists(base):
print(f"Graticule output directory ({base}) does not exist, aborting.")
outdir = os.path.normpath(os.path.dirname(graticule))
if not os.path.exists(outdir):
logger.error("Graticule outdir (%s) does not exist, aborting.",
outdir)
return
# Make sure output formats is iterable.
if not isinstance(exts, Iterable) :
exts = [exts]
exts = get_exts_list(exts, default="png")
# Make sure all requested formats are known.
for ext in exts:
if not ext.upper() in ALLOWED_EXT:
print(ext, "is not known, removing this format. Available:", *ALLOWED_EXT)
if ext.upper() not in ALLOWED_EXT:
logger.warning("Extension %s is not available, "
"removing this format.", ext)
logger.info("Available: %s", ALLOWED_EXT)
exts.remove(ext)
# If no known format, abort here.
if not exts:
print("No known output format remaining, stopping here.")
logger.error("No known output format remaining, stopping here.")
return
# '@py' to use the pyvisa-py backend
resource_mgr = visa.ResourceManager("@py")
print(resource_mgr.list_resources())
print("Getting instrument at address:", INSTRUMENT_ADDRESS)
logger.info(resource_mgr.list_resources())
logger.info("Getting instrument at address: %s", INSTRUMENT_ADDRESS)
instr = resource_mgr.open_resource(INSTRUMENT_ADDRESS)
print(repr(instr.query('*IDN?'))) # Print instrument info
logger.info(repr(instr.query('*IDN?')))
datetime = strftime("%y%m%d_%H%M%S")
......@@ -87,53 +97,54 @@ def main(fullscreen="screen_dump", graticule="", exts="png"):
instr.write(f'DISPlay:DATA? {ext.upper()},SCR')
data = instr.read_raw()
outname = f"{fullscreen}.{datetime}.{ext}"
print("Will write file:", outname)
logger.info("Will write file: %s", outname)
with open(outname, 'wb') as outf:
outf.write(filter_data(data))
if graticule:
instr.write(f'DISPlay:DATA? {ext.upper()},GRAT')
data = instr.read_raw()
outname = f"{graticule}.{datetime}.{ext}"
print("Will write file:", outname)
logger.info("Will write file: %s", outname)
with open(outname, 'wb') as outf:
outf.write(filter_data(data))
return
###############################################################################
DESCR = "Dump screen and/or graticule of MSOS 254A."
EPILOG = """
Examples:
---------
python dump_screen.py -f screenshots/full
Writes screenshots/full.YYMMDD_HHMMSS.png (folder must exist).
python dump_screen.py --exts png bmp -f fulldump -g gratonly
Writes
- fulldump.yymmdd_HHMMSS.png
- gratonly.yymmdd_HHMMSS.png
- fulldump.yymmdd_HHMMSS.bmp
- gratonly.yymmdd_HHMMSS.bmp
"""
def parse_args(argv):
# pylint: disable=missing-function-docstring
descr = "Dump screen and/or graticule of MSOS 254A."
epilog = dedent("""
Examples:
---------
python dump_screen.py -f screenshots/full
Writes screenshots/full__YYMMDD_HHMMSS.png (folder must exist).
python dump_screen.py --ext png bmp -f fulldump -g gratonly
Writes
- fulldump__yymmdd_HHMMSS.png
- gratonly__yymmdd_HHMMSS.png
- fulldump__yymmdd_HHMMSS.bmp
- gratonly__yymmdd_HHMMSS.bmp
""")
parser = ArgumentParser(description=descr, epilog=epilog,
formatter_class=RawDescriptionHelpFormatter)
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
description=DESCR, epilog=EPILOG)
parser.add_argument("--fullscreen", "-f", metavar="PATH",
help="Dump full screen to this path (without extension).")
parser.add_argument("--graticule", "-g", metavar="PATH",
help="Dump graticule only to this path (without extension).")
parser.add_argument("--ext", nargs="+", default=["png"],
parser.add_argument("--exts", nargs="+", default=["png"],
choices=list(map(lambda x: x.lower(), ALLOWED_EXT)),
help="Extensions to dump files.")
help="Extensions to dump files. Default: %(default)s")
args = parser.parse_args(argv)
return args.fullscreen, args.graticule, args.ext
return vars(parser.parse_args(argv))
if __name__ == "__main__":
main(*parse_args(sys.argv[1:]))
main(**parse_args(sys.argv[1:]))
......@@ -8,14 +8,78 @@
import os
def try_mkdir(dir_path, verbose=True):
from logger_cfg import getLogger
logger = getLogger(__name__)
def try_mkdir(dir_path):
"""Try to make a directory. If already exists, do not crash."""
try:
os.mkdir(dir_path)
except FileExistsError:
pass
logger.debug("Dir '%s' already exists, not creating.", dir_path)
else:
if verbose:
print("Created dir '{}'".format(dir_path))
logger.info("Created dir '%s'", dir_path)
return
def cut_array(data, every=None, average=None):
"""Filter an array: select one in every points or average over
successive points.
@param[in] data: np-array
@param[in] every: int, default None
Use only one in `every` point.
@param[in] average: int, default None
Average over `average` successive points.
If the length of the last dimension is not a multiple of the
averaging value, the last dimension is truncated.
For example, an input array with shape (10,10,14) and requesting
a 4-average will return an array with shape (10,10,3).
return: np.array
"""
if every is not None and average is not None:
logger.warning("Selecting one every %d points AND averaging over %d "
"points. This might not do what you expect.")
if every is not None:
data = data[...,::every]
if average is not None:
# Truncate last points is average is not a divisor of the data length.
max_len = (data.shape[-1] // average) * average
if data.shape[-1] != max_len:
logger.info("Truncating %d points due to averaging.",
data.shape[-1] - max_len)
# Idea: reshape the last dimension, and average over the newly created
# dimension. The original shape is conserved, expect for the length of
# the last dimension.
new_shape = data.shape[:-1] + (-1, average)
data = data[...,:max_len].reshape(new_shape).mean(-1)
return data
def get_exts_list(exts, default=("pdf", "png")):
"""Ensure the provided extension(s) are iterable.
@param[in] exts: None or string or list of string
If None, return the default list.
If a string, make it the only element of a list.
If a list, do nothing.
@param[in] default: string or list of string
The default list of extension if `exts` is None.
If a single string, make it the only element of a list.
If a list, take this as default extensions.
@return: list of string
List of plot extensions.
"""
if isinstance(default, str):
default = [default]
if not exts: # None or empty list
exts = list(default) # Convert to list in case that's a tuple.
if isinstance(exts, str):
exts = [exts]
return exts
......@@ -2,14 +2,20 @@
"""
@author Antoine Laudrain <antoine.laudrain@uni-mainz.de>
@date October 2020
@date October 2020, last update May 2021
@brief Collection of functions for recording and processing waveforms.
"""
import os.path
from glob import glob
from enum import Enum
import numpy as np
from logger_cfg import getLogger
logger = getLogger(__name__)
class WavFormat(Enum):
# pylint: disable=invalid-name
ASCii = 0
......@@ -86,6 +92,7 @@ def process_preamble(pre):
bw_min = float(items[23])
segcount = None
if len(pre) > 23:
logger.debug("Has segcount item.")
segcount = int(items[24])
return wav_format, wav_type, npoints, count, \
xinc, xorigin, xref, yinc, yorigin, yref, \
......@@ -101,7 +108,7 @@ def get_time_points(npoints, xinc, xorigin):
@param[in] yinc: float, gain of scope (mV/ADC).
@param[in] yorigin: float, offset of y axis.
"""
return np.arange(npoints) * xinc + xorigin
return np.arange(npoints, dtype=np.float32) * xinc + xorigin
def process_waveform(waveform, yinc, yorigin, bytes_per_point):
......@@ -118,7 +125,13 @@ def process_waveform(waveform, yinc, yorigin, bytes_per_point):
"""
# Switch dtype to read based on byteness ('h' or 'b'). SIGNED!!!
dtype = np.int16 if bytes_per_point == 2 else np.int8
return np.fromstring(waveform, dtype=dtype) * yinc + yorigin
logger.debug("bytes per point: %d => dtype: %s", bytes_per_point, dtype)
# The oscilloscope precision is 10 bits, so it doesn't make sense to use
# float64 (52-bit mantissa): a float32 is enough (23-bit mantissa).
# /!\ Do not use float16: mantissa is only 8 bits. /!\
# Casting the multiplicand to float32 forces the computation (and result)
# to happen with float32 (otherwise float64 is used).
return np.frombuffer(waveform, dtype=dtype) * np.float32(yinc) + yorigin
def process_sequence(sequence, points_per_wf, yinc, yorigin, bytes_per_point=2):
......@@ -138,6 +151,8 @@ def process_sequence(sequence, points_per_wf, yinc, yorigin, bytes_per_point=2):
of this event.
"""
bytes_per_wf = points_per_wf * bytes_per_point
logger.debug("points/wf * bytes/point = bytes/wf: %d * %d = %d",
points_per_wf, bytes_per_point, bytes_per_wf)
# Split byte-string corresponding to a sequence of waveform into an array
# of byte-strings.
......@@ -155,3 +170,198 @@ def process_sequence(sequence, points_per_wf, yinc, yorigin, bytes_per_point=2):
for waveform in wfs
]
################################################################################
# Convenience functions
def process_data_file(fpath_raw):
"""All-in-one function:
takes one raw file path and return time points and waveforms.
@param[in] fpath_raw: string
Path to the .raw data file.
Assumes the corresponding .dat file sits right next to it (same
directory and basename, just the extension changes).
@return: (1d-np.array, list of 1d-np.array)
time points, list of waveforms.
"""
logger.info("Processing: %s", fpath_raw)
fpath_raw = os.path.normpath(fpath_raw)
if not os.path.exists(fpath_raw):
# raise FileNotFoundError("{} does not exist.".format(fpath_raw))
logger.error("%s does not exist, skipping.", fpath_raw)
return np.array([]), []
if not fpath_raw.endswith(".raw"):
logger.error("%s doesn't look like RAW data, skipping", fpath_raw)
return np.array([]), []
fpath_dat = fpath_raw.replace(".raw", ".dat")
if not os.path.exists(fpath_dat):
# raise FileNotFoundError("{} does not exist.".format(fpath_raw))
logger.error("Could not find preample file %s", fpath_dat)
return np.array([]), []
logger.debug("Preamble file: %s", fpath_dat)
with open(fpath_dat) as infile:
preamble = infile.read()
(_, _, npoints, _,
xinc, xorigin, _, yinc, yorigin, _,
_, _, _, _, _,
_, _, _, _, _,
_, _, _, _, _) = process_preamble(preamble)
with open(fpath_raw, 'rb') as infile:
raw_data = infile.read()
return (get_time_points(npoints, xinc, xorigin),
process_sequence(raw_data, npoints, yinc, yorigin))
def read_data(infile_paths, flatten=True):
"""All-in-one function: take file path(s) and return time + amplitudes.
@param[in] infile_paths: string or list of string
If a single string, assume it should be used to glob files.
'*' and '**' wildcards are allowed (recursive search).
If a list of string, these are assumed to be the list of files
to read.
In both cases, only the `.raw` files should be given: a `.dat`
file is assumed to be sitting next to each `.raw` file and
automatically retrieved.
@param[in] flatten: bool, default: True.
- If False, the returned waveform structure as 3-deep nested
structure: waveforms[file_index][waveform_index][point], and
the returned time series structure contains one serie per file:
time_series[file_index][point], which should be used for all
waveforms read from this file.
- If True, the file level in the returned waveform structure is
flattened: waveform[waveform_index][point]. In order to keep the
time series structure aligned, the time series of each file is
duplicated by as many as the number of waveform in the