diff --git a/.gitignore b/.gitignore index 66ead48..9bb0db5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ dist *.egg-info # Python binary cache files __pycache__ + +.vscode +tmp* diff --git a/README.md b/README.md index 9c993c7..1b46f95 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,258 @@ # MatSense -A toolkit for matrix sensor data processing. +[![PyPi](https://img.shields.io/pypi/v/matsense.svg)](https://pypi.org/project/MatSense/) +A toolkit that supports both real-time and off-line matrix sensor data processing and 3D visualization. +![schematic](https://raw.githubusercontent.com/atomiechen/MatSense/main/img/schematic.drawio.svg) -## Features +A typical real-time data flow would be in a client-server manner: -- Matrix data processing - - Server side: - - spatial filter - - temporal filter - - calibration - - Client side: +- Matrix sensor data: collected (e.g. by Arduino) and transmitted via a serial port to the computer. +- Data processing: the series of matrix data frames are processed and served by the server. +- Applications: clients connect to server to get processed data and do further work. + +Data can also be recorded to and processed from files. + +schematic + +3D visualization tools are provided to play real-time stream or recorded data. + + + + +## Installation + +From PyPI: + +```sh +pip install MatSense +``` + +This will install [Matplotlib](https://matplotlib.org/) to implement 3D visualization tools. + +If you want to further try [PyQtGraph](https://www.pyqtgraph.org/) as visualization method: + +```sh +pip install MatSense[pyqtgraph] +``` + + + +## Usage + +### Off-the-shelf tools + +3 handy tools are provided. Pass `-h` to get detailed information. + +- `matserver` / `python -m matsense.server` + - functions: + - receive data from serial port, process and serve + - process data from file(s) and output to file + - other helpful functions + - supported processing methods: + - voltage-pressure conversion (optional for pressure data) + - spatial filter (in-frame denoising): none, ideal, butterworth, gaussian + - temporal filter (pixel-wise between-frame denoising): none, moving average, rectangular window + - calibration: static or dynamic +- `matclient` / `python -m matsense.client`: receive server data, process and visualize; or control server via interactive commands + - supported processing methods: - interpolation - - blob filter - - touch detection -- 3D Visualization of matrix data + - blob parsing +- `matdata` / `python -m matsense.data`: visualize file data, or process data + +### Configuration + +`matserver` and `matclient` can be totally configured by a YAML configuration file: + +```sh +## server console +matserver --config + +## client console +matclient --config +``` + +Priority: commandline arguments > config file > program defaults. + +A template YAML configuration: + +```yaml +## template sensor configuration + +## ~ for defaults + +server_mode: + ## enable backend service + service: ~ + ## make server provide raw data + raw: ~ + ## enable visualization or not (suppress service) + visualize: ~ + ## enumerate all serial ports + enumerate: ~ + + ## (suppress serial) simulated data source without actual serial connection + ## debug mode: true, false + debug: ~ + + ## (suppress serial) use file as data source or not: true, false + use_file: ~ + ## input filename(s), filename or a list of filenames + in_filenames: ~ + ## output filename, default filename is used when not provided + out_filename: ~ + +client_mode: + ## make client present raw data + raw: ~ + ## interactive command line mode + interactive: ~ + +sensor: + ## sensor shape: [16, 16], [8, 8], [6, 24] + shape: ~ + ## total points, can be set to ~ + total: ~ + ## 0/1 mask to exclude non-existent points + ## |- for multiline without a newline in the end + mask: ~ + +serial: + ## baudrate: 9600, 250000, 500000, 1000000 + baudrate: ~ + ## serial port timeout, in seconds + timeout: ~ + ## serial port + port: ~ + ## data transmission protocol: simple, secure + protocol: ~ + ## support IMU data + imu: ~ + +connection: ## use defaults + ## use UDP or UNIX domain socket + udp: ~ + ## udp address format: 127.0.0.1:20503 + ## UNIX deomain socket address format: /var/tmp/unix.socket.server + server_address: ~ + client_address: ~ + +process: + ## reference voltage: 255, 255/3.6*3.3 + V0: ~ + ## constant factor: 1 + R0_RECI: ~ + ## convert voltage to resistance: true + convert: ~ + ## time of warming up in seconds: 1 + warm_up: ~ + ## spatial filter: none, ideal, butterworth, gaussian + filter_spatial: ~ + ## spatial filter cut-off freq: 3.5 + filter_spatial_cutoff: ~ + ## Butterworth filter order: 2 + butterworth_order: ~ + ## temporal filter: none, moving average, rectangular window + filter_temporal: ~ + ## temporal filter size: 15 + filter_temporal_size: ~ + ## rectangular window filter cut-off frequency: 0.04 + rw_cutoff: ~ + ## calibrative frames, 0 for no calibration: 0, 200 + cali_frames: ~ + ## calibration frame window size, 0 for static and >0 for dynamic: 0, 10000 + cali_win_size: ~ + ## interpolation shape, default to sensor.shape + interp: ~ + ## interpolation order: 3 + interp_order: ~ + ## filter out blobs: true + blob: ~ + ## total blob number: 3 + blob_num: ~ + ## blob filter threshole: 0.1, 0.15 + threshold: ~ + ## special check for certain hardwares: false + special_check: ~ + ## intermediate result: 0, 1, 2 + ## 0: convert voltage to reciprocal resistance + ## 1: convert & spatial filter + ## 2: convert & spatial filter & temporal filter + intermediate: ~ + +pointing: + ## value bound for checking cursor moving state: 0 + bound: ~ + ## directly map coordinates or relatively (suppress trackpoint) + direct_map: ~ + ## use ThinkPad's TrackPoint (red dot) control style + trackpoint: ~ + ## smoothing + alpha: ~ + +visual: + ## using pyqtgraph or matplotlib + pyqtgraph: ~ + ## z-axis limit: 3, 5 + zlim: ~ + ## frame rate: 100 + fps: ~ + ## scatter plot: false + scatter: ~ + ## show text value: false + show_value: ~ +``` + + + +### Useful modules + +- `matsense.uclient` + - `Uclient`: interface to receive data from server + +- `matsense.serverkit`: server-related processing tools + - `DataHandlerPressure`: process pressure data (conversion & filtering & calibration) +- `matsense.process`: client-related processing tools + - `BlobParser` + - `Interpolator` + - `PointSmoother` + - `CursorController` + - `PressureSelector` + +- `matense.tools`: configuration and other helpful tools +- `matsense.filemanager`: file I/O tools +- `matsense.visual`: visualization tools + - `from matsense.visual.player_matplot import Player3DMatplot`: 3D player using Matplotlib + - `from matsense.visual.player_pyqtgraph import Player3DPyqtgraph`: 3D player using PyQtGraph + + + + +## Server-Client Protocol + +Use `matclient -i` to control server. + +The underlying server-client communication protocol is： + +| Name | meaning | Value | Format | Return | Return format | +| ------------ | ------------------------------------- | ------------------ | ------------ | -------------------- | -------------- | +| CLOSE | close server | 0 | 1byte | status | 1byte | +| DATA | get a data frame | 1 | 1byte | frame+index | 256double+1int | +| RAW | get a raw data frame | 2 | 1byte | frame+index | 256double+1int | +| REC_DATA | ask server to record data to file | 3(+filename) | 1byte+string | status+filename | 1byte+string | +| REC_RAW | ask server to record raw data to file | 4(+filename) | 1byte+string | status+filename | 1byte+string | +| REC_STOP | ask server to stop recording | 5 | 1byte | status | 1byte | +| RESTART | restart server with config string | 6(+config_str) | 1byte+string | status+config_string | 1byte+string | +| RESTART_FILE | restart server with config filename | 10+config_filename | 1byte+string | status+config_string | 1byte+string | +| CONFIG | get server config | 7 | 1byte | status+config_string | 1byte+string | +| DATA_IMU | get IMU data | 9 | 1byte | IMU_frame+index | 6double+1int | + +- `status` (1 byte): 0 for success and 255 for failure -## Contact +## Author -Atomie CHEN, atomic_cwh@163.com +Atomie CHEN: atomic_cwh@163.com diff --git a/img/player.png b/img/player.png new file mode 100644 index 0000000..0fa4f34 Binary files /dev/null and b/img/player.png differ diff --git a/img/schematic.drawio.svg b/img/schematic.drawio.svg new file mode 100644 index 0000000..3db659c --- /dev/null +++ b/img/schematic.drawio.svg @@ -0,0 +1,274 @@ + + + + + + + + + + +
+
+
+ Computer +
+
+
+ + + Computer + + + + + + + + + +
+
+
+ Matrix Data Source +
+ + (e.g. Arduino) + +
+
+
+ + + Matrix Data Sour... + + + + + + + + + +
+
+
+ matserver +
+
+
+ + + matserver + + + + + + + + +
+
+
+ Serial +
+
+
+ + + Serial + + + + + + + + + +
+
+
+ matclient +
+
+
+ + + matclient + + + + + + + + + +
+
+
+ matclient +
+
+
+ + + matclient + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ ... +
+
+
+ + + ... + + + + + + +
+
+
+ Socket +
+
+
+ + + Socket + + + + + + +
+
+
+ Sensor Data +
+
+
+ + + Sensor Data + + + + + + +
+
+
+ Data Processing +
+
+
+ + + Data Processing + + + + + + +
+
+
+ Applications +
+
+
+ + + Applications + + + + + + + + +
+
+
+ File Data +
+
+
+ + + File Data + + + + + + + + +
+
+
+ matdata +
+
+
+ + + matdata + + + + + + + + + + + + + + + + + + + Viewer does not support full SVG 1.1 + + + + \ No newline at end of file diff --git a/matsense/blank_template.yaml b/matsense/blank_template.yaml index de3f025..ca01dcc 100644 --- a/matsense/blank_template.yaml +++ b/matsense/blank_template.yaml @@ -12,6 +12,17 @@ server_mode: ## enumerate all serial ports enumerate: ~ + ## (suppress serial) simulated data source without actual serial connection + ## debug mode: true, false + debug: ~ + + ## (suppress serial) use file as data source or not: true, false + use_file: ~ + ## input filename(s), filename or a list of filenames + in_filenames: ~ + ## output filename, default filename is used when not provided + out_filename: ~ + client_mode: ## make client present raw data raw: ~ diff --git a/matsense/client.py b/matsense/client.py index 81d065a..aa650bb 100644 --- a/matsense/client.py +++ b/matsense/client.py @@ -10,7 +10,7 @@ from matsense.uclient import Uclient from matsense.process import Processor from matsense.tools import ( - load_config, blank_config, check_config, make_action, DEST_SUFFIX + dump_config, load_config, blank_config, check_config, make_action, DEST_SUFFIX ) N = 16 @@ -19,187 +19,150 @@ TH = 0.15 UDP = False -def print_paras(paras): - print(f"Server parameters:") - print(f" initiating frame number: {paras[0]}") - print(f" calibration window size: {paras[1]}") - print(f" filter: {paras[2]}") - my_filter = paras[2] - if my_filter == 0: - print(f" None") - elif my_filter == 1: - print(f" exponential smoothing") - print(f" * ALPHA: {paras[3]}") - print(f" * BETA: {paras[4]}") - elif my_filter == 2: - print(f" moving average") - print(f" * kernel size: {paras[3]}") - elif my_filter == 3: - print(f" sinc low-pass filter") - print(f" * kernel size: {paras[3]}") - print(f" * cut-off frequency: {paras[4]}") +interactive_commands = { + "close": CMD.CLOSE, + "data": CMD.DATA, + "raw": CMD.RAW, + "rec_data": CMD.REC_DATA, + "rec_raw": CMD.REC_RAW, + "rec_stop": CMD.REC_STOP, + "restart": CMD.RESTART, + "restart_file": CMD.RESTART_FILE, + "config": CMD.CONFIG, + "data_imu": CMD.DATA_IMU, +} -def interactive_cmd(my_client, my_cmd): - """interactive command parser +help_msg = { + "close": "close server", + "data": "get a data frame", + "raw": "get a raw data frame", + "rec_data": "send signal to record data", + "rec_raw": "send signal to record raw data", + "rec_stop": "stop recording", + "restart": "restart server, optional arg: configuration filename (relative to client path)", + "restart_file": "restart server with configuration filename (relative to server path)", + "config": "get server configuration", + "data_imu": "get an IMU data frame", +} - Show hints to help input specific parameters. - - Args: - my_cmd (CMD): a predefined command - """ - if my_cmd not in CMD.__members__.values(): - print("Unknown command!") - return +def print_config(config): + config_str = dump_config(config) + print(config_str) - try: - if my_cmd == CMD.CLOSE: - my_client.send_cmd(my_cmd) - elif my_cmd in (CMD.DATA, CMD.RAW): - my_client.send_cmd(my_cmd) - frame_idx = my_client.recv_frame()[1] - print(f"frame_idx: {frame_idx}") - elif my_cmd in (CMD.REC_DATA, CMD.REC_RAW): - print(f"recording filename:") - my_filename = input("|> ").strip() - my_client.send_cmd(my_cmd, my_filename) - ret, recv_filename = my_client.recv_string() - if ret == 0: - if my_cmd == CMD.REC_DATA: - data_mode = "processed" - else: - data_mode = "raw" - print(f"recording {data_mode} data to file: {recv_filename}") - else: - print(f"fail to write to file: {recv_filename}") - elif my_cmd == CMD.REC_STOP: - my_client.send_cmd(my_cmd) - ret, recv_str = my_client.recv_string() - if ret == 0: - print("stop recording") - else: - print("fail to stop recording!") - elif my_cmd == CMD.RESTART: - try: - print(f"initiating frame number:") - obtained = input("|> ").strip() - if obtained: - my_initcali = int(obtained) - else: - my_initcali = -1 - print(f"{my_initcali}") - print(f"calibration window size:") - obtained = input("|> ").strip() - if obtained: - my_win = int(obtained) - else: - my_win = -1 - print(f"{my_win}") - print(f"filter:") - obtained = input("|> ").strip() - if obtained: - my_filter = int(obtained) - else: - my_filter = -1 - print(f"{my_filter}") - args = [my_initcali, my_win, my_filter] - if my_filter == 1: - print(f"exponential smoothing.") - print(f"alpha:") - obtained = input("|> ").strip() - if obtained: - my_alpha = float(obtained) - else: - my_alpha = -1.0 - print(f"{my_alpha}") - print(f"beta:") - obtained = input("|> ").strip() - if obtained: - my_beta = float(obtained) - else: - my_beta = -1.0 - print(f"{my_beta}") - args.append(my_alpha) - args.append(my_beta) - elif my_filter == 2: - print(f"moving average.") - print(f"kernel size:") - obtained = input("|> ").strip() - if obtained: - my_ma_size = int(obtained) - else: - my_ma_size = -1 - print(f"{my_ma_size}") - args.append(my_ma_size) - elif my_filter == 3: - print(f"sinc low-pass filter.") - print(f"kernel size:") - obtained = input("|> ").strip() - if obtained: - my_lp_size = int(obtained) - else: - my_lp_size = -1 - print(f"{my_lp_size}") - print(f"cut-off frequency:") - obtained = input("|> ").strip() - if obtained: - my_lp_w = float(obtained) - else: - my_lp_w = -1.0 - print(f"{my_lp_size}") - args.append(my_lp_size) - args.append(my_lp_w) - - my_client.send_cmd(my_cmd, args=args) - results = my_client.recv_paras() - if results[0] == 0: - print(f"server restarting...") - else: - print("fail to restart server") - print_paras(results[1:]) - except ValueError: - print("invalid arguments to restart server") - elif my_cmd == CMD.PARAS: - my_client.send_cmd(my_cmd) - results = my_client.recv_paras() - print_paras(results[1:]) - elif my_cmd == CMD.REC_BREAK: - my_client.send_cmd(my_cmd) - ret, recv_str = my_client.recv_string() - if ret == 0: - print("successfully break") - else: - print("fail to break!") - elif my_cmd == CMD.DATA_IMU: - my_client.send_cmd(my_cmd) - data_imu, frame_idx = my_client.recv_imu() - print(f"IMU data: {data_imu}") - print(f"frame_idx: {frame_idx}") - - except (FileNotFoundError, ConnectionResetError): - print("server off-line") - except ConnectionRefusedError: - print("server refused connection") - except timeout: - print("server no response") +def print_help(): + print("Usage: ") + for key, value in interactive_commands.items(): + print(f" {key} / {value}: {help_msg[key]}") + print("Type 'help' to get this message.") def run_client_interactive(my_client): + print_help() while True: + unknown = False + try: data = input('>> ').strip() - if data and "quit".startswith(data) or data == "exit": - return except (EOFError, KeyboardInterrupt): return - try: - data = data.strip().split() - if not data: - raise Exception - my_cmd = int(data[0]) - except: + if not data: + unknown = True + elif "quit".startswith(data) or data == "exit": + return + elif data == "help": + print_help() continue + elif data in interactive_commands: + my_cmd = interactive_commands[data] + else: + try: + my_cmd = int(data) + if my_cmd not in CMD.__members__.values(): + unknown = True + except: + unknown = True + + if unknown: + print("Unknown command!") + continue + + try: + if my_cmd == CMD.CLOSE: + my_client.send_cmd(my_cmd) + elif my_cmd in (CMD.DATA, CMD.RAW): + my_client.send_cmd(my_cmd) + frame_idx = my_client.recv_frame()[1] + print(f"frame_idx: {frame_idx}") + elif my_cmd in (CMD.REC_DATA, CMD.REC_RAW): + print(f"recording filename:") + my_filename = input("|> ").strip() + my_client.send_cmd(my_cmd, my_filename) + ret, recv_filename = my_client.recv_string() + if ret == 0: + if my_cmd == CMD.REC_DATA: + data_mode = "processed" + else: + data_mode = "raw" + print(f"recording {data_mode} data to file: {recv_filename}") + else: + print(f"fail to write to file: {recv_filename}") + elif my_cmd == CMD.REC_STOP: + my_client.send_cmd(my_cmd) + ret, recv_str = my_client.recv_string() + if ret == 0: + print("stop recording") + else: + print("fail to stop recording!") + elif my_cmd == CMD.RESTART: + print("RESTART server") + print("client-side config filename:") + config_filename = input("|> ").strip() + if config_filename != "": + with open(config_filename, 'r', encoding='utf-8') as f: + config_str = f.read() + else: + config_str = "" + my_client.send_cmd(my_cmd, config_str) + ret, config = my_client.recv_config() + print("Received config:") + print_config(config) + if ret == 0: + print("server restarting...") + else: + print("server failted to restart") + elif my_cmd == CMD.RESTART_FILE: + print("RESTART server") + print("server-side config filename:") + config_filename = input("|> ").strip() + if config_filename == "": + print("must input filename!!!") + else: + my_client.send_cmd(my_cmd, config_filename) + ret, config = my_client.recv_config() + print("Received config:") + print_config(config) + if ret == 0: + print("server restarting...") + else: + print("server failted to restart") + elif my_cmd == CMD.CONFIG: + my_client.send_cmd(my_cmd) + ret, config = my_client.recv_config() + print("Received config:") + print_config(config) + elif my_cmd == CMD.DATA_IMU: + my_client.send_cmd(my_cmd) + data_imu, frame_idx = my_client.recv_imu() + print(f"IMU data: {data_imu}") + print(f"frame_idx: {frame_idx}") + + except (FileNotFoundError, ConnectionResetError): + print("server off-line") + except ConnectionRefusedError: + print("server refused connection") + except timeout: + print("server no response") - interactive_cmd(my_client, my_cmd) def prepare_config(args): ## load config and combine commandline arguments @@ -226,8 +189,8 @@ def prepare_config(args): config['visual']['zlim'] = args.zlim if config['visual']['fps'] is None or hasattr(args, 'fps'+DEST_SUFFIX): config['visual']['fps'] = args.fps - if config['visual']['pyqtgraph'] is None or hasattr(args, 'matplot'+DEST_SUFFIX): - config['visual']['pyqtgraph'] = not args.matplot + if config['visual']['pyqtgraph'] is None or hasattr(args, 'pyqtgraph'+DEST_SUFFIX): + config['visual']['pyqtgraph'] = args.pyqtgraph if config['visual']['scatter'] is None or hasattr(args, 'scatter'+DEST_SUFFIX): config['visual']['scatter'] = args.scatter if config['visual']['show_value'] is None or hasattr(args, 'show_value'+DEST_SUFFIX): @@ -236,12 +199,12 @@ def prepare_config(args): config['client_mode']['raw'] = args.raw if config['client_mode']['interactive'] is None or hasattr(args, 'interactive'+DEST_SUFFIX): config['client_mode']['interactive'] = args.interactive - check_config(config) ## some modifications if config['process']['interp'] is None: config['process']['interp'] = copy.deepcopy(config['sensor']['shape']) + check_config(config) return config @@ -258,7 +221,8 @@ def main(): parser.add_argument('-i', '--interactive', dest='interactive', action=make_action('store_true'), default=False, help="interactive mode") parser.add_argument('-z', '--zlim', dest='zlim', action=make_action('store'), default=ZLIM, type=float, help="z-axis limit") parser.add_argument('-f', dest='fps', action=make_action('store'), default=FPS, type=int, help="frames per second") - parser.add_argument('-m', '--matplot', dest='matplot', action=make_action('store_true'), default=False, help="use mathplotlib to plot") + parser.add_argument('--pyqtgraph', dest='pyqtgraph', action=make_action('store_true'), default=False, help="use pyqtgraph to plot") + # parser.add_argument('-m', '--matplot', dest='matplot', action=make_action('store_true'), default=False, help="use mathplotlib to plot") parser.add_argument('--config', dest='config', action=make_action('store'), default=None, help="specify configuration file") parser.add_argument('--scatter', dest='scatter', action=make_action('store_true'), default=False, help="show scatter plot") diff --git a/matsense/cmd.py b/matsense/cmd.py index 4137b0d..e56bf82 100644 --- a/matsense/cmd.py +++ b/matsense/cmd.py @@ -11,10 +11,10 @@ class CMD(IntEnum): REC_DATA (int): record processed data to file REC_RAW (int): record raw data to file REC_STOP (int): stop recording - RESTART (int): restart the server with processing parameters - PARAS (int): get current processing parameters of the server - REC_BREAK (int): stop current recording and start a new one + RESTART (int): restart the server with configuration string + CONFIG (int): get current configuration of the server DATA_IMU (int): get IMU data frame and frame index + RESTART_FILE (int): restart the server with configuration filename """ CLOSE = 0 @@ -24,6 +24,6 @@ class CMD(IntEnum): REC_RAW = 4 REC_STOP = 5 RESTART = 6 - PARAS = 7 - REC_BREAK = 8 + CONFIG = 7 DATA_IMU = 9 + RESTART_FILE = 10 diff --git a/matsense/data.py b/matsense/data.py index 0c2fb24..1fa3127 100644 --- a/matsense/data.py +++ b/matsense/data.py @@ -1,7 +1,7 @@ import argparse import numpy as np -from matsense.filemanager import parse_line, write_line +from matsense.filemanager import parse_line, write_line, clear_file from matsense.serverkit import Proc from matsense.process import Processor from matsense.tools import check_shape @@ -22,7 +22,8 @@ def main(): parser.add_argument('filename', action='store') parser.add_argument('-n', dest='n', action='store', default=[N], type=int, nargs='+', help="specify sensor shape") parser.add_argument('-f', dest='fps', action='store', default=FPS, type=int, help="frames per second") - parser.add_argument('-m', '--matplot', dest='matplot', action='store_true', default=False, help="use mathplotlib to plot") + parser.add_argument('--pyqtgraph', dest='pyqtgraph', action='store_true', default=False, help="use pyqtgraph to plot") + # parser.add_argument('-m', '--matplot', dest='matplot', action='store_true', default=False, help="use mathplotlib to plot") parser.add_argument('-z', '--zlim', dest='zlim', action='store', default=ZLIM, type=float, help="z-axis limit") parser.add_argument('-o', dest='output', action='store', default=None, help="output processed data to file") parser.add_argument('--interp', dest='interp', action='store', default=[INTERP], type=int, nargs='+', help="interpolated side size") @@ -49,10 +50,9 @@ def main(): print(f"writing to file: {args.output}") ## clear file content - with open(args.output, 'w') as fout: - pass + clear_file(args.output) cnt = 0 - with open(filename, 'r') as fin: + with open(filename, 'r', encoding='utf-8') as fin: for line in fin: data_parse, frame_idx, date_time = parse_line(line, args.n[0]*args.n[1], ',') data_out = my_processor.transform(data_parse, reshape=True) @@ -64,7 +64,7 @@ def main(): else: print("Data visualization mode") content = ([], []) - with open(filename, 'r') as fin: + with open(filename, 'r', encoding='utf-8') as fin: for line in fin: data_parse, frame_idx, date_time = parse_line(line, args.n[0]*args.n[1], ',') if args.convert: @@ -73,10 +73,10 @@ def main(): content[0].append(np.array(data_reshape)) content[1].append(f"frame idx: {frame_idx} {date_time}") - if args.matplot: - from matsense.visual.player_matplot import Player3DMatplot as Player - else: + if args.pyqtgraph: from matsense.visual.player_pyqtgraph import Player3DPyqtgraph as Player + else: + from matsense.visual.player_matplot import Player3DMatplot as Player my_player = Player(zlim=args.zlim, widgets=True, N=args.n) my_player.run_interactive(dataset=content[0], infoset=content[1], fps=args.fps) diff --git a/matsense/filemanager.py b/matsense/filemanager.py index 416cd69..003584f 100644 --- a/matsense/filemanager.py +++ b/matsense/filemanager.py @@ -8,6 +8,8 @@ ## Convenient module to instantly write to files by closing the file after each writing operation ## Helpful when there are multiple writings and the program may be interrupted midway +ENCODING = 'utf-8' + ## check if the file root path exists, and create if not def check_root(filename): root_dir, bare_filename = os.path.split(filename) @@ -19,19 +21,19 @@ def check_root(filename): def write(filename, content, override=False): check_root(filename) mode = 'w' if override else 'a' - with open(filename, mode) as fout: + with open(filename, mode, encoding=ENCODING) as fout: fout.write(content) ## write multiple lines to file def writelines(filename, lines, override=False): check_root(filename) mode = 'w' if override else 'a' - with open(filename, mode) as fout: + with open(filename, mode, encoding=ENCODING) as fout: fout.writelines(lines) ## read all lines in file into a list def readlines(filename): - with open(filename, 'r') as fin: + with open(filename, 'r', encoding=ENCODING) as fin: ret = fin.readlines() return ret @@ -98,5 +100,5 @@ def write_lines(filename, data, tags=None, delim=',', override=False): ## clear file content def clear_file(filename): check_root(filename) - with open(filename, "w"): + with open(filename, "w", encoding=ENCODING): pass diff --git a/matsense/server.py b/matsense/server.py index be40a69..2c80840 100644 --- a/matsense/server.py +++ b/matsense/server.py @@ -51,8 +51,9 @@ def enumerate_ports(): print(item) def task_serial(paras): + ret = None try: - if paras['debug']: + if paras['config']['server_mode']['debug']: my_setter = DataSetterDebug() else: my_setter = DataSetterSerial( @@ -89,7 +90,7 @@ def task_serial(paras): imu=paras['config']['serial']['imu'], intermediate=paras['config']['process']['intermediate'] ) - my_proc.run() + ret = my_proc.run() except KeyboardInterrupt: pass except CustomException as e: @@ -99,8 +100,9 @@ def task_serial(paras): # print(e) finally: ## close the other process - paras['pipe_server'].send((FLAG.FLAG_STOP,)) + paras['pipe_proc'].send((FLAG.FLAG_STOP,)) print("Processing stopped.") + return ret def task_server(paras): try: @@ -113,6 +115,7 @@ def task_server(paras): total=paras['config']['sensor']['total'], udp=paras['config']['connection']['udp'], pipe_conn=paras['pipe_server'], + config_copy=paras['config'], ) as my_server: my_server.run_service() except KeyboardInterrupt: @@ -124,13 +127,13 @@ def task_server(paras): # print(e) finally: ## close the other process - paras['pipe_proc'].send((FLAG.FLAG_STOP,)) + paras['pipe_server'].send((FLAG.FLAG_STOP,)) def task_file(paras): - print(f"Processed data saved to: {paras['output']}") + print(f"Processed data saved to: {paras['config']['server_mode']['out_filename']}") my_setter = DataSetterFile( paras['config']['sensor']['total'], - paras['filenames'], + paras['config']['server_mode']['in_filenames'], ) my_proc = Proc( paras['config']['sensor']['shape'], @@ -153,11 +156,11 @@ def task_file(paras): cali_frames=paras['config']['process']['cali_frames'], cali_win_size=paras['config']['process']['cali_win_size'], pipe_conn=None, - output_filename=paras['output'], + output_filename=paras['config']['server_mode']['out_filename'], copy_tags=True, ) ## clear file content - clear_file(paras['output']) + clear_file(paras['config']['server_mode']['out_filename']) my_proc.run() def prepare_config(args): @@ -189,52 +192,33 @@ def prepare_config(args): config['visual']['pyqtgraph'] = args.pyqtgraph if config['visual']['scatter'] is None or hasattr(args, 'scatter'+DEST_SUFFIX): config['visual']['scatter'] = args.scatter - if config['server_mode']['service'] is None or hasattr(args, 'service'+DEST_SUFFIX): - config['server_mode']['service'] = args.service + if config['server_mode']['service'] is None or hasattr(args, 'noservice'+DEST_SUFFIX): + config['server_mode']['service'] = not args.noservice if config['server_mode']['raw'] is None or hasattr(args, 'raw'+DEST_SUFFIX): config['server_mode']['raw'] = args.raw if config['server_mode']['visualize'] is None or hasattr(args, 'visualize'+DEST_SUFFIX): config['server_mode']['visualize'] = args.visualize if config['server_mode']['enumerate'] is None or hasattr(args, 'enumerate'+DEST_SUFFIX): config['server_mode']['enumerate'] = args.enumerate + if config['server_mode']['debug'] is None or hasattr(args, 'debug'+DEST_SUFFIX): + config['server_mode']['debug'] = args.debug + if config['server_mode']['out_filename'] is None or hasattr(args, 'output'+DEST_SUFFIX): + config['server_mode']['out_filename'] = args.output if config['serial']['imu'] is None or hasattr(args, 'imu'+DEST_SUFFIX): config['serial']['imu'] = args.imu if config['process']['intermediate'] is None or hasattr(args, 'intermediate'+DEST_SUFFIX): config['process']['intermediate'] = args.intermediate - check_config(config) - return config - -def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-e', dest='enumerate', action=make_action('store_true'), default=False, help="enumerate all serial ports") - parser.add_argument('-p', dest='port', action=make_action('store'), default=PORT, help="specify serial port") - parser.add_argument('-b', dest='baudrate', action=make_action('store'), default=BAUDRATE, type=int, help="specify baudrate") - parser.add_argument('-t', dest='timeout', action=make_action('store'), default=TIMEOUT, type=float, help="specify timeout in seconds") - parser.add_argument('-n', dest='n', action=make_action('store'), default=[N], type=int, nargs='+', help="specify sensor shape") - parser.add_argument('-s', '--service', dest='service', action=make_action('store_true'), default=False, help="run service") - parser.add_argument('-a', '--address', dest='address', action=make_action('store'), help="specify server socket address") - parser.add_argument('-u', '--udp', dest='udp', action=make_action('store_true'), default=UDP, help="use UDP protocol") - parser.add_argument('-r', '--raw', dest='raw', action=make_action('store_true'), default=False, help="raw data mode") - parser.add_argument('-nc', '--no_convert', dest='no_convert', action=make_action('store_true'), default=NO_CONVERT, help="do not apply voltage-resistance conversion") - parser.add_argument('-v', '--visualize', dest='visualize', action=make_action('store_true'), default=False, help="enable visualization") - parser.add_argument('-z', '--zlim', dest='zlim', action=make_action('store'), default=ZLIM, type=float, help="z-axis limit") - parser.add_argument('-f', dest='fps', action=make_action('store'), default=FPS, type=int, help="frames per second") - parser.add_argument('--pyqtgraph', dest='pyqtgraph', action=make_action('store_true'), default=False, help="use pyqtgraph to plot") - # parser.add_argument('-m', '--matplot', dest='matplot', action=make_action('store_true'), default=False, help="use matplotlib to plot") - parser.add_argument('--config', dest='config', action=make_action('store'), default=None, help="specify configuration file") - parser.add_argument('-d', '--debug', dest='debug', action=make_action('store_true'), default=DEBUG, help="debug mode") - - parser.add_argument('filenames', nargs='*', action='store') - parser.add_argument('-o', dest='output', action='store', default=OUTPUT_FILENAME, help="output processed data to file") - - parser.add_argument('-i', '--imu', dest='imu', action=make_action('store_true'), default=False, help="support IMU") + ## some modifications + if args.filenames: + config['server_mode']['use_file'] = True + config['server_mode']['in_filenames'] = args.filenames - parser.add_argument('--scatter', dest='scatter', action=make_action('store_true'), default=False, help="show scatter plot") - parser.add_argument('--intermediate', dest='intermediate', action=make_action('store'), default=INTERMEDIATE, type=int, help="specify intermediate result") + check_config(config) + return config - args = parser.parse_args() - config = prepare_config(args) +def run(config): + ret = None ## enumerate serial ports if config['server_mode']['enumerate']: @@ -244,29 +228,29 @@ def main(): print_sensor(config) ## shared variables + ## output data array data_out = Array('d', config['sensor']['total']) # d for double + ## raw data array data_raw = Array('d', config['sensor']['total']) # d for double + ## imu data array data_imu = Array('d', 6) # d for double - idx_out = Value('i') # i for int - idx_out_file = Value('i') - + ## frame index + idx_out = Value('i') # i for signed int + ## Proc-Userver communication pipe pipe_proc, pipe_server = Pipe(duplex=True) + ## function parameters paras = { "config": config, "data_out": data_out, "data_raw": data_raw, "data_imu": data_imu, "idx_out": idx_out, - "idx_out_file": idx_out_file, "pipe_proc": pipe_proc, "pipe_server": pipe_server, - "debug": args.debug, - "filenames": args.filenames, - "output": args.output, } - if args.filenames: + if config['server_mode']['use_file']: task_file(paras) return @@ -298,10 +282,62 @@ def main(): p_server = Process(target=task_server, args=(paras,)) p_server.start() - task_serial(paras) + ret = task_serial(paras) if config['server_mode']['service']: p_server.join() + + del data_out + del data_raw + del data_imu + del idx_out + del pipe_proc, pipe_server + + return ret + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-e', dest='enumerate', action=make_action('store_true'), default=False, help="enumerate all serial ports") + parser.add_argument('-p', dest='port', action=make_action('store'), default=PORT, help="specify serial port") + parser.add_argument('-b', dest='baudrate', action=make_action('store'), default=BAUDRATE, type=int, help="specify baudrate") + parser.add_argument('-t', dest='timeout', action=make_action('store'), default=TIMEOUT, type=float, help="specify timeout in seconds") + parser.add_argument('-n', dest='n', action=make_action('store'), default=[N], type=int, nargs='+', help="specify sensor shape") + parser.add_argument('--noservice', dest='noservice', action=make_action('store_true'), default=False, help="do not run service (only serial data receiving & processing)") + parser.add_argument('-a', '--address', dest='address', action=make_action('store'), help="specify server socket address") + parser.add_argument('-u', '--udp', dest='udp', action=make_action('store_true'), default=UDP, help="use UDP protocol") + parser.add_argument('-r', '--raw', dest='raw', action=make_action('store_true'), default=False, help="raw data mode") + parser.add_argument('-nc', '--no_convert', dest='no_convert', action=make_action('store_true'), default=NO_CONVERT, help="do not apply voltage-resistance conversion") + parser.add_argument('-v', '--visualize', dest='visualize', action=make_action('store_true'), default=False, help="enable visualization") + parser.add_argument('-z', '--zlim', dest='zlim', action=make_action('store'), default=ZLIM, type=float, help="z-axis limit") + parser.add_argument('-f', dest='fps', action=make_action('store'), default=FPS, type=int, help="frames per second") + parser.add_argument('--scatter', dest='scatter', action=make_action('store_true'), default=False, help="show scatter plot") + parser.add_argument('--pyqtgraph', dest='pyqtgraph', action=make_action('store_true'), default=False, help="use pyqtgraph to plot") + # parser.add_argument('-m', '--matplot', dest='matplot', action=make_action('store_true'), default=False, help="use matplotlib to plot") + parser.add_argument('--config', dest='config', action=make_action('store'), default=None, help="specify configuration file") + parser.add_argument('-d', '--debug', dest='debug', action=make_action('store_true'), default=DEBUG, help="debug mode") + + parser.add_argument('filenames', nargs='*', action='store', help="use file(s) as data source instead of serial port") + parser.add_argument('-o', dest='output', action=make_action('store'), default=OUTPUT_FILENAME, help="output processed data to file") + + parser.add_argument('-i', '--imu', dest='imu', action=make_action('store_true'), default=False, help="support IMU") + + parser.add_argument('--intermediate', dest='intermediate', action=make_action('store'), default=INTERMEDIATE, type=int, help="specify intermediate result") + + args = parser.parse_args() + config = prepare_config(args) + + while True: + ## run according to config + ret = run(config) + + if ret != None: + if ret[0] == 1: ## restart + config = ret[1] + continue + + ## exit program + break if __name__ == '__main__': diff --git a/matsense/serverkit/__init__.py b/matsense/serverkit/__init__.py index 6d8fb39..383602e 100644 --- a/matsense/serverkit/__init__.py +++ b/matsense/serverkit/__init__.py @@ -3,3 +3,4 @@ from .userver import Userver from .exception import CustomException, SerialTimeout, FileEnd from .data_setter import DataSetterSerial, DataSetterFile, DataSetterDebug +from .data_handler import DataHandlerPressure, DataHandlerIMU diff --git a/matsense/serverkit/data_setter.py b/matsense/serverkit/data_setter.py index 3312d3a..55dc4c5 100644 --- a/matsense/serverkit/data_setter.py +++ b/matsense/serverkit/data_setter.py @@ -138,7 +138,7 @@ def __init__(self, total, filenames): self.fin = None def open_next_file(self): - self.fin = open(self.filenames[self.file_idx], 'r') + self.fin = open(self.filenames[self.file_idx], 'r', encoding='utf-8') self.file_idx += 1 def __call__(self, data_tmp, *args, **kwargs): diff --git a/matsense/serverkit/flag.py b/matsense/serverkit/flag.py index f927761..de57ae6 100644 --- a/matsense/serverkit/flag.py +++ b/matsense/serverkit/flag.py @@ -12,7 +12,6 @@ class FLAG(IntEnum): FLAG_REC_STOP = 3 FLAG_REC_DATA = 4 FLAG_REC_RAW = 5 - FLAG_REC_BREAK = 6 FLAG_REC_RET_SUCCESS = 7 FLAG_REC_RET_STOP = 8 diff --git a/matsense/serverkit/proc.py b/matsense/serverkit/proc.py index 4a501b0..e33d9b5 100644 --- a/matsense/serverkit/proc.py +++ b/matsense/serverkit/proc.py @@ -33,7 +33,6 @@ def __init__(self, n, data_setter, data_out, data_raw, data_imu, idx_out, **kwar ## recording self.record_raw = False self.filename = None - self.filename_id = 0 self.tags = None ## copy tags from data setter to output file, ## if False, generate tags using current frame index and timestamp @@ -104,60 +103,6 @@ def post_action(self): self.tags = [self.idx_out.value, timestamp] write_line(self.filename, data_ptr, tags=self.tags) - def loop_proc(self): - print("Running processing...") - while True: - ## check signals from the other process - if self.pipe_conn is not None: - if self.pipe_conn.poll(): - msg = self.pipe_conn.recv() - # print(f"msg={msg}") - flag = msg[0] - if flag == FLAG.FLAG_STOP: - break - if flag in (FLAG.FLAG_REC_DATA, FLAG.FLAG_REC_RAW): - self.record_raw = True if flag == FLAG.FLAG_REC_RAW else True - filename = msg[1] - if filename == "": - if flag == FLAG.FLAG_REC_RAW: - filename = datetime.now().strftime(self.FILENAME_TEMPLATE_RAW) - else: - filename = datetime.now().strftime(self.FILENAME_TEMPLATE) - try: - with open(filename, 'a') as fout: - pass - if self.filename is not None: - print(f"stop recording: {self.filename}") - self.filename = filename - print(f"recording to: {self.filename}") - self.pipe_conn.send((FLAG.FLAG_REC_RET_SUCCESS,self.filename)) - except: - print(f"failed to record: {self.filename}") - self.pipe_conn.send((FLAG.FLAG_REC_RET_FAIL,)) - - elif flag == FLAG.FLAG_REC_STOP: - if self.filename is not None: - print(f"stop recording: {self.filename}") - self.filename = None - self.filename_id = 0 - elif flag == FLAG.FLAG_REC_BREAK: - self.filename_id += 1 - - try: - self.get_raw_frame() - except SerialTimeout: - continue - except FileEnd: - print(f"Processing time: {time.time()-self.start_time:.3f} s") - break - self.cur_time = time.time() - self.data_raw[:] = self.data_tmp - if not self.my_raw: - self.filter() - self.calibrate() - self.data_out[:] = self.data_tmp - self.post_action() - def warm_up(self): print("Warming up processing...") begin = time.time() @@ -167,21 +112,9 @@ def warm_up(self): except SerialTimeout: pass - def run_org(self): - if self.WARM_UP > 0: - self.warm_up() - - self.start_time = time.time() - self.reset() - self.print_proc() - - if not self.my_raw: - self.prepare_spatial() - self.prepare_temporal() - self.prepare_cali() - self.loop_proc() - def run(self): + ret = None + if self.WARM_UP > 0: self.warm_up() @@ -212,6 +145,11 @@ def gen_imu(): flag = msg[0] if flag == FLAG.FLAG_STOP: break + if flag == FLAG.FLAG_RESTART: + config_new = msg[1] + ## restart with new config + ret = (1, config_new) + break if flag in (FLAG.FLAG_REC_DATA, FLAG.FLAG_REC_RAW): self.record_raw = True if flag == FLAG.FLAG_REC_RAW else True filename = msg[1] @@ -221,7 +159,7 @@ def gen_imu(): else: filename = datetime.now().strftime(self.FILENAME_TEMPLATE) try: - with open(filename, 'a') as fout: + with open(filename, 'a', encoding='utf-8') as fout: pass if self.filename is not None: print(f"stop recording: {self.filename}") @@ -236,9 +174,6 @@ def gen_imu(): if self.filename is not None: print(f"stop recording: {self.filename}") self.filename = None - self.filename_id = 0 - elif flag == FLAG.FLAG_REC_BREAK: - self.filename_id += 1 try: self.get_raw_frame() @@ -258,3 +193,5 @@ def gen_imu(): self.handler_pressure.final() self.handler_imu.final() + + return ret diff --git a/matsense/serverkit/userver.py b/matsense/serverkit/userver.py index 5a9c48d..5d097a0 100644 --- a/matsense/serverkit/userver.py +++ b/matsense/serverkit/userver.py @@ -13,6 +13,7 @@ from .flag import FLAG from ..cmd import CMD +from ..tools import dump_config, load_config, parse_config, combine_config class Userver: @@ -35,7 +36,7 @@ class Userver: TOTAL = 16 * 16 TIMEOUT = 0.1 - BUF_SIZE = 2048 + BUF_SIZE = 8192 REC_ID = 0 def __init__(self, data_out, data_raw, data_imu, idx_out, server_addr=None, **kwargs): @@ -55,7 +56,8 @@ def __init__(self, data_out, data_raw, data_imu, idx_out, server_addr=None, **kw self.init_socket() - def config(self, *, total=None, udp=None, timeout=None, pipe_conn=None): + def config(self, *, total=None, udp=None, timeout=None, + pipe_conn=None, config_copy=None): if total is not None: self.TOTAL = total if udp is not None: @@ -64,6 +66,8 @@ def config(self, *, total=None, udp=None, timeout=None, pipe_conn=None): self.TIMEOUT = timeout if pipe_conn is not None: self.pipe_conn = pipe_conn + if config_copy is not None: + self.config_copy = config_copy def init_socket(self): if not support_unix_socket: @@ -77,7 +81,7 @@ def init_socket(self): self.my_socket = socket(AF_UNIX, SOCK_DGRAM) self.my_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) - self.my_socket.setsockopt(SOL_SOCKET, SO_SNDBUF, self.frame_size*2) + self.my_socket.setsockopt(SOL_SOCKET, SO_SNDBUF, max(self.frame_size*2, self.BUF_SIZE)) self.my_socket.settimeout(self.TIMEOUT) if not self.server_addr: @@ -109,48 +113,6 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.exit() - def proc_cmd(self): - if self.data[0] == CMD.CLOSE: - reply = pack("=B", 0) - self.my_socket.sendto(reply, self.client_addr) - self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) - return CMD.CLOSE - elif self.data[0] == CMD.DATA: - reply = pack(self.frame_format, *(self.data_out), self.idx_out.value) - self.my_socket.sendto(reply, self.client_addr) - elif self.data[0] == CMD.RAW: - reply = pack(self.frame_format, *(self.data_raw), self.idx_out.value) - self.my_socket.sendto(reply, self.client_addr) - elif self.data[0] in (CMD.REC_DATA, CMD.REC_RAW): - if self.data[0] == CMD.REC_DATA: ## processed data - self.pipe_conn.send((FLAG.FLAG_REC_DATA, str(self.data[1:], encoding = "utf-8"))) - else: ## raw data - self.pipe_conn.send((FLAG.FLAG_REC_RAW, str(self.data[1:], encoding = "utf-8"))) - msg = self.pipe_conn.recv() - flag = msg[0] - if flag == FLAG.FLAG_REC_RET_SUCCESS: - reply = pack("=B", 0) + msg[1].encode('utf-8') - else: - reply = pack("=B", 255) - self.my_socket.sendto(reply, self.client_addr) - elif self.data[0] == CMD.REC_STOP: - reply = pack("=B", 0) - self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) - self.my_socket.sendto(reply, self.client_addr) - elif self.data[0] == CMD.RESTART: - ## TODO - pass - elif self.data[0] == CMD.PARAS: - ## TODO - pass - elif self.data[0] == CMD.REC_BREAK: - reply = pack("=B", 0) - self.my_socket.sendto(reply, self.client_addr) - self.pipe_conn.send((FLAG.FLAG_REC_BREAK,)) - elif self.data[0] == CMD.DATA_IMU: - reply = pack("=6di", *(self.data_imu), self.idx_out.value) - self.my_socket.sendto(reply, self.client_addr) - def print_service(self): if self.UDP: protocol_str = 'UDP' @@ -166,17 +128,95 @@ def run_service(self): ## check signals from the other process if self.pipe_conn is not None: if self.pipe_conn.poll(): - flag = self.pipe_conn.recv() + msg = self.pipe_conn.recv() + flag = msg[0] if flag == FLAG.FLAG_STOP: break ## try to receive requests from client(s) try: self.data, self.client_addr = self.my_socket.recvfrom(self.BUF_SIZE) - ret = self.proc_cmd() - if ret == CMD.CLOSE: + + if self.data[0] == CMD.CLOSE: + reply = pack("=B", 0) + self.my_socket.sendto(reply, self.client_addr) + self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) self.pipe_conn.send((FLAG.FLAG_STOP,)) break + elif self.data[0] == CMD.DATA: + reply = pack(self.frame_format, *(self.data_out), self.idx_out.value) + self.my_socket.sendto(reply, self.client_addr) + elif self.data[0] == CMD.RAW: + reply = pack(self.frame_format, *(self.data_raw), self.idx_out.value) + self.my_socket.sendto(reply, self.client_addr) + elif self.data[0] in (CMD.REC_DATA, CMD.REC_RAW): + if self.data[0] == CMD.REC_DATA: ## processed data + self.pipe_conn.send((FLAG.FLAG_REC_DATA, str(self.data[1:], encoding = "utf-8"))) + else: ## raw data + self.pipe_conn.send((FLAG.FLAG_REC_RAW, str(self.data[1:], encoding = "utf-8"))) + msg = self.pipe_conn.recv() + flag = msg[0] + if flag == FLAG.FLAG_REC_RET_SUCCESS: + reply = pack("=B", 0) + msg[1].encode('utf-8') + else: + reply = pack("=B", 255) + self.my_socket.sendto(reply, self.client_addr) + elif self.data[0] == CMD.REC_STOP: + reply = pack("=B", 0) + self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) + self.my_socket.sendto(reply, self.client_addr) + elif self.data[0] == CMD.RESTART: + success = False + config_new = self.config_copy + try: + content = str(self.data[1:], encoding='utf-8') + if content != "": + config_new = parse_config(content) + config_new = combine_config(self.config_copy, config_new) + else: + config_new = self.config_copy + reply = pack("=B", 0) + dump_config(config_new).encode('utf-8') + success = True + except: + reply = pack("=B", 255) + dump_config(self.config_copy).encode('utf-8') + success = False + + self.my_socket.sendto(reply, self.client_addr) + + if success: + self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) + self.pipe_conn.send((FLAG.FLAG_RESTART,config_new)) + break + elif self.data[0] == CMD.RESTART_FILE: + success = False + config_new = self.config_copy + try: + filename = str(self.data[1:], encoding='utf-8') + if filename != "": + config_new = load_config(filename) + config_new = combine_config(self.config_copy, config_new) + reply = pack("=B", 0) + dump_config(config_new).encode('utf-8') + success = True + else: + reply = pack("=B", 255) + dump_config(self.config_copy).encode('utf-8') + success = False + except: + reply = pack("=B", 255) + dump_config(self.config_copy).encode('utf-8') + success = False + + self.my_socket.sendto(reply, self.client_addr) + + if success: + self.pipe_conn.send((FLAG.FLAG_REC_STOP,)) + self.pipe_conn.send((FLAG.FLAG_RESTART,config_new)) + break + elif self.data[0] == CMD.CONFIG: + reply = pack("=B", 0) + dump_config(self.config_copy).encode('utf-8') + self.my_socket.sendto(reply, self.client_addr) + elif self.data[0] == CMD.DATA_IMU: + reply = pack("=6di", *(self.data_imu), self.idx_out.value) + self.my_socket.sendto(reply, self.client_addr) + except timeout: pass except (FileNotFoundError, ConnectionResetError): diff --git a/matsense/tools.py b/matsense/tools.py index 060b803..421e7bc 100644 --- a/matsense/tools.py +++ b/matsense/tools.py @@ -28,6 +28,11 @@ def parse_ip_port(content): return (ip, port) +def dump_ip_port(ip_port): + ip, port = ip_port + return str(ip)+":"+str(port) + + def check_shape(n): try: n[1] @@ -47,18 +52,28 @@ def parse_mask(string_in): mask = np.array(mask, dtype=int) return mask +def dump_mask(mask_array): + lines = [] + for row in mask_array: + str_list = [str(item) for item in row] + lines.append(" ".join(str_list)) + return "\n".join(lines) -## recursion -def check_config(config): - def recurse(dict_default, dict_target): - for key in dict_default: - if key in dict_target: - if isinstance(dict_default[key], dict): - recurse(dict_default[key], dict_target[key]) - else: +## recursion, fill dict_target according to dict_default +def __recurse(dict_default, dict_target): + for key in dict_default: + if key in dict_target: + if dict_target[key] is None: dict_target[key] = copy.deepcopy(dict_default[key]) + elif isinstance(dict_default[key], dict): + __recurse(dict_default[key], dict_target[key]) + else: + dict_target[key] = copy.deepcopy(dict_default[key]) + + +def check_config(config): ## recurse to fill empty fields - recurse(BLANK, config) + __recurse(BLANK, config) ## some transformation for certain fields if config['sensor']['shape'] is not None: config['sensor']['shape'] = check_shape(config['sensor']['shape']) @@ -77,22 +92,45 @@ def recurse(dict_default, dict_target): # ## dangerous to use 'eval' # config['process']['V0'] = eval(config['process']['V0']) - ## some modifications - if config['process']['interp'] is None: - config['process']['interp'] = copy.deepcopy(config['sensor']['shape']) + +def parse_config(content): + config = yaml.safe_load(content) + check_config(config) + return config def load_config(filename): - with open(filename) as fin: + with open(filename, 'r', encoding='utf-8') as fin: config = yaml.safe_load(fin) check_config(config) return config +def dump_config(config): + config_copy = copy.deepcopy(config) + + ## some transformation for certain fields + if config_copy['sensor']['mask'] is not None: + config_copy['sensor']['mask'] = dump_mask(config_copy['sensor']['mask']) + if config_copy['connection']['server_address'] is not None: + config_copy['connection']['server_address'] = dump_ip_port(config_copy['connection']['server_address']) + if config_copy['connection']['client_address'] is not None: + config_copy['connection']['client_address'] = dump_ip_port(config_copy['connection']['client_address']) + + return yaml.safe_dump(config_copy) + + def blank_config(): return copy.deepcopy(BLANK) +def combine_config(configA, configB): + config = copy.deepcopy(configA) + __recurse(configB, config) + check_config(config) + return config + + def print_sensor(config, tab=''): print(f"{tab}Sensor shape: {config['sensor']['shape']}") print(f"{tab}Sensor size: {config['sensor']['total']}") diff --git a/matsense/uclient.py b/matsense/uclient.py index 9435fd2..798924f 100644 --- a/matsense/uclient.py +++ b/matsense/uclient.py @@ -4,7 +4,8 @@ """ from socket import ( - socket, AF_INET, SOCK_DGRAM, gethostname, gethostbyname + socket, AF_INET, SOCK_DGRAM, gethostname, gethostbyname, + SOL_SOCKET, SO_SNDBUF ) try: from socket import AF_UNIX @@ -19,7 +20,7 @@ from typing import Iterable import time -from .tools import check_shape +from .tools import check_shape, parse_config from .cmd import CMD @@ -63,7 +64,7 @@ class Uclient: N = 16 TIMEOUT = 0.1 - BUF_SIZE = 4096 + BUF_SIZE = 8192 def __init__(self, client_addr=None, @@ -161,6 +162,7 @@ def init_socket(self): break self.binded = True self.my_socket.settimeout(self.TIMEOUT) + self.my_socket.setsockopt(SOL_SOCKET, SO_SNDBUF, self.BUF_SIZE) if not self.server_addr: if self.UDP: @@ -250,33 +252,10 @@ def recv_string(self): label = "" return self.data[0], label - def recv_paras(self): - """receive process parameters from server - - Returns: - list: variable-length list containing: - - **ret** (*bytes*): 1 byte of return value, 0 for success and - 255 for failure - - **i** (*int*): initiating frame number - - **w** (*int*): calibration window size - - **f** (*int*): filter index - - and other parameters corresponding to the filter. - """ - paras = unpack_from("=B3i", self.data) - start = calcsize("=B3i") - my_filter = paras[-1] - if my_filter == 1: # exponential smoothing - paras += unpack_from("=2d", self.data, start) - elif my_filter == 2: # moving average - paras += unpack_from("=i", self.data, start) - elif my_filter == 3: # sinc low-pass - paras += unpack_from("=id", self.data, start) - return paras + def recv_config(self): + ret, config_str = self.recv_string() + config = parse_config(config_str) + return ret, config def recv_imu(self): result = unpack_from("=6di", self.data) diff --git a/setup.py b/setup.py index 1e89ee4..ab3277f 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ # For a discussion on single-sourcing the version across setup.py and the # project code, see # https://packaging.python.org/guides/single-sourcing-package-version/ - version='0.2.1', # Required + version='0.3.0', # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: @@ -158,6 +158,7 @@ "pyserial>=3.5", "PyYAML>=5.4.1", "pyparsing>=2.4.7", + "matplotlib>=3.3", ], # List additional groups of dependencies here (e.g. development @@ -168,10 +169,15 @@ # # Similar to `install_requires` above, these must be valid existing # projects. - # extras_require={ # Optional - # 'dev': ['check-manifest'], - # 'test': ['coverage'], - # }, + extras_require={ # Optional + 'pyqtgraph': [ + "pyqtgraph==0.11.*", + "PyOpenGL~=3.1.5", + "PyQt5==5.12.* ; python_version == '3.7'", + "PyQt5==5.15.* ; python_version == '3.8'", + "PyQt6==6.0.* ; python_version == '3.9'", + ], + }, # If there are data files included in your packages that need to be # installed, specify them here.