From d3d994e69be5587d719ecc25c9d8721128f134c6 Mon Sep 17 00:00:00 2001 From: Andrew Hankinson Date: Mon, 11 Mar 2024 16:53:37 +0100 Subject: [PATCH] Allow processing of multiple files --- import_logs.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/import_logs.py b/import_logs.py index 0ef6d87..3f4ef91 100644 --- a/import_logs.py +++ b/import_logs.py @@ -149,7 +149,7 @@ def parse_line(line: str, lineno: int, cfg: dict) -> Optional[Hit]: return create_hit(json_record, idsite) -def main(logfile_path: str, dry_run: bool, cfg: dict) -> bool: +def parse_logfile(logfile_path: str, dry_run: bool, cfg: dict) -> bool: with open(logfile_path, mode='rt', encoding="utf-8", errors="surrogateescape") as logfile: with concurrent.futures.ThreadPoolExecutor() as executor: hit_futures = [executor.submit(parse_line, line, lineno, cfg) for lineno, line in enumerate(logfile, 1)] @@ -180,9 +180,24 @@ def main(logfile_path: str, dry_run: bool, cfg: dict) -> bool: return success +def main(logfiles: list[str], dry_run: bool, cfg: dict) -> bool: + success: bool = True + + for lf in logfiles: + log.info("Processing file %s", lf) + try: + success &= parse_logfile(lf, dry_run, cfg) + except: + log.error("Error opening file %s", lf) + success &= False + continue + + return success + + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("logfile", help="Path to the nginx logfile") + parser.add_argument("logfiles", type=str, nargs="+", help="Path to the nginx logfile") parser.add_argument("--config", "-c", default="config.toml", help="The config file") parser.add_argument("--debug", "-d", action="store_true", help="Enable debug messages") parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose messages") @@ -200,7 +215,7 @@ def main(logfile_path: str, dry_run: bool, cfg: dict) -> bool: with open(args.config, "rb") as f: outer_cfg: dict = tomllib.load(f) - overall_success = main(args.logfile, args.dry_run, outer_cfg) + overall_success = main(args.logfiles, args.dry_run, outer_cfg) if not overall_success: sys.exit(-1)