Allow for better parametrisation of GA_Detect.py command-line script

GA_Detect.py

@@ -1,18 +1,29 @@
 """A script to process OpenSky ADS-B data in an attempt to detect go-around events at an airport."""
 from traffic.core import Traffic
 from datetime import timedelta
+import importlib
 import multiprocessing as mp
-from OS_Airports import VABB
+import metar_parse as MEP
 import OS_Funcs as OSF
 import glob
-
-
-def main(start_n, fidder, do_write):
+import click
+
+
+@click.command()
+@click.option('--top-dir', default='./')
+@click.option('--start-n', default=0)
+@click.option('--do-write', default=True)
+@click.option('--metars-file', default='VABB_METAR')
+@click.option('--airport', default='VABB')
+@click.option('--n-files-proc', default=55)
+@click.option('--pool-proc', default=16)
+@click.option('--verbose', default=False)
+def main(top_dir, start_n, do_write, metars_file, airport,
+         n_files_proc, pool_proc, verbose):
     """The main code for detecting go-arounds.
 
     Arguments:
     start_n -- The index of the first file to read
-    fidder -- the id of an open file to write log information into
     do_write -- boolean flag specifying whether to output data to textfile
 
     """
@@ -22,7 +33,6 @@ def main(start_n, fidder, do_write):
     # Of which go-arounds
     tot_n_ga = 0
 
-    top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/'
     # indir stores the opensky data
     indir = top_dir + 'INDATA/'
 
@@ -41,9 +51,9 @@ def main(start_n, fidder, do_write):
     odirs = [odir_pl_nm, odir_pl_ga, odir_da_nm, odir_da_ga]
 
     # Output filenames for saving data about go-arounds
-    out_file_ga = 'GA_MET_NEW.csv'
+    out_file_ga = top_dir + 'GA_MET_NEW.csv'
     # Output filenames for saving data about non-go-arounds
-    out_file_noga = 'GA_NOGA_NEW.csv'
+    out_file_noga = top_dir + 'GA_NOGA_NEW.csv'
 
     t_frmt = "%Y/%m/%d %H:%M:%S"
 
@@ -68,10 +78,9 @@ def main(start_n, fidder, do_write):
     colormap = {'GND': 'black', 'CL': 'green', 'CR': 'blue',
                 'DE': 'orange', 'LVL': 'purple', 'NA': 'red'}
 
-    # Number of files to open in one go
-    n_files_proc = 55
-
-    pool_proc = 100
+    metars = MEP.get_metars(metars_file, verbose=verbose)
+    rwy_list = getattr(importlib.import_module(f'OS_Airports.{airport}'),
+                       'rwy_list')
 
     f_data = []
     pool = mp.Pool(processes=pool_proc)
@@ -80,9 +89,6 @@ def main(start_n, fidder, do_write):
         print("Processing batch starting with "
               + str(main_count + 1).zfill(5) + " of "
               + str(fli_len).zfill(5))
-        fidder.write("Processing batch starting with "
-                     + str(main_count + 1).zfill(5) + " of "
-                     + str(fli_len).zfill(5) + '\n')
 
         p_list = []
         # First we load several files at once
@@ -113,11 +119,12 @@ def main(start_n, fidder, do_write):
             if (flight.stop + timedelta(minutes=5) < end_time):
                 p_list.append(pool.apply_async(OSF.proc_fl,
                                                args=(flight,
-                                                     VABB.rwy_list,
+                                                     metars,
+                                                     rwy_list,
                                                      odirs,
                                                      colormap,
                                                      True,
-                                                     False,)))
+                                                     verbose,)))
             else:
                 f_data.append(flight)
 
@@ -196,12 +203,5 @@ def main(start_n, fidder, do_write):
         nogfid.close()
 
 
-# Use this to start processing from a given file number.
-# Can be helpful if processing fails at some point.
-init_num = 0
-
-fid = open('/home/proud/Desktop/log.log', 'w')
-
-main(init_num, fid, False)
-
-fid.close()
+if __name__ == '__main__':
+    main()

OS_Funcs.py

@@ -2,7 +2,6 @@
 from scipy.interpolate import UnivariateSpline as UniSpl
 from traffic.core import Traffic
 from datetime import timedelta
-import metar_parse as MEP
 import pandas as pd
 
 import flightphase as flph
@@ -11,10 +10,6 @@
 import numpy as np
 
 
-# Read METARs from disk
-metars = MEP.get_metars('/home/proud/Desktop/GoAround_Paper/VABB_METAR')
-
-
 def estimate_rwy(df, rwy_list, verbose):
     """Guess which runway a flight is attempting to land on.
 
@@ -255,11 +250,12 @@ def check_ga(fd, verbose, first_pos=-1):
     return ga_flag, bpt
 
 
-def proc_fl(flight, check_rwys, odirs, colormap, do_save, verbose):
+def proc_fl(flight, metars, check_rwys, odirs, colormap, do_save, verbose):
     """Filter, assign phases and determine go-around status for a given flight.
 
     Inputs:
         -   A 'traffic' flight object
+        -   A dict of METARS, each as a metobs class
         -   A list storing potential landing runways to check
         -   A 4-element list specifying various output directories:
             -   normal plot output

README.md

@@ -10,7 +10,6 @@ Requires:
 
 Usage:
 First you must download aircraft data, which can be done using the `OpenSky_Get_Data` script. You can then point `GA_Detect` at the download location to scan for go-arounds.
-This tool is in very early development, so has manual tweaks that would ideally be changeable via a config file or directly via the command line call. The most important of these tweaks are listed below:
 
 ### `OpenSky_Get_Data.py`:
 
@@ -34,9 +33,40 @@ python OpenSky_Get_Data.py \
     --outdir=INDATA --n-jobs=1
 ```
 
-### In `GA_Detect.py`
-The directory structure is set at the beginning of `main()`. You will probably want to adjust this to your own requirements.
+### `GA_Detect.py`
 
-`n_files_proc` specifies how many files to process simultaneously. This should be changed to the optimal value for your hardware.
+The script that runs the actual go-around events.
 
-`pool_proc` specifies the number of multiprocessing threads to use. I have found that this can be set slightly higher than the number of cores available, as cores are not fully utilised anyway.
+Data is read and written based on a top-level directory.  The default
+is the current working directory, which can be overridden by passing
+the `--top-dir` command-line option.
+
+The file containing the appropriate METARS data can be passed using
+the `--metars-file` option.
+
+The airport can be specified using the `--airport` option.  See the
+`OS_Airports` subpackage which contains the runway definitions for the
+currently supported airports.
+
+The `--n-files-proc` option specifies how many files to process
+simultaneously. This should be changed to the optimal value for your
+hardware.
+
+The `--pool-proc` option specifies the number of multiprocessing
+threads to use. I have found that this can be set slightly higher than
+the number of cores available, as cores are not fully utilised anyway.
+
+The `GA_Detect.py` command-line script uses a few defaults.  The
+defaults are chosen to accommodate the defaults in the data fetching
+scripts.  As such, these two calls are equivalent:
+
+```bash
+python GA_Detect.py  # does the same thing as the next command:
+python GA_Detect.py \
+    --top-dir=. --metars-file=VABB_METAR --airport=VABB
+```
+
+## Running tests
+
+To run the included test suite, first install pytest via `pip install
+pytest` and then run `pytest -s tests`.

tests/data/VABB_METAR

@@ -0,0 +1,5 @@
+VABB,2019-08-10 00:00,VABB 100000Z 23010G20KT 2500 HZ SCT018 FEW025TCU BKN090 28/25 Q1000 NOSIG
+VABB,2019-08-10 00:30,VABB 100030Z 24010G20KT 2500 HZ SCT018 FEW025TCU BKN090 28/25 Q1000 TEMPO 1500 RA
+VABB,2019-08-10 01:00,VABB 100100Z 25011G21KT 2500 HZ SCT012 SCT018 BKN090 28/25 Q1000 TEMPO 1500 RA
+VABB,2019-08-10 01:30,VABB 100130Z 24014G24KT 2500 HZ FEW012 SCT018 BKN090 28/25 Q1001 NOSIG
+VABB,2019-08-10 02:00,VABB 100200Z 24009G19KT 2500 HZ FEW012 SCT018 BKN090 28/26 Q1001 NOSIG

tests/data/expected/GA_MET_NEW.csv

@@ -0,0 +1,2 @@
+ICAO24, Callsign, GA_Time, L_Time, Runway, Heading, Alt, Lat,                       Lon, gapt, rocvar, hdgvar, latvar, lonvar, gspvar,                       Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,                      CB, Vis, Pressure
+04c147,KQA204,2019/08/10 00:21:24,2019/08/10 00:19:25,27,263.9909940425055,1347.6201812316162,19.08824157714844,72.8559820992606,370,1.3554425701202464,0.0027863795136474465,3.5391823044835526e-06,0.00021195186832100995,0.03189928263824995,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0

tests/data/expected/GA_NOGA_NEW.csv

@@ -0,0 +1,7 @@
+ICAO24, Callsign, GA_Time, L_Time, Runway, gapt, rocvar,                       hdgvar, latvar, lonvar, gspvar,                       Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base,                      CB, Vis, Pressure
+8005ee,AIC317,2019/08/10 00:26:12,2019/08/10 00:16:54,27,0,1.5964404850235485,0.0006783574870077095,2.4668402388350134e-06,0.0001930544997218994,0.11175369742826806,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0
+800732,AIC124,2019/08/10 00:05:45,2019/08/10 00:00:01,27,0,1.4635712542703654,0.009514184629520878,2.7480109772978474e-06,0.00019246633015406858,0.026726124191242435,28.0,25.0,10.0,20.0,230.0,1800.0,0,2500.0,1000.0
+800bb6,IGO218,2019/08/10 00:45:44,2019/08/10 00:38:25,27,0,0.7520124217363429,0.00612862439723081,2.324620750136083e-06,0.00016878190829216932,0.04790138895453693,28.0,25.0,11.0,21.0,250.0,1200.0,0,2500.0,1000.0
+800c69,GOW030,2019/08/10 00:08:06,2019/08/10 00:00:01,27,0,0.8958133916464194,0.00029074779492562153,2.0333079803969063e-06,0.0001726147052984897,0.036253078686998626,28.0,25.0,10.0,20.0,230.0,1800.0,0,2500.0,1000.0
+800c71,IGO456,2019/08/10 00:48:02,2019/08/10 00:41:43,27,0,0.950105112352263,0.011971063410794125,2.681003129011264e-06,0.00020661348328495407,0.09270944570168702,28.0,25.0,11.0,21.0,250.0,1200.0,0,2500.0,1000.0
+800dac,IGO235,2019/08/10 00:32:31,2019/08/10 00:24:02,27,0,0.8887419529611325,0.01015078239663644,2.1995557883975666e-06,0.00015366319326703428,0.05454545454545455,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0

tests/test_GA_detect.py

@@ -0,0 +1,42 @@
+import os
+import shutil
+import sys
+
+from click.testing import CliRunner
+import pytest
+
+sys.path.insert(0, '.')
+
+
+@pytest.fixture
+def data_dir():
+    return os.path.join(os.path.dirname(__file__), 'data')
+
+
+@pytest.fixture
+def data_dir_copy(data_dir, tmpdir):
+    dest = str(tmpdir) + '/' + os.path.basename(data_dir)
+    shutil.copytree(data_dir, dest)
+    return dest
+
+
+class TestMain:
+    @pytest.fixture
+    def vabb(self, data_dir_copy):
+        from GA_Detect import main
+        runner = CliRunner()
+        result = runner.invoke(main, [
+            '--top-dir', data_dir_copy + '/',
+            '--metars-file', os.path.join(data_dir_copy, 'VABB_METAR'),
+            '--pool-proc', '1',
+            ])
+        assert result.exit_code == 0
+        return result
+
+    def test_csv(self, vabb, data_dir, data_dir_copy):
+        for fname in ['GA_MET_NEW.csv', 'GA_NOGA_NEW.csv']:
+            with open(os.path.join(data_dir, 'expected', fname)) as f:
+                csv_expected = f.read()
+            with open(os.path.join(data_dir_copy, fname)) as f:
+                csv_got = f.read()
+            assert csv_expected == csv_got