Allow for better parametrisation of GA_Detect.py command-line script

This includes the ability to configure the metars file via the
command-line.  The loading of the metars file has been moved to the
main process to enable this.

Author: dnouri

GA_Detect.py

@@ -1,19 +1,31 @@
 """A script to process OpenSky ADS-B data in an attempt to detect go-around events at an airport."""
 from traffic.core import Traffic
 from datetime import timedelta
+import importlib
 import multiprocessing as mp
-from OS_Airports import VABB
+import metar_parse as MEP
 import OS_Funcs as OSF
 import os
 import glob
-
-
-def main(start_n, fidder, do_write):
+import click
+
+
+@click.command()
+@click.option('--top-dir', default='./')
+@click.option('--start-n', default=0)
+@click.option('--do-write/--no-write', default=True)
+@click.option('--do-plot/--no-plot', default=True)
+@click.option('--metars-file', default='VABB_METAR')
+@click.option('--airport', default='VABB')
+@click.option('--n-files-proc', default=55)
+@click.option('--pool-proc', default=16)
+@click.option('--verbose', default=False)
+def main(top_dir, start_n, do_write, do_plot, metars_file, airport,
+         n_files_proc, pool_proc, verbose):
     """The main code for detecting go-arounds.
 
     Arguments:
     start_n -- The index of the first file to read
-    fidder -- the id of an open file to write log information into
     do_write -- boolean flag specifying whether to output data to textfile
 
     """
@@ -23,7 +35,6 @@ def main(start_n, fidder, do_write):
     # Of which go-arounds
     tot_n_ga = 0
 
-    top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/'
     # indir stores the opensky data
     indir = top_dir + 'INDATA/'
 
@@ -44,9 +55,9 @@ def main(start_n, fidder, do_write):
         os.makedirs(odir, exist_ok=True)
 
     # Output filenames for saving data about go-arounds
-    out_file_ga = 'GA_MET_NEW.csv'
+    out_file_ga = top_dir + 'GA_MET_NEW.csv'
     # Output filenames for saving data about non-go-arounds
-    out_file_noga = 'GA_NOGA_NEW.csv'
+    out_file_noga = top_dir + 'GA_NOGA_NEW.csv'
 
     t_frmt = "%Y/%m/%d %H:%M:%S"
 
@@ -71,10 +82,9 @@ def main(start_n, fidder, do_write):
     colormap = {'GND': 'black', 'GN': 'black', 'CL': 'green', 'CR': 'blue',
                 'DE': 'orange', 'LVL': 'purple', 'NA': 'red'}
 
-    # Number of files to open in one go
-    n_files_proc = 55
-
-    pool_proc = 100
+    metars = MEP.get_metars(metars_file, verbose=verbose)
+    rwy_list = getattr(importlib.import_module(f'OS_Airports.{airport}'),
+                       'rwy_list')
 
     f_data = []
     pool = mp.Pool(processes=pool_proc)
@@ -83,9 +93,6 @@ def main(start_n, fidder, do_write):
         print("Processing batch starting with "
               + str(main_count + 1).zfill(5) + " of "
               + str(fli_len).zfill(5))
-        fidder.write("Processing batch starting with "
-                     + str(main_count + 1).zfill(5) + " of "
-                     + str(fli_len).zfill(5) + '\n')
 
         p_list = []
         # First we load several files at once
@@ -116,11 +123,12 @@ def main(start_n, fidder, do_write):
             if (flight.stop + timedelta(minutes=5) < end_time):
                 p_list.append(pool.apply_async(OSF.proc_fl,
                                                args=(flight,
-                                                     VABB.rwy_list,
+                                                     metars,
+                                                     rwy_list,
                                                      odirs,
                                                      colormap,
-                                                     True,
-                                                     False,)))
+                                                     do_plot,
+                                                     verbose,)))
             else:
                 f_data.append(flight)
 
@@ -199,12 +207,5 @@ def main(start_n, fidder, do_write):
         nogfid.close()
 
 
-# Use this to start processing from a given file number.
-# Can be helpful if processing fails at some point.
-init_num = 0
-
-fid = open('/home/proud/Desktop/log.log', 'w')
-
-main(init_num, fid, False)
-
-fid.close()
+if __name__ == '__main__':
+    main()

OS_Funcs.py

@@ -2,7 +2,6 @@
 from scipy.interpolate import UnivariateSpline as UniSpl
 from traffic.core import Traffic
 from datetime import timedelta
-import metar_parse as MEP
 import pandas as pd
 
 import flightphase as flph
@@ -11,10 +10,6 @@
 import numpy as np
 
 
-# Read METARs from disk
-metars = MEP.get_metars('/home/proud/Desktop/GoAround_Paper/VABB_METAR')
-
-
 def estimate_rwy(df, rwy_list, verbose):
     """Guess which runway a flight is attempting to land on.
 
@@ -255,11 +250,12 @@ def check_ga(fd, verbose, first_pos=-1):
     return ga_flag, bpt
 
 
-def proc_fl(flight, check_rwys, odirs, colormap, do_save, verbose):
+def proc_fl(flight, metars, check_rwys, odirs, colormap, do_save, verbose):
     """Filter, assign phases and determine go-around status for a given flight.
 
     Inputs:
         -   A 'traffic' flight object
+        -   A dict of METARS, each as a metobs class
         -   A list storing potential landing runways to check
         -   A 4-element list specifying various output directories:
             -   normal plot output

README.md

@@ -15,7 +15,6 @@ Requires:
 
 Usage:
 First you must download aircraft data, which can be done using the `OpenSky_Get_Data` script. You can then point `GA_Detect` at the download location to scan for go-arounds.
-This tool is in very early development, so has manual tweaks that would ideally be changeable via a config file or directly via the command line call. The most important of these tweaks are listed below:
 
 ### `Metars_Get_Data.py`
 
@@ -53,9 +52,35 @@ python OpenSky_Get_Data.py \
     --outdir=INDATA --n-jobs=1
 ```
 
-### In `GA_Detect.py`
-The directory structure is set at the beginning of `main()`. You will probably want to adjust this to your own requirements.
+### `GA_Detect.py`
 
-`n_files_proc` specifies how many files to process simultaneously. This should be changed to the optimal value for your hardware.
+The script that runs the actual go-around events.
 
-`pool_proc` specifies the number of multiprocessing threads to use. I have found that this can be set slightly higher than the number of cores available, as cores are not fully utilised anyway.
+Data is read and written based on a top-level directory.  The default
+is the current working directory, which can be overridden by passing
+the `--top-dir` command-line option.
+
+The file containing the appropriate METARS data can be passed using
+the `--metars-file` option.
+
+The airport can be specified using the `--airport` option.  See the
+`OS_Airports` subpackage which contains the runway definitions for the
+currently supported airports.
+
+The `--n-files-proc` option specifies how many files to process
+simultaneously. This should be changed to the optimal value for your
+hardware.
+
+The `--pool-proc` option specifies the number of multiprocessing
+threads to use. I have found that this can be set slightly higher than
+the number of cores available, as cores are not fully utilised anyway.
+
+The `GA_Detect.py` command-line script uses a few defaults.  The
+defaults are chosen to accommodate the defaults in the data fetching
+scripts.  As such, these two calls are equivalent:
+
+```bash
+python GA_Detect.py  # does the same thing as the next command:
+python GA_Detect.py \
+    --top-dir=. --metars-file=VABB_METAR --airport=VABB
+```