Improved code

Cargo.toml

@@ -24,6 +24,12 @@ geoutils = {version="*", default_features = false}
 # read airport names
 csv = {version="*", default_features = false}
 
+# 
+async-trait = "*"
+
+# logging
+log = "*"
+
 # azure integration
 azure_storage = "*"
 azure_storage_blobs = "*"
@@ -36,3 +42,4 @@ async-recursion = "1.0"
 tinytemplate = "1.1"
 clap = { version = "4.4.6", features = ["derive"] }
 tokio = {version="1.0", features=["rt", "macros", "rt-multi-thread"]}
+simple_logger = "*"

README.md

@@ -1,5 +1,5 @@
 # Danish private flights
-This repository contains a small application that generates a text based summary of
+This repository contains a CLI application that generates a text based summary of
 private jet's flight information targetted to a Danish audience.
 
 ## How to use
@@ -8,6 +8,13 @@ private jet's flight information targetted to a Danish audience.
 2. run `cargo run --example single_day -- --tail-number "OY-GFS" --date "2023-10-20"`
 3. open `OY-GFS_2023-10-20_0.md`
 
+Step 2. has an optional argument, `--azure-sas-token`, specifying an Azure storage container SAS
+token for account `privatejets`, container `data`.
+When used, caching of data is done on the remote container, as opposed to local disk.
+
+Furthermore, setting `--backend azure` without `azure-sas-token` provides read-access
+to the remote container.
+
 ## Assumptions
 
 * Aircrafts are uniquely identified by a tail number (aka registration number), e.g.

examples/period.rs

@@ -1,6 +1,7 @@
 use std::error::Error;
 
 use clap::Parser;
+use simple_logger::SimpleLogger;
 
 use flights::{
     emissions, load_aircraft_owners, load_aircrafts, load_owners, Aircraft, Class, Company, Fact,
@@ -31,27 +32,51 @@ fn render(context: &Context) -> Result<(), Box<dyn Error>> {
 
     let rendered = tt.render("t", context)?;
 
-    println!("Story written to {path}");
+    log::info!("Story written to {path}");
     std::fs::write(path, rendered)?;
     Ok(())
 }
 
+#[derive(clap::ValueEnum, Debug, Clone)]
+enum Backend {
+    LocalDisk,
+    Azure,
+}
+
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Cli {
     /// The Azure token
     #[arg(short, long)]
     azure_sas_token: Option<String>,
+    #[arg(short, long, value_enum, default_value_t=Backend::LocalDisk)]
+    backend: Backend,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn Error>> {
+    SimpleLogger::new()
+        .with_level(log::LevelFilter::Info)
+        .init()
+        .unwrap();
+
     let cli = Cli::parse();
 
-    let client = cli
-        .azure_sas_token
-        .map(|token| flights::fs_azure::initialize(&token, "privatejets", "data").unwrap());
+    // optionally initialize Azure client
+    let client = match (cli.backend, cli.azure_sas_token) {
+        (Backend::LocalDisk, None) => None,
+        (Backend::Azure, None) => Some(flights::fs_azure::initialize_anonymous(
+            "privatejets",
+            "data",
+        )),
+        (_, Some(token)) => Some(flights::fs_azure::initialize_sas(
+            &token,
+            "privatejets",
+            "data",
+        )?),
+    };
 
+    // load datasets to memory
     let owners = load_owners()?;
     let aircraft_owners = load_aircraft_owners()?;
     let aircrafts = load_aircrafts(client.as_ref()).await?;
@@ -67,19 +92,19 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let aircraft_owner = aircraft_owners
         .get(tail_number)
         .ok_or_else(|| Into::<Box<dyn Error>>::into("Owner of tail number not found"))?;
-    println!("Aircraft owner: {}", aircraft_owner.owner);
+    log::info!("Aircraft owner: {}", aircraft_owner.owner);
     let company = owners
         .get(&aircraft_owner.owner)
         .ok_or_else(|| Into::<Box<dyn Error>>::into("Owner not found"))?;
-    println!("Owner information found");
+    log::info!("Owner information found");
     let owner = Fact {
         claim: company.clone(),
         source: aircraft_owner.source.clone(),
         date: aircraft_owner.date.clone(),
     };
 
     let icao = &aircraft.icao_number;
-    println!("ICAO number: {}", icao);
+    log::info!("ICAO number: {}", icao);
 
     let iter = flights::DateIter {
         from,
@@ -103,9 +128,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
         // ignore legs that are too low, as they are likely noise
         .filter(|leg| leg.maximum_altitude > 1000.0)
         .collect::<Vec<_>>();
-    println!("number_of_legs: {}", legs.len());
+    log::info!("number_of_legs: {}", legs.len());
     for leg in &legs {
-        println!(
+        log::info!(
             "{},{},{},{},{},{},{},{},{}",
             leg.from.datetime(),
             leg.from.latitude(),

examples/single_day.rs

@@ -29,6 +29,12 @@ pub struct Context {
     pub dane_years: String,
 }
 
+#[derive(clap::ValueEnum, Debug, Clone)]
+enum Backend {
+    LocalDisk,
+    Azure,
+}
+
 /// Simple program to greet a person
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -37,8 +43,20 @@ struct Cli {
     #[arg(short, long)]
     tail_number: String,
     /// The date in format `yyyy-mm-dd`
+    #[arg(short, long, value_parser = parse_date)]
+    date: time::Date,
+    /// The Azure token
     #[arg(short, long)]
-    date: String,
+    azure_sas_token: Option<String>,
+    #[arg(short, long, value_enum, default_value_t=Backend::LocalDisk)]
+    backend: Backend,
+}
+
+fn parse_date(arg: &str) -> Result<time::Date, time::error::Parse> {
+    time::Date::parse(
+        arg,
+        time::macros::format_description!("[year]-[month]-[day]"),
+    )
 }
 
 async fn flight_date(
@@ -47,16 +65,17 @@ async fn flight_date(
     owners: &Owners,
     aircraft_owners: &AircraftOwners,
     aircrafts: &Aircrafts,
+    client: Option<&fs_azure::ContainerClient>,
 ) -> Result<Vec<Event>, Box<dyn Error>> {
     let airports = airports_cached().await?;
     let aircraft_owner = aircraft_owners
         .get(tail_number)
         .ok_or_else(|| Into::<Box<dyn Error>>::into("Owner of tail number not found"))?;
-    println!("Aircraft owner: {}", aircraft_owner.owner);
+    log::info!("Aircraft owner: {}", aircraft_owner.owner);
     let company = owners
         .get(&aircraft_owner.owner)
         .ok_or_else(|| Into::<Box<dyn Error>>::into("Owner not found"))?;
-    println!("Owner information found");
+    log::info!("Owner information found");
     let owner = Fact {
         claim: company.clone(),
         source: aircraft_owner.source.clone(),
@@ -67,17 +86,17 @@ async fn flight_date(
         .get(tail_number)
         .ok_or_else(|| Into::<Box<dyn Error>>::into("Aircraft ICAO number not found"))?;
     let icao = &aircraft.icao_number;
-    println!("ICAO number: {}", icao);
+    log::info!("ICAO number: {}", icao);
 
-    let positions = positions(icao, date, 1000.0, None).await?;
+    let positions = positions(icao, date, 1000.0, client).await?;
     let legs = legs(positions);
 
-    println!("Number of legs: {}", legs.len());
+    log::info!("Number of legs: {}", legs.len());
 
     Ok(legs.into_iter().filter_map(|leg| {
         let is_leg = matches!(leg.from, Position::Grounded{..}) & matches!(leg.to, Position::Grounded{..});
         if !is_leg {
-            println!("{:?} -> {:?} skipped", leg.from, leg.to);
+            log::info!("{:?} -> {:?} skipped", leg.from, leg.to);
         }
         is_leg.then_some((leg.from, leg.to))
     }).map(|(from, to)| {
@@ -132,7 +151,7 @@ fn process_leg(
 
     let rendered = tt.render(TEMPLATE_NAME, &context)?;
 
-    println!("Story written to {path}");
+    log::info!("Story written to {path}");
     std::fs::write(path, rendered)?;
 
     Ok(())
@@ -142,29 +161,37 @@ fn process_leg(
 async fn main() -> Result<(), Box<dyn Error>> {
     let cli = Cli::parse();
 
-    std::fs::create_dir_all("database")?;
+    // optionally initialize Azure client
+    let client = match (cli.backend, cli.azure_sas_token) {
+        (Backend::LocalDisk, None) => None,
+        (Backend::Azure, None) => Some(flights::fs_azure::initialize_anonymous(
+            "privatejets",
+            "data",
+        )),
+        (_, Some(token)) => Some(flights::fs_azure::initialize_sas(
+            &token,
+            "privatejets",
+            "data",
+        )?),
+    };
 
     let owners = load_owners()?;
     let aircraft_owners = load_aircraft_owners()?;
-    let aircrafts = load_aircrafts(None).await?;
+    let aircrafts = load_aircrafts(client.as_ref()).await?;
 
     let dane_emissions_kg = Fact {
         claim: 5100,
         source: "https://ourworldindata.org/co2/country/denmark Denmark emits 5.1 t CO2/person/year in 2019.".to_string(),
         date: "2023-10-08".to_string(),
     };
 
-    let date = time::Date::parse(
-        &cli.date,
-        time::macros::format_description!("[year]-[month]-[day]"),
-    )?;
-
     let mut events = flight_date(
         &cli.tail_number,
-        date,
+        cli.date,
         &owners,
         &aircraft_owners,
         &aircrafts,
+        client.as_ref(),
     )
     .await?;
 

src/aircraft_db.rs

@@ -7,7 +7,7 @@ use reqwest;
 use serde::{Deserialize, Serialize};
 use serde_json;
 
-use crate::fs_azure;
+use crate::{fs, fs_azure};
 
 /// [`HashMap`] between tail number (e.g. "OY-TWM") and an [`Aircraft`]
 pub type Aircrafts = HashMap<String, Aircraft>;
@@ -30,56 +30,33 @@ fn cache_file_path(prefix: &str) -> String {
     format!("{DIRECTORY}/{DATABASE}/{prefix}.json")
 }
 
-fn source(prefix: &str) -> String {
+fn url(prefix: &str) -> String {
     format!("https://globe.adsbexchange.com/{DATABASE}/{prefix}.js")
 }
 
-/// Returns a map between tail number (e.g. "OYTWM": "45D2ED")
-/// Caches to disk the first time it is executed
-async fn aircrafts_prefixed_azure(
-    prefix: &str,
-    client: &fs_azure::ContainerClient,
-) -> Result<Vec<u8>, Box<dyn Error>> {
-    let path = &cache_file_path(prefix);
-
-    let data = if !fs_azure::exists(client, path).await? {
-        let source = &source(prefix);
-        let req = reqwest::get(source).await?;
-        let data = req.bytes().await?;
-        fs_azure::put(client, &path, data.clone()).await?;
-        data.into()
-    } else {
-        fs_azure::get(client, path).await?
-    };
-    Ok(data)
+async fn aircrafts(prefix: &str) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
+    Ok(reqwest::get(url(prefix))
+        .await?
+        .bytes()
+        .await
+        .map(|x| x.into())?)
 }
 
 /// Returns a map between tail number (e.g. "OYTWM": "45D2ED")
-/// Caches to disk the first time it is executed
-async fn aircrafts_prefixed_fs(prefix: &str) -> Result<Vec<u8>, Box<dyn Error>> {
-    let path = &cache_file_path(prefix);
-    if !std::path::Path::new(path).exists() {
-        let source = &source(prefix);
-        let req = reqwest::get(source).await?;
-        let data = req.text().await?;
-        std::fs::create_dir_all(format!("{DIRECTORY}/{DATABASE}"))?;
-        std::fs::write(path, data)?;
-    }
-
-    Ok(std::fs::read(path)?)
-}
-
-/// Returns  a map between tail number (e.g. "OYTWM": "45D2ED")
-/// Caches to disk the first time it is executed
+/// Caches to disk or remote storage the first time it is executed
 async fn aircrafts_prefixed(
     prefix: String,
     client: Option<&fs_azure::ContainerClient>,
 ) -> Result<(String, HashMap<String, Vec<Option<String>>>), String> {
+    let blob_name = cache_file_path(&prefix);
+    let fetch = aircrafts(&prefix);
+
     let data = match client {
-        Some(client) => aircrafts_prefixed_azure(&prefix, client).await,
-        None => aircrafts_prefixed_fs(&prefix).await,
+        Some(client) => crate::fs::cached(&blob_name, fetch, client).await,
+        None => crate::fs::cached(&blob_name, fetch, &fs::LocalDisk).await,
     }
     .map_err(|e| e.to_string())?;
+
     Ok((
         prefix,
         serde_json::from_slice(&data).map_err(|e| e.to_string())?,
@@ -105,19 +82,22 @@ async fn children<'a: 'async_recursion>(
     .map_err(|e| e.to_string())?;
 
     // recurse over all children
-    let mut _children = vec![];
-    for entry in entries.iter_mut() {
-        _children.extend(children(&mut entry.1, client).await?)
-    }
+    let mut _children = futures::future::try_join_all(
+        entries
+            .iter_mut()
+            .map(|entry| children(&mut entry.1, client)),
+    )
+    .await?;
 
-    entries.extend(_children);
+    entries.extend(_children.into_iter().flatten());
     Ok(entries)
 }
 
 /// Returns [`Aircrafts`] known in [ADS-B exchange](https://globe.adsbexchange.com) as of 2023-11-06.
 /// It returns ~0.5m aircrafts
 /// # Implementation
-/// This function is idempotent but not pure: it caches every https request to disk to not penalize adsbexchange.com
+/// This function is idempotent but not pure: it caches every https request either to disk or remote storage
+/// to not penalize adsbexchange.com
 pub async fn load_aircrafts(
     client: Option<&fs_azure::ContainerClient>,
 ) -> Result<Aircrafts, Box<dyn Error>> {
@@ -127,12 +107,14 @@ pub async fn load_aircrafts(
     let mut entries =
         futures::future::try_join_all(prefixes.map(|x| aircrafts_prefixed(x, client))).await?;
 
-    let mut _children = vec![];
-    for entry in entries.iter_mut() {
-        _children.extend(children(&mut entry.1, client).await?)
-    }
+    let mut _children = futures::future::try_join_all(
+        entries
+            .iter_mut()
+            .map(|entry| children(&mut entry.1, client)),
+    )
+    .await?;
 
-    entries.extend(_children);
+    entries.extend(_children.into_iter().flatten());
 
     Ok(entries
         .into_iter()