Moved files

.github/workflows/test.yaml

@@ -29,5 +29,5 @@ jobs:
       - name: Install Rust
         run: rustup toolchain install stable
       - uses: Swatinem/rust-cache@v2
-      - name: Generate code coverage
-        run: cargo run --example export_private_jets
+      - name: Run etl
+        run: cargo run --features="build-binary" --bin etl_private_jets

Cargo.toml

@@ -38,10 +38,47 @@ aws-config = { version = "1.1.4", features = ["behavior-version-latest"] }
 aws-sdk-s3 = "*"
 aws-credential-types = "*"
 
+clap = { version = "4.4.6", features = ["derive"], optional = true }
+tokio = { version="1.0", features=["rt", "macros", "rt-multi-thread"], optional = true }
+tinytemplate = { version = "1.1", optional = true }
+itertools = { version = "*", optional = true }
+num-format = { version = "*", optional = true }
+simple_logger = { version = "*", optional = true }
+
 [dev-dependencies]
-tinytemplate = "1.1"
-clap = { version = "4.4.6", features = ["derive"] }
 tokio = {version="1.0", features=["rt", "macros", "rt-multi-thread"]}
-simple_logger = "*"
-num-format = "*"
 itertools = "*"
+
+[features]
+build-binary = [
+    "clap",
+    "tokio",
+    "tinytemplate",
+    "itertools",
+    "num-format",
+    "simple_logger",
+]
+
+[[bin]]
+name = "etl_legs"
+required-features = ["build-binary"]
+
+[[bin]]
+name = "etl_positions"
+required-features = ["build-binary"]
+
+[[bin]]
+name = "etl_private_jets"
+required-features = ["build-binary"]
+
+[[bin]]
+name = "period"
+required-features = ["build-binary"]
+
+[[bin]]
+name = "country"
+required-features = ["build-binary"]
+
+[[bin]]
+name = "single_day"
+required-features = ["build-binary"]

README.md

@@ -32,7 +32,7 @@ This repository contains both a Rust library and a set of [`examples/`](./exampl
 to perform actual calculations. To use one of such examples:
 
 1. Install Rust
-2. run `cargo run --example single_day -- --tail-number "OY-GFS" --date "2023-10-20"`
+2. run `cargo run --features="build-binary" --bin single_day -- --tail-number "OY-GFS" --date "2023-10-20"`
 3. open `OY-GFS_2023-10-20_0.md`
 
 Step 2. has an optional arguments, `--access-key`, `--secret-access-key`, specifying
@@ -55,22 +55,22 @@ that preserves data integrity.
 
 ```bash
 # Story about Danish private jets that flew to Davos between two dates
-cargo run --example country -- --from=2024-01-13 --to=2024-01-21 --country=denmark --location=davos
+cargo run --features="build-binary" --bin country -- --from=2024-01-13 --to=2024-01-21 --country=denmark --location=davos
 # Story about Danish private jets that flew between two dates
-cargo run --example country -- --from=2024-01-13 --to=2024-01-21 --country=denmark
+cargo run --features="build-binary" --bin country -- --from=2024-01-13 --to=2024-01-21 --country=denmark
 # Story about Portuguese private jets that flew between two dates
-cargo run --example country -- --from=2024-01-13 --to=2024-01-21 --country=portugal
+cargo run --features="build-binary" --bin country -- --from=2024-01-13 --to=2024-01-21 --country=portugal
 
 # Story about German private jets that flew in 2023, where secret is on a file
-cargo run --example country -- --from=2023-01-01 --to=2024-01-01 --country=germany --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
+cargo run --features="build-binary" --bin country -- --from=2023-01-01 --to=2024-01-01 --country=germany --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
 
 # Build database of positions `[2020, 2023]`
-cargo run --release --example etl_positions -- --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
+cargo run --features="build-binary" --release --bin etl_positions -- --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
 # they are available at
 # https://private-jets.fra1.digitaloceanspaces.com/position/icao_number={icao}/month={year}-{month}/data.json
 
 # Build database of legs `[2020, 2023]` (over existing positions computed by `etl_positions`)
-cargo run --release --example etl_legs -- --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
+cargo run --features="build-binary" --release --bin etl_legs -- --access-key=DO00AUDGL32QLFKV8CEP --secret-access-key=$(cat secrets.txt)
 # they are available at
 # https://private-jets.fra1.digitaloceanspaces.com/leg/v1/data/icao_number={icao}/month={year}-{month}/data.csv
 ```

examples/clean_cache.rs

@@ -0,0 +1,57 @@
+use clap::Parser;
+
+use flights::{fs_s3::ContainerClient, BlobStorageProvider};
+use futures::StreamExt;
+use simple_logger::SimpleLogger;
+
+async fn delete(client: &ContainerClient) -> Result<(), Box<dyn std::error::Error>> {
+    let tasks = client.list("position/icao_number=3b9b60").await?;
+
+    log::info!("{}", tasks.len());
+    let tasks = tasks
+        .into_iter()
+        .map(|blob| async move { client.delete(&blob).await });
+
+    futures::stream::iter(tasks)
+        // limit concurrent tasks
+        .buffered(200)
+        // continue if error
+        .map(|r| {
+            if let Err(e) = r {
+                log::error!("{e}");
+            }
+        })
+        .collect::<Vec<_>>()
+        .await;
+
+    Ok(())
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version)]
+struct Cli {
+    /// The token to the remote storage
+    #[arg(long)]
+    access_key: String,
+    /// The token to the remote storage
+    #[arg(long)]
+    secret_access_key: String,
+    /// Optional country to fetch from (in ISO 3166); defaults to whole world
+    #[arg(long)]
+    country: Option<String>,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    SimpleLogger::new()
+        .with_level(log::LevelFilter::Info)
+        .init()
+        .unwrap();
+
+    let cli = Cli::parse();
+
+    let client = flights::fs_s3::client(cli.access_key, cli.secret_access_key).await;
+
+    delete(&client).await?;
+    Ok(())
+}

examples/country.rs → src/bin/country.rs

@@ -14,7 +14,7 @@ use time::Date;
 fn render(context: &Context) -> Result<(), Box<dyn Error>> {
     let path = format!("{}_story.md", context.country.name.to_lowercase());
 
-    let template = std::fs::read_to_string("examples/country.md")?;
+    let template = std::fs::read_to_string("src/bin/country.md")?;
 
     let mut tt = tinytemplate::TinyTemplate::new();
     tt.set_default_formatter(&tinytemplate::format_unescaped);

examples/period.rs → src/bin/period.rs

@@ -24,7 +24,7 @@ pub struct Context {
 fn render(context: &Context) -> Result<(), Box<dyn Error>> {
     let path = "story.md";
 
-    let template = std::fs::read_to_string("examples/period_template.md")?;
+    let template = std::fs::read_to_string("src/bin/period_template.md")?;
 
     let mut tt = tinytemplate::TinyTemplate::new();
     tt.set_default_formatter(&tinytemplate::format_unescaped);

examples/single_day.rs → src/bin/single_day.rs

@@ -157,7 +157,7 @@ fn process_leg(
         dane_years,
     };
 
-    let template = std::fs::read_to_string("examples/single_day_template.md")?;
+    let template = std::fs::read_to_string("src/bin/single_day_template.md")?;
 
     let mut tt = TinyTemplate::new();
     tt.set_default_formatter(&tinytemplate::format_unescaped);