diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml index 2acb4d2..db18833 100644 --- a/.github/workflows/create-release.yaml +++ b/.github/workflows/create-release.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - project: [ myceli, controller, hyphae ] + project: [ myceli, controller, hyphae, watcher ] target: [ armv7-unknown-linux-gnueabihf, @@ -60,6 +60,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + project: [ myceli, watcher ] target: [ armv7-unknown-linux-gnueabihf, @@ -73,14 +74,14 @@ jobs: - name: clean cargo run: cargo clean - name: cargo/cross build - run: CROSS_CONFIG=Cross.toml cross build --bin myceli --target ${{ matrix.target }} --profile small --features small --no-default-features + run: CROSS_CONFIG=Cross.toml cross build --bin ${{ matrix.project }} --target ${{ matrix.target }} --profile small --features small --no-default-features - name: Compress binary run: | - tar -cvf myceli-small-${{ github.ref_name }}-${{ matrix.target }}.tar -C ./target/${{ matrix.target }}/small myceli - xz -9 --keep --extreme myceli-small-${{ github.ref_name }}-${{ matrix.target }}.tar - gzip --best myceli-small-${{ github.ref_name }}-${{ matrix.target }}.tar + tar -cvf ${{ matrix.project }}-small-${{ github.ref_name }}-${{ matrix.target }}.tar -C ./target/${{ matrix.target }}/small ${{ matrix.project }} + xz -9 --keep --extreme ${{ matrix.project }}-small-${{ github.ref_name }}-${{ matrix.target }}.tar + gzip --best ${{ matrix.project }}-small-${{ github.ref_name }}-${{ matrix.target }}.tar - name: upload binary - run: gh release upload "$GITHUB_REF" myceli-small-${{ github.ref_name }}-${{ matrix.target }}.tar.?z + run: gh release upload "$GITHUB_REF" *-small-${{ github.ref_name }}-${{ matrix.target }}.tar.?z env: GITHUB_TOKEN: ${{ github.token }} shell: bash diff --git a/.idea/space.iml b/.idea/space.iml index c28b209..34493f2 100644 --- a/.idea/space.iml +++ b/.idea/space.iml @@ -15,6 +15,7 @@ + diff --git a/Cargo.lock b/Cargo.lock index ef14700..cb62981 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -467,6 +467,7 @@ dependencies = [ "anyhow", "dirs", "figment", + "log", "serde", "transports", ] @@ -483,11 +484,11 @@ version = "0.6.5" dependencies = [ "anyhow", "clap", + "env_logger", + "log", "messages", "serde_json", "tokio", - "tracing", - "tracing-subscriber", "transports", ] @@ -653,6 +654,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "errno" version = "0.3.1" @@ -1009,6 +1023,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.26" @@ -1042,15 +1062,15 @@ dependencies = [ "chrono", "cid", "clap", + "env_logger", "figment", "ipfs-unixfs", + "log", "messages", "reqwest", "serde", "serde_json", "thiserror", - "tracing", - "tracing-subscriber", "transports", ] @@ -1375,6 +1395,7 @@ dependencies = [ "assert_fs", "bytes", "cid", + "env_logger", "futures", "ipfs-unixfs", "log", @@ -1384,7 +1405,6 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "tracing", ] [[package]] @@ -1411,15 +1431,6 @@ dependencies = [ "libc", ] -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata", -] - [[package]] name = "memchr" version = "2.5.0" @@ -1580,6 +1591,8 @@ dependencies = [ "blake2", "cid", "config", + "env_logger", + "figment", "file-hashing", "futures", "ipfs-unixfs", @@ -1590,8 +1603,6 @@ dependencies = [ "serde", "smalog", "tokio", - "tracing", - "tracing-subscriber", "transports", ] @@ -2032,15 +2043,6 @@ dependencies = [ "regex-syntax 0.7.2", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - [[package]] name = "regex-syntax" version = "0.6.29" @@ -2093,10 +2095,10 @@ name = "rfm69-service" version = "0.1.0" dependencies = [ "clap", + "env_logger", + "log", "tokio", "tokio-serial", - "tracing", - "tracing-subscriber", ] [[package]] @@ -2256,15 +2258,6 @@ dependencies = [ "keccak", ] -[[package]] -name = "sharded-slab" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" -dependencies = [ - "lazy_static", -] - [[package]] name = "slab" version = "0.4.8" @@ -2369,6 +2362,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + [[package]] name = "termtree" version = "0.4.1" @@ -2555,27 +2557,13 @@ dependencies = [ "once_cell", ] -[[package]] -name = "tracing-subscriber" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" -dependencies = [ - "matchers", - "once_cell", - "regex", - "sharded-slab", - "thread_local", - "tracing", - "tracing-core", -] - [[package]] name = "transports" version = "0.6.5" dependencies = [ "anyhow", "cid", + "env_logger", "log", "messages", "parity-scale-codec", @@ -2584,7 +2572,6 @@ dependencies = [ "serde", "serde_derive", "smalog", - "tracing", ] [[package]] @@ -2795,12 +2782,14 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" name = "watcher" version = "0.1.0" dependencies = [ + "anyhow", "config", + "env_logger", "log", + "messages", "notify", "smalog", - "tracing", - "tracing-subscriber", + "transports", ] [[package]] diff --git a/config/Cargo.toml b/config/Cargo.toml index 386841b..926fb8c 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -7,5 +7,6 @@ edition = "2021" anyhow.workspace = true dirs = "5.0.1" figment.workspace = true +log.workspace = true serde.workspace = true transports.workspace = true \ No newline at end of file diff --git a/config/src/config.rs b/config/src/config.rs index 606f36b..7e79f4e 100644 --- a/config/src/config.rs +++ b/config/src/config.rs @@ -3,6 +3,7 @@ use figment::{ providers::{Format, Serialized, Toml}, Figment, Provider, }; +use log::debug; use serde::{Deserialize, Serialize}; use transports::MAX_MTU; @@ -69,7 +70,7 @@ impl Config { let mut config = Figment::from(Serialized::defaults(Config::default())); if let Some(path) = path.or(default_path()) { let toml_values = Toml::file(&path); - println!("Config values in file {}: {:?}", &path, toml_values.data()); + debug!("Config values in file {}: {:?}", &path, toml_values.data()); config = config.merge(toml_values); } let config: Self = config.extract()?; diff --git a/local-storage/src/storage.rs b/local-storage/src/storage.rs index 31e65d6..74850ee 100644 --- a/local-storage/src/storage.rs +++ b/local-storage/src/storage.rs @@ -12,7 +12,7 @@ use std::fs::File as FsFile; use std::io::Write; use std::path::Path; -use log::{error, info}; +use log::{debug, error, info}; pub struct Storage { pub provider: Box, @@ -28,6 +28,7 @@ impl Storage { } pub fn import_path(&self, path: &Path) -> Result { + debug!("import_path({:?})", &path); let rt = tokio::runtime::Runtime::new()?; let blocks: Result> = rt.block_on(async { let file: File = FileBuilder::new() diff --git a/myceli/Cargo.toml b/myceli/Cargo.toml index bb37c20..fc7bab6 100644 --- a/myceli/Cargo.toml +++ b/myceli/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true cid = { workspace = true, features = ["scale-codec"] } +config.workspace = true env_logger = { workspace = true, optional = true } figment.workspace = true local-storage = { path = "../local-storage", default-features = false } diff --git a/myceli/src/listener.rs b/myceli/src/listener.rs index 68270af..a59a8c1 100644 --- a/myceli/src/listener.rs +++ b/myceli/src/listener.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex}; use std::thread::spawn; use transports::Transport; -use log::{debug, error, info}; +use log::{error, info}; pub struct Listener { storage_path: String, @@ -100,7 +100,7 @@ impl Listener { } } Err(_e) => { - debug!("Receive message failed: {_e}"); + println!("Receive message failed: {_e}"); } } } @@ -112,7 +112,7 @@ impl Listener { sender_addr: &str, shipper_sender: Sender<(DataProtocol, String)>, ) -> Result> { - info!("Handling {message:?}"); + println!("Handling {message:?}"); let resp = match message { Message::ApplicationAPI(ApplicationAPI::TransmitDag { cid, diff --git a/watcher/Cargo.toml b/watcher/Cargo.toml index b2d66ee..87a9bff 100644 --- a/watcher/Cargo.toml +++ b/watcher/Cargo.toml @@ -6,15 +6,17 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow.workspace = true config.workspace = true +env_logger = { workspace = true, optional = true } log.workspace = true +messages.workspace = true notify = "6.0.1" smalog = { workspace = true, optional = true } -tracing = { workspace = true, optional = true } -tracing-subscriber = { workspace = true, optional = true } +transports.workspace = true [features] big = ["good_log"] small = ["small_log"] -good_log = ["dep:tracing", "dep:tracing-subscriber"] +good_log = ["dep:env_logger"] small_log = ["dep:smalog"] diff --git a/watcher/src/handler.rs b/watcher/src/handler.rs index d0b8624..99d7546 100644 --- a/watcher/src/handler.rs +++ b/watcher/src/handler.rs @@ -1,20 +1,52 @@ -use log::{debug, error, info}; -use notify::{Event, EventKind}; +use log::{debug, error, info, trace}; +use messages::{ApplicationAPI, Message}; +use notify::{event::ModifyKind, Event, EventKind}; +use std::path::Path; +use std::time::Duration; +use transports::{Transport, UdpTransport}; -pub(crate) fn handle_event(event: notify::Result, cfg: &config::Config) { - println!("handle_event({:?})", &event); - match event { - Err(err) => { - error!("FileSystem error: {:?}", err); - } - Ok(ev) => match ev.kind { - EventKind::Modify(_) => { - info!("File modified, import: {:?}", &ev); - for p in ev.paths { - crate::net::send(&p, cfg); - } +pub(crate) struct Handler { + trx: UdpTransport, + target_addr: String, +} + +impl Handler { + pub fn new(cfg: &config::Config) -> Result { + let trx = UdpTransport::new("0.0.0.0:0", cfg.mtu, cfg.chunk_transmit_throttle)?; + let target_addr = cfg.listen_address.clone(); + Ok(Self { trx, target_addr }) + } + + pub fn handle_event(&self, event: notify::Result) { + trace!("handle_event({:?})", &event); + match event { + Err(err) => { + error!("FileSystem error: {:?}", err); } - _ => debug!("Ignoring FileSystem event: {:?}", &ev), - }, + Ok(ev) => match ev.kind { + EventKind::Modify(ModifyKind::Data(_)) => { + //Some of these events can occur while the file is still being modified + std::thread::sleep(Duration::from_millis(100)); + info!("File modified, import: {:?}", &ev); + for p in ev.paths { + self.send(&p); + } + } + _ => debug!("Ignoring FileSystem event: {:?}", &ev), + }, + } + } + pub fn send(&self, path: &Path) { + let path = if let Some(p) = path.as_os_str().to_str() { + p.to_owned() + } else { + error!("Path {:?} can't be turned into string?!", &path); + return; + }; + let m = ApplicationAPI::ImportFile { path }; + let m = Message::ApplicationAPI(m); + if let Err(e) = self.trx.send(m, &self.target_addr) { + error!("Error sending: {:?}", &e); + } } } diff --git a/watcher/src/main.rs b/watcher/src/main.rs index d3d5f6e..7fd8618 100644 --- a/watcher/src/main.rs +++ b/watcher/src/main.rs @@ -2,7 +2,9 @@ use notify::Watcher; use std::{fs, path::PathBuf, time::Duration}; mod handler; -mod net; + +#[cfg(all(not(feature = "small"), not(feature = "big")))] +compile_error! {"Select either big or small feature"} fn watched_dir(cfg: &config::Config) -> PathBuf { let mut result = PathBuf::new(); @@ -17,35 +19,34 @@ fn watched_dir(cfg: &config::Config) -> PathBuf { fn main() { #[cfg(feature = "good_log")] - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::builder() - .with_default_directive(tracing::metadata::LevelFilter::INFO.into()) - .from_env_lossy(), - ) - .init(); + env_logger::init(); + #[cfg(feature = "small_log")] smalog::init(); let config_path = std::env::args().nth(1); let cfg = config::Config::parse(config_path).expect("Failed to parse config"); - let c2 = cfg.clone(); + let hndr = handler::Handler::new(&cfg).expect("Failed to configure transport & event handler"); let dir = watched_dir(&cfg); - let mut watcher = notify::recommended_watcher(move |e| handler::handle_event(e, &cfg)) + let mut watcher = notify::recommended_watcher(move |e| hndr.handle_event(e)) .expect("Unable to create directory watcher."); watcher .watch(&dir, notify::RecursiveMode::NonRecursive) .expect("Unable to watch directory."); + let hndr = + handler::Handler::new(&cfg).expect("Failed to configure second transport & event handler"); let mut preexisting = fs::read_dir(&dir).expect("Can't list watched_directory - does it exist?"); - let mut t = 0; + let mut t = 2; while dir.is_dir() { - t += 1; std::thread::sleep(Duration::from_secs(t)); if let Some(Ok(f)) = preexisting.next() { if f.metadata().map(|d| d.is_file()).unwrap_or(false) { - net::send(&f.path(), &c2); + hndr.send(&f.path()); } + } else if let Ok(rd) = fs::read_dir(&dir) { + preexisting = rd; + t *= 2; } } } diff --git a/watcher/src/net.rs b/watcher/src/net.rs deleted file mode 100644 index a7b4d1f..0000000 --- a/watcher/src/net.rs +++ /dev/null @@ -1,6 +0,0 @@ -use config::Config; -use std::path::Path; - -pub(crate) fn send(p: &Path, _cfg: &Config) { - println!("TODO {:?}", &p); -}