Move features

src/engine.rs

@@ -3,3 +3,9 @@ pub mod port_buffer;
 pub mod regressor;
 pub mod graph;
 pub mod optimizer;
+pub mod hogwild;
+pub mod persistence;
+pub mod quantization;
+pub mod buffer_handler;
+pub mod multithread_helpers;
+pub mod serving;

src/engine/block/ffm.rs

@@ -12,16 +12,16 @@ use merand48::*;
 
 use optimizer::OptimizerTrait;
 
-use crate::engine::block::iterators;
+use crate::engine::block::{file, iterators};
 use crate::engine::block::iterators::OptimizerData;
-use crate::feature_buffer;
-use crate::feature_buffer::{FeatureBuffer, HashAndValueAndSeq};
+use crate::namespace::feature_buffer;
+use crate::namespace::feature_buffer::{FeatureBuffer, HashAndValueAndSeq};
 use crate::engine::graph;
 use crate::model_instance;
 use crate::engine::optimizer;
 use crate::engine::port_buffer;
 use crate::engine::port_buffer::PortBuffer;
-use crate::quantization;
+use crate::engine::quantization;
 use crate::engine::regressor;
 use crate::engine::regressor::{BlockCache, BlockTrait, FFM_CONTRA_BUF_LEN};
 
@@ -838,11 +838,11 @@ impl<L: OptimizerTrait + 'static> BlockTrait for BlockFFM<L> {
     ) -> Result<(), Box<dyn Error>> {
         if use_quantization {
             let quantized_weights = quantization::quantize_ffm_weights(&self.weights);
-            iterators::write_weights_to_buf(&quantized_weights, output_bufwriter, false)?;
+            file::write_weights_to_buf(&quantized_weights, output_bufwriter, false)?;
         } else {
-            iterators::write_weights_to_buf(&self.weights, output_bufwriter, false)?;
+            file::write_weights_to_buf(&self.weights, output_bufwriter, false)?;
         }
-        iterators::write_weights_to_buf(&self.optimizer, output_bufwriter, false)?;
+        file::write_weights_to_buf(&self.optimizer, output_bufwriter, false)?;
         Ok(())
     }
 
@@ -854,10 +854,10 @@ impl<L: OptimizerTrait + 'static> BlockTrait for BlockFFM<L> {
         if use_quantization {
             quantization::dequantize_ffm_weights(input_bufreader, &mut self.weights);
         } else {
-            iterators::read_weights_from_buf(&mut self.weights, input_bufreader, false)?;
+            file::read_weights_from_buf(&mut self.weights, input_bufreader, false)?;
         }
 
-        iterators::read_weights_from_buf(&mut self.optimizer, input_bufreader, false)?;
+        file::read_weights_from_buf(&mut self.optimizer, input_bufreader, false)?;
         Ok(())
     }
 
@@ -889,9 +889,9 @@ impl<L: OptimizerTrait + 'static> BlockTrait for BlockFFM<L> {
         if use_quantization {
             quantization::dequantize_ffm_weights(input_bufreader, &mut forward.weights);
         } else {
-            iterators::read_weights_from_buf(&mut forward.weights, input_bufreader, false)?;
+            file::read_weights_from_buf(&mut forward.weights, input_bufreader, false)?;
         }
-        iterators::skip_weights_from_buf::<OptimizerData<L>>(
+        file::skip_weights_from_buf::<OptimizerData<L>>(
             self.ffm_weights_len as usize,
             input_bufreader,
         )?;
@@ -1205,7 +1205,7 @@ mod tests {
     use crate::assert_epsilon;
     use crate::engine::block::test::{slearn2, spredict2, spredict2_with_cache, ssetup_cache2};
     use crate::engine::block::loss_functions;
-    use crate::feature_buffer::HashAndValueAndSeq;
+    use crate::namespace::feature_buffer::HashAndValueAndSeq;
     use crate::engine::graph::BlockGraph;
     use crate::model_instance::Optimizer;
 

src/engine/block/iterators.rs

@@ -1,6 +1,6 @@
 use crate::engine::optimizer::OptimizerTrait;
 
-use crate::feature_buffer;
+use crate::namespace::feature_buffer;
 use crate::engine::port_buffer;
 use crate::engine::regressor::{BlockCache, BlockTrait};
 

src/engine/block/loss_functions.rs

@@ -2,8 +2,8 @@ use std::any::Any;
 use std::error::Error;
 
 use crate::engine::block::iterators;
-use crate::feature_buffer;
-use crate::feature_buffer::FeatureBuffer;
+use crate::namespace::feature_buffer;
+use crate::namespace::feature_buffer::FeatureBuffer;
 use crate::engine::graph;
 use crate::engine::port_buffer;
 use crate::engine::port_buffer::PortBuffer;

src/engine/block/lr.rs

@@ -4,7 +4,7 @@ use crate::engine::graph;
 use crate::model_instance;
 use crate::engine::optimizer;
 use crate::engine::regressor;
-use crate::{feature_buffer, parser};
+use crate::namespace::{feature_buffer, parser};
 
 use std::error::Error;
 use std::io;

src/engine/block/misc.rs

@@ -3,7 +3,7 @@ use std::error::Error;
 
 use crate::engine::block::iterators;
 use crate::engine::graph;
-use crate::feature_buffer::FeatureBuffer;
+use crate::namespace::feature_buffer::FeatureBuffer;
 use crate::engine::port_buffer::PortBuffer;
 use crate::engine::regressor::BlockCache;
 use crate::engine::regressor::BlockTrait;

src/engine/block/neural.rs

@@ -17,7 +17,7 @@ use iterators::OptimizerData;
 use optimizer::OptimizerTrait;
 use crate::engine::regressor::BlockTrait;
 
-use crate::feature_buffer::FeatureBuffer;
+use crate::namespace::feature_buffer::FeatureBuffer;
 use crate::engine::port_buffer::PortBuffer;
 use blas::*;
 

src/engine/block/normalize.rs

@@ -2,7 +2,7 @@ use std::any::Any;
 use std::error::Error;
 
 use crate::engine::block::iterators;
-use crate::feature_buffer::FeatureBuffer;
+use crate::namespace::feature_buffer::FeatureBuffer;
 use crate::engine::graph;
 use crate::model_instance;
 use crate::engine::port_buffer::PortBuffer;

src/engine/block/relu.rs

@@ -2,7 +2,7 @@ use std::any::Any;
 use std::error::Error;
 
 use crate::engine::block::iterators;
-use crate::feature_buffer::FeatureBuffer;
+use crate::namespace::feature_buffer::FeatureBuffer;
 use crate::engine::graph::{BlockGraph, BlockPtrOutput, InputSlot, OutputSlot};
 use crate::model_instance;
 use crate::engine::port_buffer::PortBuffer;

src/engine/block/test.rs

@@ -1,8 +1,5 @@
 #[cfg(test)]
-use crate::{feature_buffer, engine:: {graph, port_buffer}};
-
-#[cfg(test)]
-use crate::engine::regressor::BlockCache;
+use crate::{namespace::feature_buffer, engine:: {graph, port_buffer, regressor::BlockCache}};
 
 #[cfg(test)]
 #[macro_export]

src/hogwild.rs → src/engine/hogwild.rs

@@ -3,9 +3,9 @@ use std::sync::{mpsc, Arc, Mutex};
 use std::thread;
 use std::thread::JoinHandle;
 
-use crate::feature_buffer::FeatureBufferTranslator;
+use crate::namespace::feature_buffer::FeatureBufferTranslator;
 use crate::model_instance::ModelInstance;
-use crate::multithread_helpers::BoxedRegressorTrait;
+use crate::engine::multithread_helpers::BoxedRegressorTrait;
 use crate::engine::port_buffer::PortBuffer;
 
 static CHANNEL_CAPACITY: usize = 100_000;

src/persistence.rs → src/engine/persistence.rs

@@ -9,9 +9,9 @@ use std::io::Read;
 
 use crate::model_instance;
 use crate::engine::regressor;
-use crate::vwmap;
+use crate::namespace::vwmap;
 
-use crate::multithread_helpers::BoxedRegressorTrait;
+use crate::engine::multithread_helpers::BoxedRegressorTrait;
 use crate::engine::regressor::Regressor;
 
 const REGRESSOR_HEADER_MAGIC_STRING: &[u8; 4] = b"FWRE"; // Fwumious Wabbit REgressor
@@ -207,8 +207,8 @@ mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use super::*;
     use crate::assert_epsilon;
-    use crate::feature_buffer;
-    use crate::feature_buffer::{HashAndValue, HashAndValueAndSeq};
+    use crate::namespace::feature_buffer;
+    use crate::namespace::feature_buffer::{HashAndValue, HashAndValueAndSeq};
     use crate::model_instance::Optimizer;
     use crate::engine::optimizer;
     use crate::engine::optimizer::OptimizerTrait;

src/engine/regressor.rs

@@ -14,8 +14,9 @@ use crate::engine::block::neural;
 use crate::engine::block::neural::InitType;
 use crate::engine::block::normalize;
 use crate::engine::block::relu;
-use crate::{engine::block, feature_buffer};
-use crate::feature_buffer::HashAndValueAndSeq;
+use crate::engine::block;
+use crate::namespace::feature_buffer;
+use crate::namespace::feature_buffer::HashAndValueAndSeq;
 use crate::engine::graph;
 use crate::model_instance;
 use crate::engine::port_buffer;
@@ -538,7 +539,7 @@ impl Regressor {
 mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use super::*;
-    use crate::feature_buffer::HashAndValue;
+    use crate::namespace::feature_buffer::HashAndValue;
     use crate::engine::optimizer;
 
     /* LR TESTS */

src/serving.rs → src/engine/serving.rs

@@ -9,14 +9,14 @@ use std::sync::Arc;
 use std::sync::Mutex;
 use std::thread;
 
-use crate::feature_buffer;
+use crate::namespace::feature_buffer;
 use crate::model_instance;
-use crate::multithread_helpers::BoxedRegressorTrait;
-use crate::parser;
+use crate::engine::multithread_helpers::BoxedRegressorTrait;
+use crate::namespace::parser;
 use crate::persistence;
 use crate::engine::port_buffer;
 use crate::engine::regressor;
-use crate::vwmap;
+use crate::namespace::vwmap;
 
 pub struct Serving {
     listening_interface: String,
@@ -257,7 +257,7 @@ impl Serving {
 mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use super::*;
-    use crate::feature_buffer;
+    use crate::namespace::feature_buffer;
     use crate::engine::regressor;
     use mockstream::{FailingMockStream, SharedMockStream};
     use std::io::ErrorKind;

src/lib.rs

@@ -1,34 +1,24 @@
 pub mod engine;
-pub mod buffer_handler;
-pub mod cache;
 pub mod cmdline;
-pub mod feature_buffer;
-pub mod hogwild;
 pub mod logging;
 pub mod model_instance;
-pub mod multithread_helpers;
-pub mod parser;
-pub mod persistence;
-pub mod quantization;
-pub mod radix_tree;
-pub mod serving;
-pub mod vwmap;
-pub mod feature;
+pub mod namespace;
 
 extern crate blas;
 extern crate half;
 extern crate intel_mkl_src;
 
-use crate::feature_buffer::FeatureBufferTranslator;
-use crate::multithread_helpers::BoxedRegressorTrait;
-use crate::parser::VowpalParser;
+use crate::namespace::feature_buffer::FeatureBufferTranslator;
+use crate::engine::multithread_helpers::BoxedRegressorTrait;
+use crate::namespace::parser::VowpalParser;
 use crate::engine::port_buffer::PortBuffer;
 use crate::engine::regressor::BlockCache;
-use crate::vwmap::NamespaceType;
+use crate::namespace::vwmap::NamespaceType;
 use shellwords;
 use std::ffi::CStr;
 use std::io::Cursor;
 use std::os::raw::c_char;
+use engine::persistence;
 
 const EOF_ERROR_CODE: f32 = -1.0;
 const EXCEPTION_ERROR_CODE: f32 = -1.0;

src/main.rs

@@ -23,21 +23,21 @@ extern crate intel_mkl_src;
 extern crate nom;
 extern crate core;
 
-use fw::cache::RecordCache;
-use fw::feature_buffer::FeatureBufferTranslator;
-use fw::hogwild::HogwildTrainer;
+use fw::namespace::cache::RecordCache;
+use fw::namespace::feature_buffer::FeatureBufferTranslator;
+use fw::engine::hogwild::HogwildTrainer;
 use fw::model_instance::{ModelInstance, Optimizer};
-use fw::multithread_helpers::BoxedRegressorTrait;
-use fw::parser::VowpalParser;
-use fw::buffer_handler::create_buffered_input;
+use fw::engine::multithread_helpers::BoxedRegressorTrait;
+use fw::namespace::parser::VowpalParser;
+use fw::engine::buffer_handler::create_buffered_input;
 use fw::persistence::{
     new_regressor_from_filename, save_regressor_to_filename, save_sharable_regressor_to_filename,
 };
 use fw::engine::regressor;
 use fw::engine::regressor::{get_regressor_with_weights, Regressor};
-use fw::serving::Serving;
-use fw::vwmap::VwNamespaceMap;
-use fw::{cmdline, feature_buffer, logging};
+use fw::engine::serving::Serving;
+use fw::namespace::vwmap::VwNamespaceMap;
+use fw::{cmdline, logging};
 
 fn main() {
     logging::initialize_logging();

src/model_instance.rs

@@ -4,10 +4,10 @@ use std::io::ErrorKind;
 
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use crate::feature;
 
-use crate::feature::parser;
-use crate::vwmap::{NamespaceDescriptor, VwNamespaceMap};
+use crate::namespace::feature;
+use crate::namespace::feature::parser;
+use crate::namespace::vwmap::{NamespaceDescriptor, VwNamespaceMap};
 
 const WEIGHT_DELIM: &str = ":";
 const VERBOSE_FIELD_DELIM: &str = ",";
@@ -555,7 +555,7 @@ impl ModelInstance {
 mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use super::*;
-    use crate::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap};
+    use crate::namespace::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap};
 
     fn ns_desc(i: u16) -> NamespaceDescriptor {
         NamespaceDescriptor {

src/namespace.rs

@@ -0,0 +1,6 @@
+pub mod feature;
+pub mod parser;
+pub mod radix_tree;
+pub mod vwmap;
+pub mod feature_buffer;
+pub mod cache;

src/cache.rs → src/namespace/cache.rs

@@ -7,7 +7,7 @@ use std::io::Write;
 use std::path;
 use std::{mem, slice};
 
-use crate::vwmap;
+use crate::namespace::vwmap;
 
 const CACHE_HEADER_MAGIC_STRING: &[u8; 4] = b"FWCA"; // Fwumious Wabbit CAche
 const CACHE_HEADER_VERSION: u32 = 11;

src/feature/executors.rs → src/namespace/feature/executors.rs

@@ -1,4 +1,4 @@
-use crate::{feature, vwmap};
+use crate::namespace::{feature, vwmap};
 use std::error::Error;
 use std::io::Error as IOError;
 use std::io::ErrorKind;
@@ -8,10 +8,10 @@ use std::cell::RefCell;
 use dyn_clone::{clone_trait_object, DynClone};
 use fasthash::murmur3;
 
-use crate::feature::transformers::{
+use crate::namespace::feature::transformers::{
     TransformerBinner, TransformerCombine, TransformerLogRatioBinner, TransformerWeight,
 };
-use crate::feature::{parser, transformers};
+use crate::namespace::feature::{parser, transformers};
 
 pub fn default_seeds(to_namespace_index: u32) -> [u32; 5] {
     let to_namespace_index = to_namespace_index ^ 1u32 << 31; // compatibility with earlier version
@@ -244,8 +244,8 @@ clone_trait_object!(FunctionExecutorTrait);
 mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use super::*;
-    use crate::feature::executors::default_seeds;
-    use crate::parser;
+    use crate::namespace::feature::executors::default_seeds;
+    use crate::namespace::parser;
 
     fn ns_desc(i: u16) -> vwmap::NamespaceDescriptor {
         vwmap::NamespaceDescriptor {