docs + build extensions with musl

.github/workflows/build-so.yaml

@@ -33,6 +33,10 @@ jobs:
       uses: actions/checkout@v4
       with:
         ref: stable
+    - name: Install MUSL toolchain
+      run: |
+        apt install -y musl-dev musl-tools
+        rustup target add x86_64-unknown-linux-musl
     - name: Build extensions
       run: |
         cd jyafn-ext
@@ -42,7 +46,7 @@ jobs:
       with:
         name: extensions-linux-x64
         if-no-files-found: error
-        path: ~/.jyafn/extensions/*-latest.so
+        path: ~/.jyafn/export/*.so
         retention-days: 1
 
   build-dylib:
@@ -79,10 +83,9 @@ jobs:
       with:
         name: extensions-macos-arm64
         if-no-files-found: error
-        path: ~/.jyafn/extensions/*-latest.dylib
+        path: ~/.jyafn/export/*.dylib
         retention-days: 1
 
-
   release:
     needs:
     - build-so

Makefile

@@ -23,7 +23,7 @@ build-linux-wheels:
 	bash ./utils/build-linux-wheels.sh
 
 build-macos-wheels: 
-	bash ./utils/build-macos-wheels.sh
+	cd jyafn-python && bash ../utils/build-macos-wheels.sh
 
 build-wheels: build-linux-wheels build-macos-wheels
 

jyafn-ext/extensions/dummy/Makefile

@@ -13,16 +13,23 @@ ROOT := $(shell cargo metadata --format-version=1 | jq -r '.workspace_root')
 UNAME := $(shell uname)
 
 ifeq ($(UNAME), Linux)
-	SO_EXT := so
-endif
-ifeq ($(UNAME), Darwin)
-	SO_EXT := dylib
+	BUILD := RUSTFLAGS="-C target-feature=-crt-static" \
+		cargo build --release --target x86_64-unknown-linux-musl
+	EXT := so
+	LIBPATH := $(ROOT)/target/x86_64-unknown-linux-musl/release/lib$(LIBNAME).$(EXT)
+else ifeq ($(UNAME), Darwin)
+	BUILD := cargo build --release
+	EXT := dylib
+	LIBPATH := $(ROOT)/target/release/lib$(LIBNAME).$(EXT)
 endif
 
 build:
-	cargo build --release
+	$(BUILD)
 
 install: build
 	mkdir -p ~/.jyafn/extensions/
-	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
-	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-latest.$(SO_EXT)
+	cp $(LIBPATH) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(EXT)
+
+export:
+	mkdir -p ~/.jyafn/export/
+	cp $(LIBPATH) ~/.jyafn/export/$(NAME)-latest.$(EXT)

jyafn-ext/extensions/dummy/src/lib.rs

@@ -1,4 +1,19 @@
-use jyafn_ext::{Resource, Method};
+//! This crate implements the `dummy` extension for jyafn. This extension is intended for
+//! testing and debugging purposes.
+//! 
+//! The only resource declared by this extension is the `Dummy` resource, with three methods:
+//! ```
+//! // Gets the divison of `x` by the number supplied in the resource creation.
+//! get(x: scalar) -> scalar;
+//! // Always errors.
+//! err(x: scalar) -> scalar;
+//! // Always panics.
+//! // NOTE: the panic is caught by the macros in `jyafn-ext` and transformed into an
+//! // error. Panics can never propagate to jyafn code, ever!
+//! panic(x: scalar) -> scalar;
+//! ```
+
+use jyafn_ext::{Method, Resource};
 
 jyafn_ext::extension! {
     Dummy
@@ -25,21 +40,13 @@ impl Dummy {
 
     jyafn_ext::method!(get);
 
-    fn err(
-        &self,
-        _: jyafn_ext::Input,
-        _: jyafn_ext::OutputBuilder,
-    ) -> Result<(), String> {
+    fn err(&self, _: jyafn_ext::Input, _: jyafn_ext::OutputBuilder) -> Result<(), String> {
         Err("oops! wrooong!!".to_string())
     }
 
     jyafn_ext::method!(err);
 
-    fn panic(
-        &self,
-        _: jyafn_ext::Input,
-        _: jyafn_ext::OutputBuilder,
-    ) -> Result<(), String> {
+    fn panic(&self, _: jyafn_ext::Input, _: jyafn_ext::OutputBuilder) -> Result<(), String> {
         panic!("g-g-g-g-ghost!")
     }
 

jyafn-ext/extensions/lightgbm/Makefile

@@ -13,16 +13,23 @@ ROOT := $(shell cargo metadata --format-version=1 | jq -r '.workspace_root')
 UNAME := $(shell uname)
 
 ifeq ($(UNAME), Linux)
-	SO_EXT := so
-endif
-ifeq ($(UNAME), Darwin)
-	SO_EXT := dylib
+	BUILD := RUSTFLAGS="-C target-feature=-crt-static" \
+		cargo build --release --target x86_64-unknown-linux-musl
+	EXT := so
+	LIBPATH := $(ROOT)/target/x86_64-unknown-linux-musl/release/lib$(LIBNAME).$(EXT)
+else ifeq ($(UNAME), Darwin)
+	BUILD := cargo build --release
+	EXT := dylib
+	LIBPATH := $(ROOT)/target/release/lib$(LIBNAME).$(EXT)
 endif
 
 build:
-	cargo build --release
+	$(BUILD)
 
 install: build
 	mkdir -p ~/.jyafn/extensions/
-	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
-	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-latest.$(SO_EXT)
+	cp $(LIBPATH) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(EXT)
+
+export:
+	mkdir -p ~/.jyafn/export/
+	cp $(LIBPATH) ~/.jyafn/export/$(NAME)-latest.$(EXT)

jyafn-ext/extensions/lightgbm/src/lib.rs

@@ -1,5 +1,18 @@
-use lightgbm3::Booster;
+//! This crate implements the `lightgbm` extension for jyafn. It exposes a minimal API
+//! for evaluating models in runtime.
+//! 
+//! The only resource declared by this extension is the `Lightgbm` resource, with three methods:
+//! ```
+//! // Predicts the probability of each class, given a list of feature values.
+//! predict(x: [scalar; n_features]) -> [scalar; n_classes];
+//! // The number of features in this model.
+//! num_features() -> scalar;
+//! // The number of classes in this model.
+//! num_classes() -> scalar;
+//! ```
+
 use jyafn_ext::{Input, Method, OutputBuilder, Resource};
+use lightgbm3::Booster;
 
 jyafn_ext::extension! {
     Lightgbm
@@ -13,7 +26,6 @@ struct Lightgbm {
 unsafe impl Send for Lightgbm {}
 unsafe impl Sync for Lightgbm {}
 
-
 impl Resource for Lightgbm {
     fn from_bytes(bytes: &[u8]) -> Result<Self, impl ToString> {
         let booster = Booster::from_string(&String::from_utf8_lossy(bytes))?;

jyafn-ext/src/io.rs

@@ -18,28 +18,46 @@ impl<'a> Input<'a> {
         Self(std::slice::from_raw_parts(input as *const u64, n_slots))
     }
 
+    /// Gets the data at index `idx` as an `f64`.
     pub fn get_f64(&self, idx: usize) -> f64 {
         f64::from_ne_bytes(self.0[idx].to_ne_bytes())
     }
 
+    /// Gets the data at index `idx` as an `u64`.
     pub fn get_u64(&self, idx: usize) -> u64 {
         self.0[idx]
     }
 
+    /// Gets the data at index `idx` as an `i64`.
+    pub fn get_i64(&self, idx: usize) -> u64 {
+        self.0[idx]
+    }
+
+    /// Gets the data at index `idx` as a `bool`.
     pub fn get_bool(&self, idx: usize) -> bool {
         self.0[idx] == 1
     }
 
+    /// Represents itself as a slice of `f64`s.
     pub fn as_f64_slice(&self) -> &[f64] {
         self.0
             .as_byte_slice()
             .as_slice_of()
             .expect("f64 and u64 have the same size")
     }
 
+    /// Represents itself as a slice of `u64`s.
     pub fn as_u64_slice(&self) -> &[u64] {
         self.0
     }
+
+    /// Represents itself as a slice of `i64`s.
+    pub fn as_i64_slice(&self) -> &[i64] {
+        self.0
+            .as_byte_slice()
+            .as_slice_of()
+            .expect("i64 and u64 have the same size")
+    }
 }
 
 /// A convenience wrapper over the output data pointer, given the information on its size.
@@ -81,6 +99,11 @@ impl<'a> OutputBuilder<'a> {
         self.position += 1;
     }
 
+    pub fn push_i64(&mut self, val: i64) {
+        self.slice[self.position].write(val as u64);
+        self.position += 1;
+    }
+
     pub fn push_bool(&mut self, val: bool) {
         self.slice[self.position].write(val as u64);
         self.position += 1;
@@ -98,6 +121,12 @@ impl<'a> OutputBuilder<'a> {
         }
     }
 
+    pub fn copy_from_i64(&mut self, src: &[i64]) {
+        for &val in src {
+            self.push_i64(val);
+        }
+    }
+
     pub fn copy_from_bool(&mut self, src: &[bool]) {
         for &val in src {
             self.push_bool(val);

jyafn-ext/src/layout.rs

@@ -1,19 +1,34 @@
 use serde_derive::{Deserialize, Serialize};
 
+/// The `strptime` format for ISO 8601, the standard used in the [`Layout::DateTime`]
+/// variant.
 pub const ISOFORMAT: &str = "%Y-%m-%dT%H:%M:%S%.f";
 
+/// A struct is a kind of layout of _ordered_ key-value pairs. Each value is layed out
+/// sequentially in memory.
 #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
 pub struct Struct(pub Vec<(String, Layout)>);
 
+/// A layout is a how jyafn makes the correspondence of structured data (like, but not
+/// necessarily exactly JSON) and buffers of binary data.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Layout {
+    /// An empty value.
     #[default]
     Unit,
+    /// A floating point number. Jyafn does not support integers directly.
     Scalar,
+    /// A boolean. Can be either true or false. This is represented as u64 1 or 0
+    /// respectively. All other values are invalid.
     Bool,
+    /// A date-time with a given format string. Internally, this is represented as a
+    /// timestamp integer in microseconds.
     DateTime(String),
+    /// An imutable piece of text.
     Symbol,
+    /// An ordered sequence of values, layed out in memory sequentially.
     Struct(Struct),
+    /// A layout repeated a given number of times.
     List(Box<Layout>, usize),
 }
 
@@ -23,6 +38,17 @@ impl From<Struct> for Layout {
     }
 }
 
+/// Builds a [`Layout`] usng the jyafn layout display notation.
+///
+/// # Usage
+///
+/// This declares a struct layout with two fields: `x`, a scalar and `y` a date.
+/// ```
+/// layout!({
+///     x: scalar,
+///     y: datetime "%Y-%m-%d"
+/// })
+/// ```
 #[macro_export]
 macro_rules! layout {
     ({$($key:literal : $ty:tt),*}) => {
@@ -51,6 +77,7 @@ macro_rules! layout {
     }
 }
 
+/// Builds a [`Struct`] layout out of a collection of keys and values.
 #[macro_export]
 macro_rules! r#struct {
     ($($key:tt : $ty:tt),*) => {
@@ -60,12 +87,13 @@ macro_rules! r#struct {
     };
 }
 
+/// Builds a [`Struct`] field, given a key and a value layout.
 #[macro_export]
 macro_rules! struct_field {
     ($key:literal : $ty:tt) => {
         ($key.to_string(), $crate::layout!($ty))
     };
     ($key:ident : $ty:tt) => {
         (stringify!($key).to_string(), $crate::layout!($ty))
-    }
+    };
 }

jyafn-ext/src/lib.rs

@@ -150,6 +150,8 @@ macro_rules! extension {
     };
 }
 
+/// Declares a single resource for this extension, given a type. This writes all the
+/// boilerplate code thar corresponds to the extension side of the API.
 #[macro_export]
 macro_rules! resource {
     ($ty:ty) => {
@@ -314,7 +316,7 @@ macro_rules! method {
     };
 }
 
-/// A convenience macro to get references to methods created with [`make_method`].
+/// A convenience macro to get references to methods created with [`method`].
 #[macro_export]
 macro_rules! get_method_ptr {
     ($safe_interface:ident) => {

jyafn/src/const.rs

@@ -1,10 +1,17 @@
+//! Constant values in the computational graph. Constants need to have a type and a binary
+//! representation as a 64-bit peice of data.
+
 use super::Type;
 
 use std::fmt::Debug;
 
+/// A constant. Constants need to have a type and a binary representation as a 64-bit
+/// peice of data.
 #[typetag::serde(tag = "type")]
 pub trait Const: 'static + Debug + Send {
+    /// The primitive. type of this constant.
     fn annotate(&self) -> Type;
+    /// The binary representation of this constant.
     fn render(&self) -> u64;
 }