commit after merge

.github/workflows/DocCleanup.yml

@@ -0,0 +1,26 @@
+name: Doc Preview Cleanup
+
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  doc-preview-cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+      - name: Delete preview and history + push changes
+        run: |
+            if [ -d "previews/PR$PRNUM" ]; then
+              git config user.name "Documenter.jl"
+              git config user.email "documenter@juliadocs.github.io"
+              git rm -rf "previews/PR$PRNUM"
+              git commit -m "delete preview"
+              git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+              git push --force origin gh-pages-new:gh-pages
+            fi
+        env:
+            PRNUM: ${{ github.event.number }}

.github/workflows/Documentation.yaml

@@ -0,0 +1,27 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - main # update to match your development branch (master, main, dev, trunk, ...)
+      - dev
+    tags: '*'
+  pull_request:
+
+jobs:
+  build:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1.9'
+      - name: Install dependencies
+        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(;verbose=true)'
+      - name: Build and deploy
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key
+        run: julia --project=docs/ docs/make.jl

.github/workflows/TagBot.yml

@@ -0,0 +1,15 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

.gitignore

@@ -9,4 +9,6 @@ Manifest.toml
 *.png
 
 /docs/build/
-/docs/site/
+/docs/site/
+
+*.ipynb

README.md

@@ -0,0 +1,6 @@
+QuantizedNetworks.jl
+---
+
+[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://ctuavastlab.github.io/QuantizedNetworks.jl/stable)
+[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://ctuavastlab.github.io/QuantizedNetworks.jl/dev)
+

docs/make.jl

@@ -1,5 +1,3 @@
-push!(LOAD_PATH,"../src/")
-
 using Documenter
 using QuantizedNetworks
 
@@ -14,23 +12,37 @@ DocMeta.setdocmeta!(
 # content
 api = joinpath.("./api/", [
     "utilities.md",
+    "estimators.md",
     "quantizers.md",
     "layers.md",
+    "blocks.md",
+    "l0gate.md",
+    "optimizers.md"
+])
+
+examples = joinpath.("./examples/", [
+    "mnist.md",
+    "flower.md"
 ])
 
 makedocs(
     sitename = "QuantizedNetworks",
-    format = Documenter.HTML(),
+    format=Documenter.HTML(;
+        prettyurls=get(ENV, "CI", "false") == "true",
+        collapselevel=1,
+        ansicolor=true
+    ),
     modules = [QuantizedNetworks],
     pages = [
         "Home" => "index.md",
+        "Examples" => examples,
         "Api" => api,
-    ]
+    ],
+    doctest = true,  # Disable doctests
+    clean = true,
+
 )
 
-# Documenter can also automatically deploy documentation to gh-pages.
-# See "Hosting Documentation" and deploydocs() in the Documenter manual
-# for more information.
-#=deploydocs(
-    repo = "<repository url>"
-)=#
+deploydocs(;
+    repo="github.com/CTUAvastLab/QuantizedNetworks.jl.git"
+)

docs/src/api/blocks.md

@@ -0,0 +1,54 @@
+# Blocks
+
+## DenseBlock
+```@docs
+DenseBlock
+```
+### Standard Dense Layer
+```@docs
+QuantizedNetworks.Flux.Dense(l::DenseBlock)
+```
+
+### Examples
+```jldoctest
+julia> using Random, QuantizedNetworks; Random.seed!(3);
+
+julia> db = DenseBlock(2=>2)
+DenseBlock(Chain(QuantizedDense(2 => 2; bias=false, quantizer=Ternary(0.05, STE(2))), BatchNorm(2), Sign(STE(2))))
+
+julia> x = rand(Float32, 2, 4)
+2×4 Matrix{Float32}:
+ 0.940675  0.100403   0.789168  0.582228
+ 0.999979  0.0921143  0.698426  0.496285
+
+julia> db(x)
+2×4 Matrix{Float32}:
+ -1.0   1.0   1.0   1.0
+  1.0  -1.0  -1.0  -1.0
+```
+
+
+## FeatureBlock
+```@docs
+FeatureBlock
+```
+
+### Examples
+```jldoctest
+julia> using Random, QuantizedNetworks; Random.seed!(3);
+
+julia> fb = FeatureBlock(2, 2)
+FeatureBlock(Parallel(vcat, FeatureQuantizer(2 => 4; quantizer=Sign(STE(2)))))
+
+julia> x = rand(Float32, 2, 1)
+2×1 Matrix{Float32}:
+ 0.8521847
+ 0.7965402
+
+julia> fb(x)
+4×1 Matrix{Float32}:
+  1.0
+  1.0
+  1.0
+ -1.0
+```

docs/src/api/estimators.md

@@ -0,0 +1,12 @@
+# Estimators
+```@docs
+QuantizedNetworks.AbstractEstimator
+```
+
+## Straight through estimators
+```@docs
+QuantizedNetworks.STE
+QuantizedNetworks.PolynomialSTE
+QuantizedNetworks.SwishSTE
+QuantizedNetworks.StochasticSTE
+```

docs/src/api/l0gate.md

@@ -0,0 +1,21 @@
+# L0 gate
+
+```@docs
+L0Gate
+```
+
+### Functions
+
+```@docs
+QuantizedNetworks.isactive(c::L0Gate)
+QuantizedNetworks.Flux.testmode!(c::L0Gate, mode=true)
+```
+
+### Helper functions
+```@docs
+QuantizedNetworks._shape(s, ::Colon)
+QuantizedNetworks._shape(s, dims)
+QuantizedNetworks.shift(x::T, lo::Real = -0.1, hi::Real = 1.1) where {T}
+QuantizedNetworks.l0gate_train(x::AbstractArray{T}, logα, β; dims = :) where {T}
+QuantizedNetworks.l0gate_test(::AbstractArray{T}, logα, β) where {T}
+```

docs/src/api/layers.md

@@ -1,8 +1,65 @@
-# Layers and Blocks
+# Layers
 
+## QuantDense
 ```@docs
-QuantizedNetworks.QuantizedDense
-QuantizedNetworks.DenseBlock
-QuantizedNetworks.FeatureQuantizer
-QuantizedNetworks.FeatureBlock
+QuantDense
+```
+### Logic
+```@docs
+QuantizedNetworks.nn2logic(layer::QuantDense)
+```
+
+### Examples
+
+```jldoctest
+julia> using Random, QuantizedNetworks; Random.seed!(3);
+
+julia> x = rand(Float32, 1, 2);
+
+julia> qd = QuantDense(1 => 2)
+QuantDense(1 => 2, identity; weight_lims=(-1.0f0, 1.0f0), bias=false, Ternary(0.05, STE(2)), Sign(STE(2)))
+
+julia> qd(x)
+2×2 Matrix{Float32}:
+ -1.0  -1.0
+  1.0   1.0
+
+julia> d = QuantizedNetworks.nn2logic(qd)
+Dense(1 => 2, Sign(STE(2)))  # 4 parameters
+
+julia> d([3])
+2-element Vector{Float32}:
+ -1.0
+  1.0
+```
+
+## FeatureQuantizer
+```@docs
+FeatureQuantizer
+```
+
+### Forward pass
+```@docs
+QuantizedNetworks._forward_pass(w, b, x)
+```
+
+### Backpropagation
+```@docs
+QuantizedNetworks.ChainRulesCore.rrule(::typeof(QuantizedNetworks._forward_pass), w, b, x)
+```
+
+### Examples
+```jldoctest
+julia> using Random, QuantizedNetworks; Random.seed!(3);
+
+julia> x = Float32.([1 2 ;3 4]);
+
+julia> fq = FeatureQuantizer(2,2);
+
+julia> fq(x)
+4×2 Matrix{Float32}:
+  1.0   1.0
+  1.0  -1.0
+ -1.0  -1.0
+  1.0   1.0
 ```

docs/src/api/optimizers.md

@@ -0,0 +1,14 @@
+# Optimizers
+
+# Bop
+```@docs
+Bop
+QuantizedNetworks.Flux.Optimise.apply!(b::Bop, x, Δ)
+```
+
+# Case optimizer
+```@docs
+CaseOptimizer
+QuantizedNetworks.Flux.Optimise.apply!(o::CaseOptimizer, x, Δ)
+```
+

docs/src/api/quantizers.md

@@ -1,17 +1,23 @@
-# Quantizers and Estimators
-## Estimators
+# Quantizers 
 ```@docs
-QuantizedNetworks.AbstractEstimator
-QuantizedNetworks.STE
-QuantizedNetworks.PolynomialSTE
-QuantizedNetworks.SwishSTE
-QuantizedNetworks.StochasticSTE
+QuantizedNetworks.AbstractQuantizer
+QuantizedNetworks.forward_pass(q::AbstractQuantizer, x)
+QuantizedNetworks.pullback(q::AbstractQuantizer, x)
 ```
 
-## Quantizers
+## Binary
 ```@docs
-QuantizedNetworks.AbstractQuantizer
 QuantizedNetworks.Sign
 QuantizedNetworks.Heaviside
+```
+
+## Ternary
+```@docs
 QuantizedNetworks.Ternary
-```
+```
+
+## References
+
+- [`Binarized Neural Networks: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1`](https://arxiv.org/abs/1602.02830)
+- [`Bi-Real Net: Enhancing the Performance of 1-bit CNNs With Improved Representational Capability and Advanced Training Algorithm`](https://arxiv.org/abs/1808.00278)
+- [`Regularized Binary Network Training`](https://arxiv.org/abs/1812.11800)

docs/src/api/utilities.md

@@ -1,5 +1,6 @@
 # Utilities
 
+## ClippedArray
 ```@docs
 QuantizedNetworks.ClippedArray
 ```

docs/src/examples/flower.md

@@ -0,0 +1 @@
+# Flower Example