Merge pull request #154 from slds-lmu/entropy_plots

Add plots for entropy re-ordering and maximal entropy for uniform

slides/information-theory/rsrc/make_entropy_plot.R

@@ -8,12 +8,44 @@ library(gridExtra)
 
 # DATA -------------------------------------------------------------------------
 
-p1 <- c(0.2, 0.2, 0.2, 0.2, 0.2)
-p2 <- c(0.1, 0.2, 0.4, 0.2, 0.1)
-p3 <- c(0.4, 0.2, 0.2, 0.1, 0.1)
-p4 <- c(0.1, 0.1, 0.6, 0.1, 0.1)
+p1 <- c(0.0, 0.0, 1, 0, 0)
+p2 <- c(0, 0.1, 0.8, 0.1, 0)
+p3 <- c(0.1, 0.1, 0.6, 0.1, 0.1)
+p4 <- c(0.15, 0.15, 0.4, 0.15, 0.15)
+p5 <- c(0.2, 0.2, 0.2, 0.2, 0.2)
 
-# PLOTS ------------------------------------------------------------------------
+# ENTROPY MAXIMAL FOR UNIFORM DISTRIBUTION PLOTS ------------------------------------------------------------------------
+
+plot_ent <- function(p) {
+  n <- length(p)
+  lp <- -log(p)
+  lp <- ifelse(p == 0, 0, lp)
+  r <- p * lp
+  H <- sum(r)
+  dd <- data.frame(x = 1:n, p = p, lp = lp, r = r)
+  pl <- ggplot(data = dd, aes(x = x)) +
+    geom_bar(aes(y = p), stat = "identity") +
+    ggtitle(sprintf("Entropy H(p) = %.1f", H)
+    )
+  return(pl)
+}
+
+
+plot_list = lapply(list(p1, p2, p3, p4, p5), plot_ent)
+
+for (i in 1:length(plot_list)){
+  ggsave(paste0(".../figure/max_entropy_", i, ".pdf"), plot_list[[i]], width = 6.5, height = 5)
+}
+
+
+# DATA -------------------------------------------------------------------------
+
+p1 <- c(0.4, 0.2, 0.2, 0.1, 0.1)
+p2 <- c(0.2, 0.4, 0.2, 0.1, 0.1)
+p3 <- c(0.1, 0.2, 0.4, 0.2, 0.1)
+p4 <- c(0.1, 0.1, 0.2, 0.4, 0.2)
+
+# ENTROPY INVARIANT TO REORDERING PLOTS ------------------------------------------------------------------------
 
 plot_ent <- function(p) {
   n <- length(p)
@@ -30,4 +62,4 @@ plot_ent <- function(p) {
 }
 
 p <- grid.arrange(grobs = lapply(list(p1, p2, p3, p4), plot_ent), nrow = 2, ncol = 2)
-ggsave(filename = "../figure/entropy_plot.png", plot = p, width = 5, height = 3)
+ggsave(filename = ".../figure/entropy_plot_reordering.png", plot = p, width = 5, height = 3)

slides/information-theory/slides-info-entropy.tex

@@ -77,14 +77,34 @@
 \end{itemize}
 \end{vbframe}
 
+\begin{frame}{Entropy Maximal for Uniform}
 
-\begin{vbframe}{Entropy Examples}
+\begin{center}
+        \begin{figure}
+            \centering
+            \foreach \x in{1,2,3,4,5} {
+            \includegraphics<\x>[width=0.6\linewidth]{figure/max_entropy_\x.pdf}\par
+            }
+        \end{figure}
+\end{center}
+
+\begin{itemize}
+   \item Naive observations: Entropy minimal for peaked distribution and maximal for uniform distribution.
+\end{itemize}
+
+\end{frame}
+
+
+\begin{vbframe}{Entropy Re-Ordering}
 
 \begin{center}
-\includegraphics[width = 8cm ]{figure/entropy_plot.png} \\
+\includegraphics[width = 10cm ]{figure/entropy_plot_reordering.png} \\
 \end{center}
 
-Naive observations: Uniform seems maximal, re-ordering does not matter, and the more peaked, the less entropy.
+\begin{itemize}
+   \item Naive observation: Re-ordering does not matter.
+\end{itemize}
+
 \end{vbframe}
 
 \begin{vbframe}{Entropy of Bernoulli distribution}