diff --git a/slides/information-theory/figure/kl_calculation_plot.png b/slides/information-theory/figure/kl_calculation_plot.png new file mode 100644 index 00000000..5f3b99bc Binary files /dev/null and b/slides/information-theory/figure/kl_calculation_plot.png differ diff --git a/slides/information-theory/figure/normal_entropy.png b/slides/information-theory/figure/normal_entropy.png index 2c3e608c..0ff12fe2 100644 Binary files a/slides/information-theory/figure/normal_entropy.png and b/slides/information-theory/figure/normal_entropy.png differ diff --git a/slides/information-theory/figure/uni_entropy.png b/slides/information-theory/figure/uni_entropy.png index 4f828077..3694263b 100644 Binary files a/slides/information-theory/figure/uni_entropy.png and b/slides/information-theory/figure/uni_entropy.png differ diff --git a/slides/information-theory/rsrc/make_diff_entropy_plots.R b/slides/information-theory/rsrc/make_diff_entropy_plots.R index 49ee51d0..326368b1 100644 --- a/slides/information-theory/rsrc/make_diff_entropy_plots.R +++ b/slides/information-theory/rsrc/make_diff_entropy_plots.R @@ -69,15 +69,17 @@ uni_plot = function(a, b){ geom_segment(aes(x = b, y = 0, xend = b, yend = uniform_density), color = "blue", size = 1) + geom_segment(aes(x = a, y = 0, - xend = a, yend = uniform_density), color = "blue", size = 1) + xend = a, yend = uniform_density), color = "blue", size = 1) + + xlim(0, 1.5) + + ylim(0, 1) return(p) } -plot1 = uni_plot(0, 1) -plot2 = uni_plot(2,8) +plot1 = uni_plot(0, 1.5) +plot2 = uni_plot(0, 1) p = grid.arrange(plot1, plot2, ncol = 2) -ggsave("..figure/uni_entropy.png", plot = p, width = 8, height = 3) +ggsave("..figure/uni_entropy.png", plot = p, width = 7, height = 3) ########## CREATE NORMAL DISTRIBUTIONS @@ -97,12 +99,14 @@ normal_plot = function(mu,sigma){ p = ggplot(data, aes(x = x)) + geom_line(aes(y = NormalDensity), color = "blue", size = 1) + labs(title = paste("Differential entropy:", entropy_normal), x = "x", - y = sprintf("N(%d,%s)", mu, sigma)) + y = sprintf("N(%d,%s)", mu, sigma)) + + xlim(-4, 4) + + ylim(0, 0.4) return(p) } plot1 = normal_plot(0, 1) -plot2 = normal_plot(2,8) +plot2 = normal_plot(0, 1.5) p = grid.arrange(plot1, plot2, ncol = 2) ggsave("..figure/normal_entropy.png", plot = p, width = 8, height = 3) diff --git a/slides/information-theory/rsrc/make_kl_calculation_plot.R b/slides/information-theory/rsrc/make_kl_calculation_plot.R new file mode 100644 index 00000000..85aaed58 --- /dev/null +++ b/slides/information-theory/rsrc/make_kl_calculation_plot.R @@ -0,0 +1,60 @@ +library(ggplot2) +library(gridExtra) +library(extraDistr) + +####### PLOT KL DIVERGENCE FOR NORMAL AND LAPLACE DISTRIBUTION + +set.seed(123) + +x <- seq(-4, 4, length.out = 1000) + +norm_density <- dnorm(x, 0, 1) +lp_density <- dlaplace(x, 0, 1.5) +ratio_norm_lp <- norm_density/lp_density +log_ratio <- log(ratio_norm_lp) +dens_ratio <- norm_density*log_ratio +data <- data.frame(x = x, NormalDensity = norm_density, + LaPlaceDensity = lp_density, + Ratio_Density = ratio_norm_lp, + LogRatio = log_ratio, + DensityRatio = dens_ratio) + +integrand <- function(x) { + + n_density <- dnorm(x, 0, 1) + l_density <- dlaplace(x, 0, 1.5) + log_ratio <- log(n_density/l_density) + n_density*log_ratio +} + +result <- integrate(integrand, lower = 0, upper = 1) +kl <- round(result$value, 2) + +plot1 = ggplot(data, aes(x = x)) + + geom_line(aes(y = NormalDensity), color = "blue", size = 1, linetype = "solid") + + geom_line(aes(y = LaPlaceDensity), color = "red", size = 1, linetype = "solid") + + labs(title = "N(0,1) and LP(0,1.5) Densities", x = "x", y = "Density") + + scale_color_manual(values = c("blue")) + +plot2 = ggplot(data, aes(x = x)) + + geom_line(aes(y = Ratio_Density), color = "darkgreen", size = 1, linetype = "solid") + + labs(title = "Ratio of Densities", x = "x", y = "p(x)/q(x)") + + scale_color_manual(values = c("red")) + +plot3 = ggplot(data, aes(x = x)) + + geom_line(aes(y = LogRatio), color = "purple", size = 1, linetype = "solid") + + labs(title = "Log-Ratio of Densities", x = "x", y = "log(p(x)/q(x))") + + scale_color_manual(values = c("red")) + +plot4 = ggplot(data, aes(x = x)) + + geom_line(aes(y = DensityRatio), color = "orange", size = 1, linetype = "solid") + + labs(title = "Integrand", x = "x", y = "p(x)*log(p(x)/q(x))") + + geom_ribbon(aes(ymax = DensityRatio, ymin = 0), fill = "grey", alpha = 0.5) + + geom_text(aes(x = 2.5, y = 0.1, label = paste("D_KL =",kl)), color = "black", size = 3) + + scale_color_manual(values = c("orange")) + +p = grid.arrange(plot1, plot2, plot3, plot4, ncol = 2) +ggsave("..figure/kl_calculation_plot.png", plot = p, width =8, height = 5) + + + diff --git a/slides/information-theory/slides-info-kl.tex b/slides/information-theory/slides-info-kl.tex index 45cec42f..7ce041b1 100644 --- a/slides/information-theory/slides-info-kl.tex +++ b/slides/information-theory/slides-info-kl.tex @@ -48,6 +48,18 @@ \end{vbframe} +\begin{vbframe} {KL-Divergence Example} + +Consider the KL-Divergence between two continuous distributions with $p(X)=N(0,1)$ and $q(X)=LP(0, 1.5)$ given by + + $$ D_{KL}(p \| q) = \int_{x \in \Xspace} p(x) \cdot \log \frac{p(x)}{q(x)}. $$ + +\begin{figure} +\includegraphics[width = 8cm ]{figure/kl_calculation_plot.png} +\end{figure} + +\end{vbframe} + \begin{vbframe} {Information Inequality} $ D_{KL}(p \| q) \geq 0$ holds always true for any pair of distributions, and holds with equality if and only if $p=q$.