pre-meeting OL updates on advriskmin

latex-math/basic-ml.tex

@@ -154,12 +154,14 @@
 \newcommand{\Lxyt}{L\left(y, \fxt\right)} % loss with f parameterized
 \newcommand{\Lxyit}{L\left(\yi, \fxit\right)} % loss of observation with f parameterized
 \newcommand{\Lxym}{L\left(\yi, f\left(\bm{\tilde{x}}^{(i)} ~|~ \thetab\right)\right)} % loss of observation with f parameterized
+\newcommand{\Lpiy}{L\left(y, \pi\right)} % loss in classification
 \newcommand{\Lpixy}{L\left(y, \pix\right)} % loss in classification
 \newcommand{\Lpiv}{L\left(y, \piv\right)} % loss in classification
 \newcommand{\Lpixyi}{L\left(\yi, \pixii\right)} % loss of observation in classification
 \newcommand{\Lpixyt}{L\left(y, \pixt\right)} % loss with pi parameterized
 \newcommand{\Lpixyit}{L\left(\yi, \pixit\right)} % loss of observation with pi parameterized
 \newcommand{\Lhxy}{L\left(y, \hx\right)} % L(y, h(x)), loss function on discrete classes
+\newcommand{\Lhy}{L\left(y, h\right)} % L(y, h), loss function on discrete classes
 \newcommand{\Lr}{L\left(r\right)} % L(r), loss defined on residual (reg) / margin (classif)
 \newcommand{\lone}{|y - \fx|} % L1 loss
 \newcommand{\ltwo}{\left(y - \fx\right)^2} % L2 loss

slides/advriskmin/chapter-order.tex

@@ -4,11 +4,11 @@
 % slides-advriskmin-regression-l2
 % slides-advriskmin-regression-l1
 % slides-advriskmin-regression-further-losses
-% slides-advriskmin-cassification-01
-% slides-advriskmin-cassification-bernoulli
+% slides-advriskmin-classification-01
+% slides-advriskmin-classification-bernoulli
 % slides-advriskmin-logreg-deepdive
-% slides-advriskmin-cassification-brier
-% slides-advriskmin-cassification-furtherlosses
+% slides-advriskmin-classification-brier
+% slides-advriskmin-classification-furtherlosses
 % slides-advriskmin-classification-deepdive
 % slides-advriskmin-max-likelihood-l2
 % slides-advriskmin-max-likelihood-other

slides/advriskmin/rsrc/bernoulli.R

@@ -0,0 +1,57 @@
+# ------------------------------------------------------------------------------
+# classification bernoulli
+
+# FIG: Bernoulli loss (1) on margin (2) on probability (3) on margin no quote
+# ------------------------------------------------------------------------------
+
+library(ggplot2)
+theme_set(theme_minimal())
+
+# DATA -------------------------------------------------------------------------
+
+x_1 <- seq(-4, 4, by = 0.01)
+y <- log(1L + exp(-x_1))
+
+bernoulli = function(y, pix){
+  -y * log(pix) - (1 - y) * log(1 - pix)
+}
+
+x_2 <- seq(0, 1, by = 0.001)
+df <- data.frame(x = x_2, y = 1L, pi = bernoulli(1L, x_2))
+df <- rbind(df, data.frame(x = x_2, y = 0L, pi = bernoulli(0L, x_2)))
+df$y <- as.factor(df$y)
+
+# PLOTS ------------------------------------------------------------------------
+
+p_1 <- ggplot(data.frame(x_1, y), aes(x = x_1, y = y)) + 
+  geom_line(size = 1.2) +
+  xlab(expression(yf)) +
+  ylab(expression(L(y, f)))
+
+p_1 <- p_1 + 
+  annotate(
+    "text",
+    x = 2,
+    y = 2,
+    label = expression(L(y, f) == ln(1 + exp(-y * f))),
+    size = 7)
+
+p_1 <- p_1 + theme(text = element_text(size = 20))
+
+ggsave("../figure/bernoulli_margin.png", p_1, height = 4, width = 9)
+
+p_2 <- ggplot(data = df, aes(x = x, y = pi, color = y)) + 
+  geom_line(size = 1.2) + 
+  xlab(expression(pi)) + 
+  ylab(expression(L(y, pi))) + 
+  theme(text = element_text(size = 20)) +
+  scale_color_viridis_d(end = 0.9)
+
+ggsave("../figure/bernoulli_prob.png", p_2, height = 4, width = 6)
+
+p_3 <- ggplot(data.frame(x_1, y), aes(x = x_1, y = y)) + 
+  geom_line(size = 1.6) +
+  xlab(expression(yf)) +
+  ylab(expression(L(y, f)))
+
+ggsave("../figure/bernoulli.png", p_3, height = 2.85, width = 5)

slides/advriskmin/rsrc/make_loss_brier_plot.R → slides/advriskmin/rsrc/brier.R

@@ -1,31 +1,29 @@
-# ------------------------------------------------------------------------------
-# FIG: BRIER SCORE
-# ------------------------------------------------------------------------------
-
-library(ggplot2)
-
-# DATA -------------------------------------------------------------------------
-
-x <- seq(0L, 1L, by = 0.01)
-
-df <- rbind(
-  data.frame(x, y = 1L, pi = (1 - x)^2),
-  data.frame(x, y = 0L, pi = x^2))
-
-df$y <- as.factor(df$y)
-
-# PLOTS ------------------------------------------------------------------------
-
-p <- ggplot2::ggplot(data = df, aes(x = x, y = pi, color = y)) + 
-  geom_line(size = 1.2) + 
-  xlab(expression(pi(x))) + 
-  ylab(expression(L(y, pi(x)))) + 
-  theme_minimal() +
-  theme(text = element_text(size = 20L)) +
-  scale_color_viridis_d(end = 0.9)
-
-ggplot2::ggsave(
-  "../figure/plot_brier.png", 
-  p,
-  height = 4L,
-  width = 10L)
+# ------------------------------------------------------------------------------
+# classification brier
+
+# FIG: Brier score
+# ------------------------------------------------------------------------------
+
+library(ggplot2)
+
+# DATA -------------------------------------------------------------------------
+
+x <- seq(0, 1, by = 0.01)
+
+df <- rbind(
+  data.frame(x, y = 1, pi = (1 - x)^2),
+  data.frame(x, y = 0, pi = x^2))
+
+df$y <- as.factor(df$y)
+
+# PLOTS ------------------------------------------------------------------------
+
+p <- ggplot(data = df, aes(x = x, y = pi, color = y)) + 
+  geom_line(size = 1.2) + 
+  xlab(expression(pi)) + 
+  ylab(expression(L(y, pi))) + 
+  theme_minimal() +
+  theme(text = element_text(size = 20)) +
+  scale_color_viridis_d(end = 0.9)
+
+ggsave("../figure/brier.png", p, height = 4, width = 9)

slides/advriskmin/rsrc/make_loss_exponential_plot.R → slides/advriskmin/rsrc/exponential.R

@@ -1,27 +1,25 @@
-# ------------------------------------------------------------------------------
-# FIG: EXPONENTIAL LOSS
-# ------------------------------------------------------------------------------
-
-library(ggplot2)
-
-# DATA -------------------------------------------------------------------------
-
-x <- seq(-4L, 4L, by = 0.01)
-y <- exp(-x)
-
-# PLOTS ------------------------------------------------------------------------
-
-p <- ggplot2::ggplot(data.frame(x, y), aes(x = x, y = y)) + 
-  geom_line(size = 1.2) +
-  scale_x_continuous(breaks = seq(-4L, 4L)) +
-  xlab(expression(yf(x))) +
-  ylab(expression(L(y, f(x)))) + 
-  theme_minimal() +
-  theme(text = element_text(size = 30L))
-
-ggplot2::ggsave(
-  "../figure/plot_loss_exponential.png", 
-  p, 
-  height = 4L, 
-  width = 12L)
-
+# ------------------------------------------------------------------------------
+# classification further losses
+
+# FIG: exponential loss
+# ------------------------------------------------------------------------------
+
+library(ggplot2)
+
+# DATA -------------------------------------------------------------------------
+
+x <- seq(-4, 4, by = 0.01)
+y <- exp(-x)
+
+# PLOTS ------------------------------------------------------------------------
+
+p <- ggplot(data.frame(x, y), aes(x = x, y = y)) + 
+  geom_line(size = 1.2) +
+  scale_x_continuous(breaks = seq(-4, 4)) +
+  xlab(expression(yf)) +
+  ylab(expression(L(y, f))) + 
+  theme_minimal() +
+  theme(text = element_text(size = 30))
+
+ggsave("../figure/exponential.png", p, height = 4, width = 12)
+

slides/advriskmin/rsrc/make_loss_hinge_plot.R → slides/advriskmin/rsrc/hinge.R

@@ -1,56 +1,57 @@
-# ------------------------------------------------------------------------------
-# FIG: HINGE LOSS
-# ------------------------------------------------------------------------------
-
-library(ggplot2)
-
-# DATA -------------------------------------------------------------------------
-
-x <- seq(-1L, 2L, by = 0.01)
-l_01 <- as.numeric(x < 0L)
-l_hinge <- ifelse(x < 1L, 1L - x, 0L)
-l_hinge_squared <- ifelse(x < 1L, (1L - x)^2, 0L)
-
-df <- tidyr::gather(
-  data.frame(x, l_hinge_squared, l_hinge, l_01), 
-  "loss", 
-  "value", 
-  -x)
-
-# PLOTS ------------------------------------------------------------------------
-
-p_1 <- ggplot2::ggplot(
-  df[df$loss != "l_hinge_squared", ],
-  aes(x = x, y = value, col = loss)) +
-  geom_line(size = 1.2) +
-  scale_color_viridis_d(
-    end = 0.9,
-    name = "Loss",
-    labels = c("0-1", "hinge"),
-    direction = -1L) + 
-  xlab(expression(yf(x))) +
-  ylab(expression(L(y, f(x)))) +
-  theme_minimal() + 
-  theme(text = element_text(size = 30L))
-
-ggplot2::ggsave("../figure/plot_loss_hinge.png", p_1, height = 4L, width = 12L)
-
-p_2 <- ggplot2::ggplot(
-  df,
-  aes(x = x, y = value, col = loss)) +
-  geom_line(size = 1.2) +
-  scale_color_viridis_d(
-    end = 0.9,
-    name = "Loss",
-    labels = c("0-1", "hinge", "squared hinge"),
-    direction = -1L) + 
-  xlab(expression(r = yf(x))) +
-  ylab(expression(L(y, f(x)))) +
-  theme_minimal() + 
-  theme(text = element_text(size = 30L))
-
-ggplot2::ggsave(
-  "../figure/plot_loss_hinge_squared.png", 
-  p_2, 
-  height = 6L, 
-  width = 12L)
+# ------------------------------------------------------------------------------
+# classification further losses
+
+# FIG: 
+#   (1) hinge loss & 0-1 loss
+#   (2) hinge loss , 0-1 loss & hige-squared loss
+# ------------------------------------------------------------------------------
+
+library(ggplot2)
+library(tidyr)
+
+# DATA -------------------------------------------------------------------------
+
+x <- seq(-1, 2, by = 0.01)
+l_01 <- as.numeric(x < 0)
+l_hinge <- ifelse(x < 1, 1 - x, 0)
+l_hinge_squared <- ifelse(x < 1, (1 - x)^2, 0)
+
+df <- gather(
+  data.frame(x, l_hinge_squared, l_hinge, l_01), 
+  "loss", 
+  "value", 
+  -x)
+
+# PLOTS ------------------------------------------------------------------------
+
+p_1 <- ggplot(
+  df[df$loss != "l_hinge_squared", ],
+  aes(x = x, y = value, col = loss)) +
+  geom_line(size = 1.2) +
+  scale_color_viridis_d(
+    end = 0.9,
+    name = "Loss",
+    labels = c("0-1", "hinge"),
+    direction = -1L) + 
+  xlab(expression(yf)) +
+  ylab(expression(L(y, f))) +
+  theme_minimal() + 
+  theme(text = element_text(size = 30))
+
+ggsave("../figure/hinge.png", p_1, height = 4, width = 12)
+
+p_2 <- ggplot(
+  df,
+  aes(x = x, y = value, col = loss)) +
+  geom_line(size = 1.2) +
+  scale_color_viridis_d(
+    end = 0.9,
+    name = "Loss",
+    labels = c("0-1", "hinge", "squared hinge"),
+    direction = -1L) + 
+  xlab(expression(r = yf)) +
+  ylab(expression(L(y, f))) +
+  theme_minimal() + 
+  theme(text = element_text(size = 30))
+
+ggsave("../figure/hinge_squared.png", p_2, height = 6, width = 12)

slides/advriskmin/rsrc/logistic.R

@@ -0,0 +1,54 @@
+# ------------------------------------------------------------------------------
+# classification bernoulli
+
+# FIG: logistic function and its inverse logit function
+# ------------------------------------------------------------------------------
+
+library(ggplot2)
+theme_set(theme_minimal())
+
+# DATA -------------------------------------------------------------------------
+
+logistic = function(f){
+  1 / (1 + exp(-f))
+}
+
+logit = function(p){
+  log(p / (1 - p))
+}
+
+f <- seq(-4, 4, by = 0.01)
+y_log <- logistic(f)
+
+pix <- seq(0, 1, by = 0.001)
+y_logit <- logit(pix)
+
+df_1 <- data.frame(x = f, y = y_log)
+df_2 <- data.frame(x = pix, y = y_logit)
+
+# PLOT -------------------------------------------------------------------------
+
+p_1 <- ggplot(df_1, aes(x = x, y = y)) + 
+  geom_line(size = 1.2) +
+  xlab(expression(f)) +
+  ylab(expression(pi))
+
+p_1 <- p_1 + 
+  annotate(
+    "text",
+    x = 2,
+    y = 0.3,
+    label = bquote(pi ~ "=" ~ (1 + exp(-f))^-1),
+    size = 7)
+
+p_1 <- p_1 + theme(text = element_text(size = 20))
+
+ggsave("../figure/logistic.png", p_1, height = 4, width = 6)
+
+p_2 <- ggplot(df_2, aes(x = x, y = y)) + 
+  geom_line(size = 1.2) +
+  xlab(expression(p)) +
+  ylab(expression(f^"*")) +
+  theme(text = element_text(size = 20))
+
+ggsave("../figure/logistic_inverse.png", p_2, height = 6, width = 8)