\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{bm}
\usepackage{epsfig}
\usepackage[tight]{subfigure}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}
\newcommand{\defeq}[0]{\ensuremath{\stackrel{\triangle}{=}}}
\def\x{\mathbf{x}}
\def\w{\mathbf{w}}
\def\K{\mathbf{K}}
\lecture{Statistical Techniques in Robotics (16-831, F09)}{Lecture \#22 (Nov 5, 2009)}{Lecturer: Drew Bagnell}{Scribe: Tomas Simon$^1$}{Kernel Machines / Functional Gradient Descent}
\let\thefootnote\relax\footnotetext{$^1$Based on the scribe work of Daniel Munoz and Bryan Low}
\section{Motivation}
We have seen how to use online convex programming to learn linear functions by optimizing costs of the following form:
\begin{equation}
	\underbrace{|y_t - \w^T \x_t|}_{loss} + \underbrace{\lambda \w^T\w^2}_{regularization / prior} \nonumber
\end{equation}

We want generalize this to learn over a space of more general functions $f: \mathbb{R}^n \rightarrow \mathbb{R}$. The high-level idea is to learn non-linear models using the same gradient-based approach used to learn linear models.

\begin{equation}
	|y_t - f(\x_t)| + \lambda ||f||^2 \nonumber
\end{equation}

Up till now we have only considered functions of the form $f(\x)=\w^T \x$, but we will now extend this to a more general space of functions, the Reproducing Kernel Hilbert Space.

\section{Reproducing Kernel Hilbert Space}

The Reproducing Kernel Hilbert Space (RKHS), denoted by $\mathcal{H}_k$, is the space of functions $f(\cdot)$ that can be written as $\sum_i \alpha_i k(\x_i,\cdot)$, where $k(\x_i,\x_j)$ satisfies certain properties described below.

To be able to manipulate objects in this space of functions, we will look at some key properties:

The \emph{inner product} of $f, g \in \mathcal{H}_k$ is defined as
$$\langle f,g \rangle \defeq \sum_i \sum_j \alpha_i \beta_j k(\x_i, \x_j) = \bm{\alpha}^{\top} \mathbf{K} \bm{\beta}$$
where $f(\cdot) = \sum_i \alpha_i k(\x_i,\cdot)$, $g(\cdot) = \sum_j \beta_j k(\x_j,\cdot)$, $\bm{\alpha}$ and $\bm{\beta}$ are vectors comprising, respectively, $\alpha_i$ and $\beta_i$ components, and $\mathbf{K}$ is $n$ by $m$ (where $n$ is the number of $\x_i$ in $f$, and $m$ those in $g$) with $K_{ij}=k(\x_i, \x_j)$.

Note that this will satisfy linearity (in both arguments):
\begin{itemize}
	\item
	   $\langle \lambda f, g\rangle = \lambda \langle f,g\rangle$
	\item
	   $\langle f_1+f_2, g \rangle = \langle f_1,g\rangle + \langle f_2,g\rangle$
\end{itemize}

With this inner product, the \emph{norm} will be: $|| f ||^2 = \langle f,f \rangle = \bm{\alpha}^{\top} \mathbf{K} \bm{\alpha}$.

For these definitions to work, we can see that the \emph{kernel} $k(\cdot, \cdot)$ will have to satisfy the following conditions:
\begin{itemize}
	\item
	$\mathbf{K}$ is symmetric ($K_{ij} = K_{ji}$), so $k(\x_i,\x_j)=k(\x_j,\x_i)$
	\item
	$\mathbf{K}$ is positive-definite
		($\forall \mathbf{x} \in \mathbb{R}^n : \mathbf{x} \neq \mathbf{0}, \x^T \mathbf{K} \x > 0$)
\end{itemize}

An example of a valid kernel for $\x \in \mathbb{R}^n$ is the inner product: $k(\x_i,\x_j) = \x_i^T \x_j$. Intuitively, the kernel measures the \textit{correlation} between $\x_i$ and $\x_j$. 

A very commonly used kernel is the RBF or Radial Basis Function kernel, which takes the form $k(\x_i, \x_j)= e^{-\frac{1}{\gamma}||\x_i - \x_j||^2}$.
With this kernel in mind, a function can be considered as a weighted (by $\alpha_i$) composition of bumps (the kernels) centered at the $Q$ locations $\mathbf{x_i}$: \begin{equation}
%  \label{eq:rkhs-function}
	f(\cdot) = \sum_{i=1}^Q \alpha_i K(\x_i, \cdot), \nonumber
\end{equation}
See figure \ref{fig:kernel_update} for an illustration.

Note that as long as you can define a valid kernel $k(\x_i, \x_j)$, the samples $\x$ themselves can in principle be any object, eg. text strings. The kernel function might in this case measure the similarity between samples using some sort of edit distance. Each type of data will require kernel functions that are appropriate for that particular data.

\section{Loss Minimization}
Let us consider again our cost function defined over all functions $f$ in our RKHS (but using the hinge loss this time):
\begin{equation}
	L[f] = max(0, 1- y_t f(\x_t)) + \lambda \langle f, f \rangle
\end{equation}

The purpose of $\langle f, f \rangle$ is to penalize the complexity of the solution $f$. Here it acts like the log of a gaussian prior over functions. Intuitively, the probability can be thought of as being distributed according to $P(f)=\frac{1}{Z} e^{-\frac{1}{2} \langle f,f \rangle}$ (in practice this expression doesn't work because $Z$ becomes infinite).

We want to find the best function $f$ in our RKHS so as to minimize this cost, and we will do this by moving in the direction of the negative gradient: $f - \alpha \nabla L$. To do this, we will first have to be able to express the gradient of a function of functions (ie. a \emph{functional} such as $L[f]$).

%%%%%%%%%%%%
\subsection{Functional gradient}
A gradient can be thought of as:
	\begin{itemize}
		\item
            Vector of partial derivatives
		\item
			Direction of steepest ascent
		\item
			Linear approximation of the function (or functional), ie. $f(x_0 + \epsilon) = f(x_0) + \epsilon \cdot \underbrace{\nabla f(x_0)}_{gradient}  + O(\epsilon^2)$.
	\end{itemize}
We will use the third definition. 
A \textit{functional} $F : f \rightarrow \mathbb{R}$ is a function of functions $f \in \mathcal{H}_K$.
Examples:
	\begin{itemize}
		\item
			$F[f] = ||f||^2$
		\item
			$F[f] = (f(x) - y)^2$
		\item
			$F[f] = \frac{\lambda}{2} ||f||^2 + \sum_i (f(x_i) - y_i)^2$
	\end{itemize}

%%%%%%%%%%%%%

A functional gradient $\nabla F[f]$ is defined implictly
as the linear term of the change in a function due to a small
perturbation $\epsilon$ in its input: $F[f + \epsilon g] = F[f] + \epsilon \langle \nabla F[f], g \rangle + O(\epsilon^2)$
	\begin{itemize}
		\item Example: $\nabla F[f] = \nabla ||f||^2 = 2f$
		\begin{eqnarray*}
			F[f + \epsilon g] & = &\langle f + \epsilon g, f + \epsilon g\rangle \\
			& = & ||f|| + 2 \langle f, \epsilon g \rangle + \epsilon^ 2	||g|| \\
			& = & ||f|| + \epsilon \langle 2f , g \rangle + O(\epsilon^2)
		\end{eqnarray*}
	\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{More functional gradients}
\begin{itemize}
\item
The \emph{evaluation functional} evaluates $f$ at the specified $x$: $F_x[f] = f(x)$
	\begin{itemize}
		\item
		Gradient is $\nabla F_x = K(x, \cdot)$
		\begin{eqnarray*}
			F_x[f + \epsilon g] &=& f(x) + \epsilon g(x) + 0 \\
			& = & f(x) + \epsilon \langle K(x, \cdot) , g \rangle + 0 \\
			& = & F_x[f] + \epsilon \langle \nabla F_x , g \rangle + O(\epsilon^2)
		\end{eqnarray*}
		
		\item It is called a \textit{linear functional} due to the lack of a multiplier on perturbation $\epsilon$.
	\end{itemize}
\item
Consider \textit{differentiable} functions $C : \mathbb{R} \rightarrow \mathbb{R}$ that are functions
of functionals $G$, $C(G[f])$.  Our cost function $L[f]$ from before was such a function, these are precisely the functions that we are interested in minimizing.

\item
The derivative of these functions follows the chain rule: $\nabla C(G[f]) = C'(G[f]) \nabla G[f]$
	\begin{itemize}
		\item
		Example: If $C = (||f||^2)^3$, then $\nabla C = 3(||f||^2)^2(2f)$
	\end{itemize}
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Functional gradient descent}
\begin{itemize}
\item
Consider the regularized least squares loss function $L[f]$
\begin{eqnarray*}
L[f] &=& (f(x_i) - y_i)^2 + \lambda ||f||^2 \\
L[f] &=& (F_{x_i}[f] - y_i)^2 + \lambda ||f||^2 \\
\nabla L[f] &=& 2(f(x_i) - y_i)K(x_i, \cdot) + 2 \lambda f
\end{eqnarray*}

\item
Update rule:
\begin{eqnarray*}
f^{t+1} & \leftarrow & f^t - \eta_t \nabla L \\
& \leftarrow & f^t - \eta_t (2(f^t(x_i) - y_i)K(x_i, \cdot) + 2 \lambda f^t) \\
& \leftarrow & f^t(1 - 2 \eta_t \lambda) - \eta_t (2(f^t(x_i) - y_i)K(x_i, \cdot))
\end{eqnarray*}

\item
Need to perform $O(T)$ work at each time step

\item
Example: Figure \ref{fig:kernel_update} shows an update over 3 points $\{(x_1,+), (x_2,-) , (x_3,+)\}$.
The individual kernels centered at the points are \textbf{independently} drawn
with colored lines.  After 3 updates, the function $f$ looks like the solid black line.
\begin{figure}[ht]
\label{fig:kernel_update}
\centering
\includegraphics[width=0.85\linewidth]{kernel_update}
\caption{Illustration of function after 3 updates}
\end{figure}

\item
\textbf{Representer Theorem} (informally): Given a loss function and regularizer objective
with many data points $\{x_i\}$, the minimizing solution $f^*$ can be represented as
\begin{equation}
f^*(\cdot) = \sum_i \alpha_i K(x_i, \cdot)
\end{equation}

\item
Alternate idea from class: perform gradient descent in the space of $\alpha$ coefficients: $\nabla_{\alpha} L$
	\begin{itemize}
	\item Takes $n^2$ iterations to get same performance ($n$ = number of iterations of functional gradient descent)
	\item Every iteration is $O(T^2)$
	\end{itemize}

\end{itemize}

\section{Kernel SVM}
\begin{itemize}
\item
General loss function: $L[f] = \frac{\lambda}{2}||f||^2 + C_t(F_{x_i}[f])$
\item
General update rule: $f_{t+1} \leftarrow (1 - \lambda \eta_t)f_t - \eta_t C_t'(F_{x_i}[f]) K(x_i, \cdot)$
\item
SVM cost function: $C_t(F_{x_i}) = \max(0, 1-f(x_i)y_i)$
\end{itemize}
\begin{equation}
\nabla C_t =
	\begin{cases}
	0 &,  1 - y_i f(x_i) \leq 0 \\
	(C'(F_{x_i}[f])) (\nabla F_{x_i}[f])  = -y_i K(x_i, \cdot) & ,  \text{otherwise}
	\end{cases}
\end{equation}
\end{document}
