\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{bm}
\usepackage{amsthm}
\usepackage[pdftex]{epsfig}
\usepackage[tight]{subfigure}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}
\newcommand{\defeq}[0]{\ensuremath{\stackrel{\triangle}{=}}}
\lecture{Statistical Techniques in Robotics (16-831, F08)}{Lecture \#22 (Nov. 6)}{Lecturer: Drew Bagnell}{Scribe: Bryan Low}{Reproducing Kernel Hilbert Space}

The Reproducing Kernel Hilbert Space (RKHS), denoted by $\mathcal{H}_k$, is the space of functions that can be written as $\sum_i \alpha_i k(x_i,x)$. 
For RKHS, linearity holds, that is, if $f \in \mathcal{H}_k$ and $g \in \mathcal{H}_k$, $\alpha f + \beta g \in \mathcal{H}_k$.

The \emph{inner product} of $f, g \in \mathcal{H}_k$ is defined as
$$\langle f,g \rangle \defeq \sum_i \sum_j \alpha_i \beta_j k(x_i, x_j) = \bm{\alpha}^{\top} \mathbf{K} \bm{\beta}$$
where $f(.) = \sum_i \alpha_i k(x_i,.)$, $g(.) = \sum_j \beta_j k(x_j,.)$, $\bm{\alpha}$ and $\bm{\beta}$ are vectors comprising, respectively, $\alpha_i$ and $\beta_i$ components, and $\mathbf{K}$ is a matrix comprising $k(x_i, x_j)$ components. Further, $|| f ||^2 \defeq \langle f,f \rangle = \bm{\alpha}^{\top} \mathbf{K} \bm{\alpha}$.
Note that $\mathbf{K}$ must satisfy the positive definite (hence, norm is positive) and symmetric (i.e., $k(x_i, x_j) = k(x_j, x_i)$) conditions.

For example, if $f(.) = k(x_i,.)$ and $g(.) = k(x_j,.)$, $\langle f,g \rangle = k(x_i, x_j)$.
Assuming that the kernels are Gaussian, if they are far apart, the inner product tends to $0$.
If they are very close, the inner product tends to $k(x_i,x_i)$ (often assumed to be $1$);
the inner product becomes $-1$ if $g(.) = -k(x_j,.)$.

The loss functional $L[f]$ is defined as
$$L[f] \defeq \sum_i (f(x_i) - y_i)^2 + \frac{\lambda}{2}||f||^2_k \ .$$
If the update rule is of the form $f \leftarrow f - \alpha \nabla{L}$, we need to find the expression for $\nabla{L}$. 

A functional $F: f \rightarrow \mathbb{R}$ accepts a function $f$ as input and returns a real number as output. Using local linearization, we have
$$F[f+\varepsilon g] = F[f] + \varepsilon \langle \nabla{F}, g \rangle + \mathcal{O}(\varepsilon^2) \ .$$

For example, if $F[f] = ||f||^2$, $\nabla{F} = 2f$.

\emph{Proof.}
\begin{equation*}
\begin{array}{rl}
F[f+\varepsilon g] =& \langle f+\varepsilon g, f+\varepsilon g \rangle\\
=& \langle f,f \rangle + 2  \langle f,\varepsilon g\rangle + \varepsilon^2 \langle g,g \rangle\\
=& ||f||^2 + \varepsilon \langle 2f,g\rangle + \varepsilon^2 ||g||^2 \ \ \ \ \ _\Box
\end{array}
\end{equation*}

Supposing $L[f] = ||f||^2$ (i.e., $\nabla{L} = 2f$), the update rule becomes
$$ f \leftarrow f - 2\alpha f = (1 - 2\alpha)f \ ,$$
which shrinks the function $f$.

The reproducing property of RKHS can be observed from
\begin{equation*}
\begin{array}{rl}
F_{x_i}[f] \defeq& f(x_i)\\
=& \sum_j \alpha_j k(x_j,x_i) \\
=& \langle f, k(x_i, .) \rangle \ .

\end{array}
\end{equation*}
%{Multi-Module Systems and Deep Modular Systems}

%Why make deep modular systems?
%Build in prior information/domain knowledge/structure.

%Yann LeCunn

%How do we improve performance of these deep machines?
%backpropagation.

%recursively compute derivatives of energy with respect to inputs and weights.

%very deep systems, gradient gets exponentially smaller, i.e., diffused out.


\end{document}
