\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage[tight]{subfigure}
\usepackage{algorithmic}
\usepackage{algorithm}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

%\renewcommand{\baselinestretch}{1.25}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Document header: change the lecture number, date, scribe name, and
% lecture topic.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{Statistical Techniques in Robotics (16-831, F10)}
{Lecture\#11 (Tuesday September 28)}
{Lecturer: Drew Bagnell}
{Scribe:Siyuan Feng \footnotemark}
{The Project Gradient Method and Regret Bounds}

% Be sure to acknowledge all previous scribes.
\footnotetext{Some content adapted from previous scribes: Dmitry Berenson, Forrest Rogers-Marcovitz}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Document body: Scribe your notes here.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Online gradient descent}
\subsection{Instantaneous regret}
Let $l_i = (w_t^Tf_i - y_i)^2$ our loss functions, where $w$ is an expert and $f$ is a feature. We want to minimize the total regret in retrospect with respect to the best expert $w^*$:
\begin{equation}
    R(w) = \sum^T_{t=0}l_t(w_t) - l_t(w^*).
\end{equation}
We call $l_t(w_t) - l_t(w^*)$ the instantaneous regret for some $w_t$ at time $t$.
\subsection{Subgradient}
Subgradient at point $x$, $\nabla{f(x)}$, is a vector / hyperplane that lower bounds the function globally. For non-differentiable points, there exists more than one subgradients. For $x\neq y$, we have
\begin{equation}
    f(y) \geq f(x)+\nabla{f(x)}^T(y-x).
\end{equation}
For our instantaneous regret, we have
\begin{equation}
    \begin{split}
        l_t(w^*) &\geq l_t(w_t) + \nabla{l_t(w_t)}^T(w^*-w_t) \\
        l_t(w_t) - l_t(w^*) &\leq \nabla{l_t(w_t)}^T(w_t-w^*).
    \end{split}
\end{equation}
The left hand side is the instantaneous regret, and the right hand side is some linear function times $(w_t-w^*)$. Thus our total regret will be bounded by $\sum^T_{t=0}\nabla{l_t(w_t)}^T(w_t-w^*)$.

\newpage

\subsection{Algorithm for projected online subgradient descent}
This algorithm is a method to minimize the regret for a online convex optimization problem. \\
\begin{algorithm}
    \caption{Projected Subgradient Descent():}
    \begin{algorithmic}[1]
        \STATE choose $w_0$
        \FOR{$t = 1...T$}
        \STATE Incur loss $l(w_t)$ and receive any $\nabla l_t(w_t)$
        \STATE $\hat{w}_{t+1} \gets w_t - \alpha \nabla l_t(w_t)$
        \STATE $w_{t+1} \gets Proj_c[\hat{w}_{t+1}]$
        \ENDFOR
    \end{algorithmic}
\end{algorithm}
Line 5 projects $\hat{w}_{t+1}$ back into the convex set $C$, and $\alpha$ in line 4 is the learning rate. Smaller $\alpha$ pays a larger upfront cost but is more likely to converge and has a lower regret over time. $\alpha$ can also be dependent on $t$.

Note that the projection will not cause the loss to grow, because it will bring $\hat{w}_{t+1}$ closer to any member of $C$, and thus closer to the optimal expert $w^*$ too.

\section{Regret bounds for projected subgradient descent}
\subsection{Distance between $w_t$ and $w^*$}
The distance between $w_t$ and $w^*$ at time $t$ is defined as
\begin{equation}
    D(w_t, w^*) = (w_t-w^*)^T(w_t-w^*)
\end{equation}
Now we look at
\begin{equation}
    \begin{split}
        & D(w_{t+1}, w^*) - D(w_t, w^*) \\
        &= (w_t - \alpha\nabla{l_t(w_t)} - w^*)^2 - (w_t-w^*)^2 \\
        &= (z_t - \alpha \nabla{l_t(w_t)})^2 - z_t^2 \\
        &= \alpha^2 (\nabla{l_t(w_t)})^2 - 2\alpha\nabla{l_t^T(w_t)}z_t,
    \end{split}
\end{equation}
where $z_t = w_t - w^*$.
If we sum all the term over time, the intermediate terms will all cancel out and leave us just $D(w_T, w^*) - D(w_0, w^*)$.
\begin{equation}
    \begin{split}
        &= \sum_tD(w_{t+1, w^*}) - D(w_t, w^*) \\
        &= -2\alpha\sum_t(w_{t}-w^*)\nabla{l_t} + \alpha^2\sum_t|\nabla{l_t}|^2 \\
        &= D(w_T, w^*) - D(w_0, w^*) \\
        &\leq -2\alpha\sum_t(w_{t}-w^*)\nabla{l_t} + \alpha^2GT,
    \end{split}
\end{equation}
where $|\nabla{l_t}|^2 \leq G$. Thus we have
\begin{equation}
    2\alpha R_T \leq 2\alpha\sum_t(w_{t}-w^*)\nabla{l_t} \leq D(w_0, w^*) - D(w_T, w^*) + \alpha^2GT
\end{equation}
Sine the distance between $w_T$ and $w^*$ is always non negative, we can throw away the $D(w_T, w^*)$ term and still keep the inequality valid.
\begin{equation}
    R_T \leq \sum_t(w_{t}-w^*)\nabla{l_t} \leq \frac{D(w_0, w^*)}{2\alpha} + \frac{\alpha GT}{2} \leq \frac{\alpha GT}{2} + \frac{F}{2\alpha},
\end{equation}
where $F$ is the largest distance between any two experts in the set. \\
Suppose we set $alpha = \sqrt{\frac{F}{GT}}$, then the upper bound for total regret is bounded by $\sqrt{GTF}$, growing sub linearly of $T$.

\section{Portfolio optimization}
We want to invest in $n$ different stocks, and given a set of investment weights $w_i$ s.t. $w_i \geq 0$, and $\sum w_i = 1$. We also know about market returns ratios $r_i = \frac{value^i_{t+1}}{value^i_t}$. So the daily increase in wealth is $w_t^Tr_t$, and the total wealth over time is $m\Pi w_t^Tr_t$, where $m$ is the total value of initial investment. We want to maximize $\log\Pi w_t^Tr_t = \sum \log w_t^Tr_t$. We compare the policy to a constantly rebalancing portfolio that maintains a set constant investment ratios. \\
We can use use the algorithm presented above with some modifications
\begin{itemize}
    \item $w^i_0 = \frac{1}{n}$
    \item $w^i_{t+1} = Proj[w^i_{t} + \alpha\frac{r^i_t}{\sum w^j_tr^j_t}]$
\end{itemize}

\end{document}
