\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage[tight]{subfigure}
\usepackage{algorithmic}
\usepackage{algorithm}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}

\lecture{Statistical Techniques in Robotics (16-831, F08)}{Lecture \#8 (Thursday September 18)}{Lecturer: Drew Bagnell}{Scribe: Forrest Rogers-Marcovitz}{Convex Optimization and Gradient Descent}

\section{Online Convex Programming}

\subsection{Review}
Convex loss functions reduce the computation time needed for large sets of experts in the weighted expert algorithm. We want to minimize the convex loss function, $l(w)$, where $w$ is an element of the convex set $C$.  In online convex programming, there is a loss function at each time step, $l_t(w)$, which is convex.  Every expert, $w^i$ is represented as a point within the convex set. We want to minimize our regret with regard to the best expert, $w^*$:

\begin{equation}
	R(w) = \displaystyle\sum_{t=0}^T l_t(w) - argMin_{w^* \in C} \displaystyle\sum_{t=0}^T l_t(w^*)
\end{equation}	

\subsection{Examples}
\begin{itemize}
	\item Smart Thermometer: \\
	Set the temperature in the room based on feedback from what the person has set the temperature in the past.
	We want to the algorithm, $w$ to predict the desired temperature:
	\begin{equation}
		w^{\top}x
	\end{equation}
	Given possible features:
	\begin{itemize}
		\item $x_0$ = Is it summer? ($1_{summer}$)
		\item $x_1$ = Amount of light
		\item $x_2$ = Is it Wednesday? ($1_{Wed}$)
	\end{itemize}
	Each day the loss function is:
	\begin{equation}
		l_t(w) = ( w^{\top} x_t - y_t)^2
	\end{equation}
	where $y_t$ is the user selected temperature that day.
	
	\item Tree Detection: \\
	Given good object detection up-close, trees in this example, we want to learn to recognize similar objects far away. An example of this would be to have good laser scan data for short distances and trying to extend that to far objects detected via image characteristics (shape, texture, color). The algorithm will try to detect objects far away and then check if it was right when closer; this is an example of self-supervised online learning. Because what we predict influences what we see in the future, regret might not be a good measure of performance as it might change depending on the examples given.  We will return to this example later.
	
\end{itemize}		

\section{Subgradients}

\subsection{Definition}
Subgradient methods are algorithms for solving convex optimization problems. Subgradient methods can be used with a non-differentiable objective function. When the objective function is differentiable, subgradient methods for unconstrained problems use the same search direction as the method of steepest descent. The subgradient at point $x$ is a vector $\nabla f_x$ such that it lower bounds the function globally. Convex functions have subgradients at all points along it.

Given some point $y$ that is further from the minimal of the convex set than $x$. We know that:
\begin{equation}
	f(y) \ge \nabla f_x^{\top}(y-x) + f(x)
\end{equation}	

\subsection{Subgradients for online learning}
Given a convex loss function, we can use subgradients to reduce the regret of online learning.
\begin{proof}
\begin{equation}
	l(w^*) \ge \nabla l_t(w_t)^{\top}(w^*-w_t) + l_t(w_t)
\end{equation}
\begin{equation}
	l_t(w_t) - l(w^*) \le \nabla l_t(w_t)^{\top}(w_t-w^*) 
\end{equation}
\begin{equation}
	R(w_t) \le \displaystyle\sum_{t=0}^T [l_t(w_t) - l(w^*)] \le  \displaystyle\sum_{t=0}^T [\nabla l_t(w_t)^{\top}(w_t-w^*)]
\end{equation}
\renewcommand{\qedsymbol}{}
\end{proof}

We see that regret is bounded by a linear function. This means that it is hardest to optimize on linear (flat) functions.  Also, regret is highest if the subgradient is parallel to $w_t-w^*$ and in the opposite direction. The next algorithm will take advantage of this fact.

\section{Projected Subgradient Descent}

\subsection{Algorithm}
This algorithm is a method to minimize the regret for a online convex optimization problem.

\begin{algorithm}
  \caption{Projected Subgradient Descent():}
  \begin{algorithmic}[1]
    \STATE Predict $w_0$
    \FOR{$t = 1...T$}
    \STATE Receive $l(w_t)$ and $\nabla l(w_t)$
    \STATE $\hat{w}_{t+1} \gets w_t - \alpha \nabla l(w_t)$
    \STATE $w_{t+1} \gets Proj_c[\hat{w}_{t+1}]$
    \ENDFOR    
  \end{algorithmic}
\end{algorithm}

Line $5$ projects the weights back into the convex set. Based on the convex set, this may be difficult for an exact solution. 
\\\\
In line $4$, $\alpha$ represents the step size, or learning rate. Large values of $\alpha$ will learn faster but are less likely to converge as it may step over the minimal. Smaller values pay a larger upfront cost but are more likely to converge and have a lower regret over time.  There are various methods where $\alpha_t$ decreases over time.

\subsection{Examples}
\begin{itemize}
	\item Tree Prediction: \\
	Using the example described above, we are given the following state features:
	\begin{itemize}
		\item $x_1 = Red, x_2 = Blue, x_3 = Green$
		\item $x_4$ = point spread
		\item $x_5$ = response of Gaber filter on image patch (texture)
		\\
		\item $y_t = \{1,-1\} \leftarrow$ tree or no tree
	\end{itemize}
	Using the loss function:
	\begin{equation}
		l_t(w) = ( w_t^{\top} x_t - y_t)^2
	\end{equation}
	We can use the following update rule:
	\begin{equation}
		w_{t+1} \gets w_t - 2 \alpha  (w_t^{\top}x_t - y_t) \cdot x_t
	\end{equation}
	where $(w_T^{\top}x_t - y_t)$ is the residual.
	
	\item Portfolio Optimization: \\
	We are given a set of investment weights, $w_i$, and daily market return ratios, $r_i$ such that:
	\begin{equation}
		w_i \ge 0
	\end{equation}
	\begin{equation}
		\sum w_i = 1
	\end{equation}		
	
	The daily increase in wealth is:
	\begin{equation}
		w_t^{\top}r_t
	\end{equation}
	and total wealth over time is:
	\begin{equation}
		\displaystyle\prod_{t=1}^T w_t^{\top}r_t
	\end{equation}
	
	We want to optimize the log-gain function:
	\begin{equation}
		\displaystyle\sum_{t=1}^T log(w_t^{\top}r_t)
	\end{equation}
	
	We will compare the policy to a constantly rebalancing portfolio that adjusts the investments each day to have constant investment ratios.
	
	We make to following adjustments to the Projected Subgradient Descent algorithm:
	\begin{itemize}
		\item $w_0^i = \frac{1}{N}$
		\item $w_{t+1}^i \gets w_t^i + \alpha \frac{r_t^i}{\sum_j w_t^j \cdot r_t^j} $
		\item Project each $w_{t+1}^i$ back to the convex function (simplex)
	\end{itemize}
	
	The following problems exist with this algorithm, so you won't get "fabulously wealthy":
	\begin{itemize}
		\item Constant balance may not necessarily be the best policy over time
		\item Fixed transaction costs add up
		\item Large actions will affect the stock's market price
	\end{itemize}	
\end{itemize}	

\end{document}