\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage{url}
\usepackage[tight]{subfigure}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}

\lecture{Statistical Techniques in Robotics (16-831, F10)}{Lecture \#28 (November 30, 2010)}{Lecturer: Drew Bagnell}{Scribe: Scott Satkin}{How to apply Machine Learning}

\begin{enumerate}
\item
Plot your data
\begin{itemize}
\item
Histogram each feature
\item
Scatter plot 2D/3D pairs or triplets of data
\item
Perform PCA and scatter plot projected data
\item
Unbalanced classes
\begin{enumerate}
\item
Weight loss function or gradient
\item
Upsample (duplicate data)
\begin{itemize}
\item
Downsides: overfitting and computational overhead
\end{itemize}
\item
Downsample (``costing'')
\begin{itemize}
\item
Downside: doesn't use all the data
\end{itemize}
\end{enumerate}
\end{itemize}
\item
Your hold-out data is sacred
\begin{itemize}
\item
Split data into training, validation and test sets.  Be cautious of trajectory data (sequential information) or groupable data (partial exchangability, \emph{e.g.}: different scenes or locations).
\end{itemize}
\item
Cris Dima's rule:
\begin{itemize}
\item
``Always start by overfitting.''  (Get an upper bound on performance)
\end{itemize}
\item
More data is better
\item
Understand what features matter
\begin{itemize}
\item
Too many features
\begin{itemize}
\item
Computational problems
\item
Overfitting problems
\end{itemize}
\item
Ablative analysis
\begin{itemize}
\item
Remove features one at a time and see if performance drops and iterate.
\item
Backwards or forwards greedy (\emph{e.g.}: Boosting)
\item
Analysis:
\begin{itemize}
\item
Greedy:  $F_1 G_{\infty} (\log d) \sqrt{T}$ 
\newline
Not good for sparse features.
\item
Alternative:   $F_2 G_2 \sqrt{T}$
\newline
$\sum{| w_i |} \le F_1 \subset F_2$
\end{itemize}
\end{itemize}
\end{itemize}
\item
Never underestimate the power of a linear predictor (Linear SVM, Logistic Regression, Linear Regression...).
\item
Never underestimate the power of ``well tuned'' Gradient Descent.  (Nelder-Mead if no derivatives).
\begin{itemize}
\item
Normalize features $[-1, 1]$
\item
Normalize standard deviations
\item
Normalize each feature vector
\item
Randomize data-points
\end{itemize}
\item
Understand Overfitting vs. Underfitting
\begin{itemize}
\item
Overfitting
\begin{itemize}
\item
Test on development set
\item
Regularize more
\item
Feature selection
\item
Get more data
\end{itemize}
\item
Underfitting
\begin{itemize}
\item
More features
\item
More sophisticated classifier
\item
Less regularization
\item
Kernelize or boost
\item
Optimize better
\end{itemize}
\end{itemize}
\item
Don't buy the hype!
\begin{itemize}
\item
Rich Caruana\footnote{\url{http://www.cs.cornell.edu/~caruana/}} papers
\item
SVM
\item
Random forests (``Random Kitchen Sinks'')
\item
ANN (often 2nd best option)
\end{itemize}
\item
Building Large Learning Systems
\begin{itemize}
\item 
Premature statistical optimization  (spend time on the parts that matter)
\item
Unit test learning code
\end{itemize}
\end{enumerate}
\end{document}

