\documentclass[11pt]{article}
\usepackage{packages/mlapa}
\usepackage{packages/fullpage}
\usepackage{graphics}
\usepackage{graphicx}
\usepackage{psfrag}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{psfig}
\usepackage{amsfonts}
\usepackage{latexsym}
\usepackage{amscd}

\renewcommand{\baselinestretch}{1.1}
\newcommand{\ignore}[1]{}
\newcommand{\deff}{\mbox{$\;\stackrel{\mbox{\tiny\rm def}}{=}\;$}}
\newcommand{\bb}{\boldsymbol}
\newcommand{\dd}{\mbox}
\def\norm#1{\left\|#1\right\|}

\newtheorem{prop}{Proposition}
\newtheorem{lemma}{Lemma}
\newtheorem{thrm}{Theorem}
\newtheorem{defn}{Definition}
\newtheorem{corr}{Corollary}

\def\cA{\mathcal{A}} % caligraphic math letters
\def\cX{\mathcal{X}}
\def\cY{\mathcal{Y}}
\def\cM{\mathcal{M}}
\def\cN{\mathcal{N}}
\def\cP{\mathcal{P}}
\def\cF{\mathcal{F}}
\def\cE{\mathcal{E}}
\def\cQ{\mathcal{Q}}
\def\cD{\mathcal{D}}
\def\cL{\mathcal{L}}
\def\Rn{\mathbb{R}^n} % blackboard math letters
\def\R{\mathbb{R}}
\def\Pn{\mathbb{P}_n}
\def\Pm{\mathbb{P}_m}
\def\P{\mathbb{P}}
\def\H{\mathbb{H}}
\def\N{\mathbb{N}}
\def\cJ{\mathcal{J}}
\def\Rn1{\mathbb{R}^{n+1}}
\def\Sn{\mathbb{S}^{n}}
\def\Snp{\mathbb{S}^{n}_{+}}
\def\ss#1{\mbox{\it\scriptsize #1}} % font sizes
\def\st#1{\mbox{\it\tiny #1}}
\def\argmin{\operatornamewithlimits{arg\, min}} % new operators
\def\argmax{\operatornamewithlimits{arg\, max}}
\def\dotp#1#2{\left\langle{#1}, {#2}\right\rangle}
\def\dvol{\text{dvol}\,}
\def\seq{\;=\;}
\def\given{\,|\,}
\def\det{\text{det}\,}
\def\div{\text{div}\,}
\def\grad{\text{grad}\,}



%-----------------------------------------------------------------
%-----------------------------------------------------------------
%-----------------------------------------------------------------
\title{Riemannian Geometry and Statistical Machine Learning
\\ \vspace{0.5in} \normalsize{\textsc{Doctoral Thesis\\
\vspace{0.3in} }}} \vspace{1in}
\author{Guy Lebanon\\
\small{Language Technologies Institute}\\
\small{School of Computer Science}\\
\small{Carnegie Mellon University}\\
\small{\texttt{lebanon@cs.cmu.edu}}\\}
\date{\today}
\begin{document}
\maketitle
\vspace{1in}
\begin{abstract}
Statistical machine learning algorithms deal with the problem of
selecting an appropriate statistical model from a model space
$\Theta$ based on a training set $\{x_i\}_{i=1}^N \subset \cX$ or
$\{(x_i,y_i)\}_{i=1}^N \subset \cX\times \mathcal{Y}$.
In doing so they either implicitly or explicitly make
assumptions on the geometries of the model space $\Theta$
and the data space $\cX$. Such assumptions are crucial
to the success of the algorithms as different geometries
are appropriate for different models and data spaces.
By studying these assumptions we are able to develop new theoretical
results that enhance our understanding of several popular learning
algorithms. Furthermore, using geometrical reasoning we are able to
adapt existing algorithms such as radial basis kernels and linear
margin classifiers to non-Euclidean geometries.
Such adaptation is shown to be useful when the data space
does not exhibit Euclidean geometry.
In particular, we focus in our experiments on the space of text documents
that is naturally associated with the Fisher information metric
on corresponding multinomial models.
\end{abstract}
\vspace{1in}
\begin{tabular}{ll}
Thesis Committee:& John Lafferty (chair)\\
 &Geoffrey J. Gordon, Michael I. Jordan,
Larry Wasserman
\end{tabular}


\newpage
\section*{Acknowledgements} This thesis contains work that I did
during the years 2001-2004 at Carnegie Mellon University. During
that period I received a lot of help from faculty, students and
friends. However, I feel that I should start by thanking my family:
Alex, Anat and Eran Lebanon, for their support during my time at
Technion and Carnegie Mellon. Similarly, I thank Alfred Bruckstein,
Ran El-Yaniv, Michael Lindenbaum and Hava Siegelmann from the
Technion for helping me getting started with research in computer
science.

\paragraph{}
At Carnegie Mellon University I received help from a number of
people, most importantly my advisor John Lafferty. John helped me in
many ways. He provided excellent technical hands-on assistance, as
well as help on high-level and strategic issues. Working with John
was a very pleasant and educational experience. It fundamentally
changed the way I do research  and turned me into a better
researcher. I also thank John for providing an excellent environment
for research without distractions and for making himself available
whenever I needed.

\paragraph{}
I thank my thesis committee members Geoffrey J. Gordon, Michael I.
Jordan and Larry Wasserman for their helpful comments. I benefited
from interactions with several graduate students at Carnegie Mellon
University. Risi Kondor, Leonid Kontorovich, Luo Si, Jian Zhang and
Xioajin Zhu provided helpful comments on different parts of the
thesis. Despite not helping directly on topics related to the
thesis, I benefited from interactions with Chris Meek at Microsoft
Research, Yoram Singer at the Hebrew University and Joe Verducci and
Douglas Critchlow at Ohio State University. These interactions
improved my understanding of machine learning and helped me write a
better thesis.

\paragraph{}
Finally, I want to thank Katharina Probst, my girlfriend for the
past two years. Her help and support made my stay at Carnegie Mellon
very pleasant, despite the many stressful factors in the life of a
PhD student. I also thank her for patiently putting up with me
during these years.

\newpage\tableofcontents\newpage

\section*{Mathematical Notation}
Following is a list of the most frequent mathematical notations in
the thesis.
\paragraph{}
\begin{tabular}{ll}
$\cX$ & Set/manifold of data points\\
$\Theta$ & Set/manifold of statistical models\\
$X\times Y$ & Cartesian product of sets/manifold\\
$X^k$ & The Cartesian product of  $X$ with itself $k$ times\\
$\|\cdot\|$ & The Euclidean norm \\
$\dotp{\cdot}{\cdot}$ & Euclidean dot product between two vectors\\
$p(x\,;\theta)$ & A probability model for $x$ parameterized by $\theta$\\
$p(y|x\,;\theta)$ & A probability model for $y$, conditioned on $x$
 and  parameterized by $\theta$\\
$D(\cdot,\cdot),D_r(\cdot,\cdot)$ & $I$-divergence between two models\\
$T_x \cM$ & The tangent space to the manifold $\cM$ at $x\in\cM$\\
$g_x(u,v)$ & A Riemannian metric at $x$ associated with the tangent vectors $u,v\in T_x \cM$\\
$\{\partial_i\}_{i=1}^n,\{e_i\}_{i=1}^n$ &
The standard basis associated with the vector space $T_x\cM\cong \R^n$\\
$G(x)$ & The Gram matrix of the metric $g$, $[G(x)]_{ij}=g_x(e_i,e_j)$\\
$\cJ_{\theta}(u,v)$ & The Fisher information metric at the model $\theta$
associated with the vectors $u,v$\\
$\delta_{x}(u,v)$ & The induced Euclidean local metric $\delta_x(u,v)=\dotp{u}{v}$\\
$\delta_{x,y}$ & Kronecker's delta $\delta_{x,y}=1$ if $x=y$ and 0 otherwise\\
$\iota:A\to X$ & The inclusion map $\iota(x)=x$ from $A\subset X$ to $X$.\\
$\mathbb{N},\mathbb{Q},\mathbb{R}$ & The natural, rational and real numbers
respectively\\
$\R_{+}$ & The set of positive real numbers\\
$\R^{k\times m}$ & The set of real $k\times m$ matrices\\
$[A]_i$ & The $i$-row of the matrix $A$\\
$\{\partial_{ab}\}_{a,b=1}^{k,m}$ & The standard basis
associated with $T_x \R^{k\times m}$\\
$\overline X$ & The topological closure of $X$\\
$\H^n$ & The upper half plane $\{x\in\R^n:x_n\geq 0\}$\\
$\Sn$ & The $n$-sphere $\Sn=\{x\in\R^{n+1}:\sum_i x_i^2=1\}$\\
$\Snp$ & The positive orthant of the $n$-sphere  $\Snp=\{x\in\R^{n+1}_{+}:\sum_i x_i^2=1\}$\\
$\Pn$ & The $n$-simplex $\Pn=\{x\in\R_{+}^{n+1}:\sum_i x_i=1\}$\\
$f\circ g$ & Function composition $f\circ g(x)=f(g(x))$\\
$C^{\infty}(\cM,\cN)$ & The set of infinitely differentiable functions from $\cM$ to $\cN$\\
$f_*u$ & The push-forward map $f_*:T_x\cM\to T_{f(x)}\cN$ of $u$
associated with $f:\cM\to\cN$\\
$f^*g$ & The pull-back metric on $\cM$ associated with $(\cN,g)$ and $f:\cM\to \cN$\\
$d_g(\cdot,\cdot),d(\cdot,\cdot)$ & The geodesic distance associated with the metric $g$\\
$\dvol g(\theta)$ & The volume element of the metric $g_{\theta}$,
$\dvol g(\theta)=\sqrt{\det g_{\theta}}=\sqrt{\det G(\theta)}$\\
$\nabla_X Y$ & The covariant derivative of the vector
field $Y$ in the direction \\
& of the vector field $X$ associated with the connection $\nabla$
\end{tabular}
\newpage
\begin{tabular}{ll}
$\ell(\theta)$ & The log-likelihood function\\
$\cE(\theta)$ & The AdaBoost.M2 exponential loss function\\
$\tilde p$ & Empirical distribution associated with a training set $D\subset \cX\times \cY$\\
$\hat u$ & The $L^2$ normalized version of the vector $u$\\
$d(x,S)$ & The distance from a point to a set
$d(x,S)=\inf_{y\in S}d(x,y)$\\
\end{tabular}
\newpage


%-----------------------------------------------------------------
%-----------------------------------------------------------------
%-----------------------------------------------------------------
\section{Introduction} \label{sec:introduction}
There are two fundamental spaces in machine learning.
The first space $\cX$ consists of data points
and the second space $\Theta$ consists of possible learning models.
In statistical learning, $\Theta$ is usually a space of
statistical models, $\{p(x\,;\theta):\theta\in\Theta\}$ in the generative case
or $\{p(y\given x\,;\theta):\theta\in\Theta\}$  in the discriminative case.
The space $\Theta$ can be either a low dimensional parametric space as in
parametric statistics, or the space of all possible models as in non-parametric
statistics.


\paragraph{}
Learning algorithms select a model $\theta\in\Theta$ based on a training sample
$\{x_i\}_{i=1}^n\subset \cX$ in the generative case
or $\{(x_i,y_i)\}_{i=1}^n\subset \cX\times\mathcal{Y}$ in the
discriminative case.
In doing so they, either implicitly or explicitly, make certain
assumptions about the geometries of $\cX$ and $\Theta$.
In the supervised case, we focus on the classification setting, where
$\mathcal{Y}=\{y_1,\ldots,y_k\}$
is a finite set of unordered classes. By this we mean that
the space
\[\cX\times\mathcal{Y}=\cX\times\cdots\times\cX
=\cX^k\]
has the product geometry over $\cX$.
This is a common assumption that makes sense in many practical situations,
where there is no clear relation or hierarchy between the classes.
As a result, we will mostly ignore the role of $\mathcal{Y}$ and restrict our
study of data spaces to $\cX$.


\paragraph{}
Data and model spaces are rarely Euclidean spaces. In data space,
there is rarely any meaning to adding or subtracting two data points
or multiplying a data point by a real scalar. For example, most
representations of text documents as vectors are non-negative and
multiplying them by a negative scalar does not yield another
document. Similarly, images $I$ are usually represented as matrices
whose entries are in some bounded interval of the real line $I\in
[a,b]^{k\times m}$ and there is no meaning to matrices with values
outside that range that are obtained by addition or scalar
multiplication. The situation is similar in model spaces. For
example, typical parametric families such as normal, exponential,
Dirichlet or multinomial, as well as the set of non-negative,
normalized distributions are not Euclidean spaces.

\paragraph{}
In addition to the fact that data and model spaces are rarely $\R^n$
as topological spaces, the geometric structure of Euclidean spaces,
expressed through the Euclidean distance
\[ \|x-y\| \deff \sqrt{\sum_i |x_i-y_i|^2}\]
is artificial on most data and model spaces. This holds even in many
cases when the data or models are real vectors. To study the
geometry of $\cX$ and $\Theta$, it is essential to abandon the realm
of Euclidean geometry in favor of a new, more flexible class of
geometries. The immediate generalization of Euclidean spaces, Banach
and Hilbert spaces, are still vector spaces, and by the arguments
above, are not a good model for $\cX$ and $\Theta$. Furthermore, the
geometries of Banach and Hilbert spaces are quite restricted, as is
evident from the undesirable linear scaling of the distance
\[  d_E(cx,cy)=|c|\, d_E(x,y) \qquad \forall c\in\R.\]

\paragraph{}
Despite the fact that most data and model  spaces are not Euclidean,
they share two important properties: they are smooth and they are
locally Euclidean. Manifolds are the natural generalization of
Euclidean spaces to locally Euclidean spaces and differentiable
manifolds are their smooth counterparts. Riemannian geometry is a
mathematical theory of geometries on such smooth, locally Euclidean
spaces. In this framework, the geometry of a space is specified by a
local inner product $g_x(\cdot,\cdot), x\in\cX$ between tangent
vectors called the Riemannian metric. This inner product translates
into familiar geometric quantities such as distance, curvature and
angles. Using the Riemannian geometric approach to study the
geometries of $\cX$ and $\Theta$ allows us to draw upon the vast
mathematical literature in this topic. Furthermore, it is an
adequate framework as it encompasses most commonly used geometries
in machine learning. For example, Euclidean geometry on
$\cX=\mathbb{R}^n$ is achieved by setting the local metric
$g_x(u,v), x\in\cX$ to be the Euclidean inner product
\[\delta_x(u,v) \deff \dotp{u}{v} \deff \sum_{i=1}^n u_iv_i.\]
The information geometry on a space
of statistical models $\Theta\subset\mathbb{R}^n$
is achieved by setting the metric
$g_{\theta}(u,v), \theta\in\Theta$ to be the Fisher information
$\cJ_{\theta}(u,v)$
\begin{align}
\cJ_{\theta}(u,v)
&\deff \sum_{i=1}^n\sum_{j=1}^n u_i v_j
\,\int p(x\,;\theta)
    \,\frac{\partial}{\partial \theta_i}\log p(x\,;\theta)
    \,\frac{\partial}{\partial \theta_j}\log p(x\,;\theta)\,\mbox{d} x
\end{align}
where the above integral is replaced with a sum if $\cX$ is discrete.


\paragraph{}
This thesis is concerned with applying tools from Riemannian geometry to
study the relationship between statistical learning algorithms and
different geometries of $\cX$ and $\Theta$. As a result,
we gain considerable insight into current learning algorithms and
we are able to design powerful
new techniques that often outperform the current state-of-the-art.

\paragraph{}
We start the thesis with  Section~\ref{sec:RiemannianGeometry} that
contains background from Riemannian geometry that is relevant to
most subsequent sections. Additional background that is relevant to
a specific section will appear in that section alone. The treatment
in Section~\ref{sec:RiemannianGeometry} is short and often not
rigorous. For a more complete description most textbooks in
Riemannian geometry will suffice. Section~\ref{sec:previousWork}
gives an overview of relevant research in the interface of machine
learning and statistics, and Riemannian geometry and
Section~\ref{sec:probSimplexGeom} applies the mathematical theory of
Section~\ref{sec:RiemannianGeometry} to spaces of probability
models.

\paragraph{}
In Section~\ref{sec:boostMl} we study the geometry of the space
$\Theta$ of conditional models  underlying the algorithms logistic
regression and AdaBoost. We prove the surprising result that both
algorithms solve the same primal optimization problem with the only
difference being that AdaBoost lacks normalization constraints,
hence resulting in non-normalized models. Furthermore, we show that
both algorithms implicitly minimize the conditional $I$-divergence
\begin{align*}
D(p,q) = \sum_{i=1}^n \sum_y \left(
p(y| x_i) \log \frac{p(y| x_i)}{q(y| x_i)} - p(y| x_i) + q(y| x_i) \right)
\end{align*}
to a uniform model $q$. Despite the fact that the $I$-divergence is not
a metric distance function, it is related to a distance under a specific
geometry on $\Theta$. This geometry is the product Fisher information geometry
whose study is pursued in Section~\ref{sec:axiomaticGeometry}.


\paragraph{}
By generalizing the theorems of \v{C}encov and Campbell,
Section~\ref{sec:axiomaticGeometry} shows that the only geometry
on the space of conditional distributions
consistent with a basic set of axioms is the product Fisher information.
The results of Sections~\ref{sec:boostMl} and \ref{sec:axiomaticGeometry},
provide an axiomatic characterization of the geometries underlying
logistic regression and AdaBoost.
Apart from providing a substantial new understanding of logistic
regression and AdaBoost, this analysis provides,
for the first time a theory of information geometry for conditional models.

\paragraph{}
The axiomatic framework mentioned above provides a natural geometry
on the space of distributions $\Theta$. It is less clear what should
be an appropriate geometry for the data space $\cX$. A common
pitfall shared by many classifiers is to assume that $\cX$ should be
endowed with a Euclidean geometry. Many algorithms, such as radial
basis machines and Euclidean $k$-nearest neighbor, make this
assumption explicit. On the other hand, Euclidean geometry is
implicitly assumed in linear classifiers such as logistic
regression, linear support vector machines, boosting and the
perceptron. Careful selection of an appropriate geometry for $\cX$
and designing classifiers that respect it should produce better
results than the naive Euclidean approach.

\paragraph{}
In Section~\ref{sec:embedding} we propose the following principle to
obtain a natural geometry on the data space. By assuming the data is
generated by statistical models in the space $\cM$, we embed data
points $x\in\cX$ into $\cM$ and thus obtain a natural geometry on
the data space -- namely the Fisher geometry on $\cM$. For example,
consider the data space  $\cX$ of text documents in normalized term
frequency (tf) representation embedded in the space of all
multinomial models $\cM$. Assuming the documents are generated from
multinomial distributions we obtain the maximum likelihood embedding
$\hat{\theta}_{MLE}:\cX\to\cM$ which is equivalent to the inclusion
map
\[\iota:\cX\hookrightarrow\cM, \qquad \iota(x)=x.\]
Turning to \v{C}encov's theorem and selecting the Fisher geometry
on $\cM$ we obtain a natural geometry on the closure of the data space
$\overline{\cX}=\cM$.


\paragraph{}
The embedding principle leads to a general framework for adapting
existing algorithms to the Fisher geometry. In
Section~\ref{sec:hyperplane} we generalize the notion of linear
classifiers to non-Euclidean geometries and derive in detail the
multinomial analogue of logistic regression. Similarly, in
Section~\ref{sec:diffusion} we generalize radial basis kernel to
non-Euclidean geometries by approximating the solution of the
geometric diffusion equation. In both cases, the resulting
non-Euclidean generalizations outperform its Euclidean counterpart,
as measured by classification accuracy in several text
classification tasks.

\paragraph{}
The Fisher geometry is a natural choice if the only known information
is the statistical family that generates the data. In the presence of actual
data $\{x_1,\ldots,x_n\}\subset \cX$ it may be possible to induce
a geometry that is better suited for it.
In Section~\ref{sec:metricLearning}
we formulate a learning principle for the geometry of $\cX$
that is based on maximizing the inverse volume of the given training set.
When applied to the space of text documents in tf representation, the
learned geometry is similar to, but outperforms the popular
tf-idf geometry.


\paragraph{}
We conclude with a discussion in Section~\ref{sec:discussion}.
The first two appendices contain technical information
relevant to Sections~\ref{sec:boostMl} and \ref{sec:metricLearning}.
Appendix~\ref{sec:contributionSummary} contains
a summary of the major contributions included in this thesis,
along with relevant publications.


%----------------------------------------------------------------
\section{Relevant Concepts from Riemannian Geometry}
\label{sec:RiemannianGeometry}
In this section we describe concepts from
Riemannian geometry that are relevant to most of the thesis, with the possible
exception of Section~\ref{sec:boostMl}. Other concepts from Riemannian geometry that
are useful only for a specific section will be introduced later in the thesis as needed.
For more details refer to any textbook discussing Riemannian geometry.
\emcite{Milnor:63} and \emcite{Spivak1975} are particularly well known
classics and \emcite{Lee2002} is a well-written contemporary textbook.


\paragraph{}
Riemannian manifolds are built out of three layers of structure. The
topological layer is suitable for treating topological notions such
as continuity and convergence. The differentiable layer allows
extending the notion of differentiability to the manifold and the
Riemannian layer defines rigid geometrical quantities such as
distances, angles and curvature on the manifold. In accordance with
this philosophy, we start below with the definition of topological
manifold and quickly proceed to defining differentiable manifolds
and Riemannian manifolds.

\subsection{Topological and Differentiable Manifolds}
A homeomorphism between two topological spaces $X$ and $Y$ is a
bijection $\phi:X\to Y$  for which both $\phi$ and $\phi^{-1}$ are
continuous. We then say that $X$ and $Y$ are homeomorphic and
essentially equivalent from a topological perspective. An
$n$-dimensional topological manifold $\cM$ is a topological subspace
of $\mathbb{R}^m, m\geq n$ that is locally equivalent to
$\mathbb{R}^n$ i.e. for every point $x\in\cM$ there exists an open
neighborhood $U\subset \cM$ that is homeomorphic to $\mathbb{R}^n$.
The local homeomorphisms in the above definition
$\phi_U:U\subset\cM\to\mathbb{R}^n$ are usually called charts. Note
that this definition of a topological manifold makes use of an
ambient Euclidean space $\mathbb{R}^m$. While sufficient for our
purposes, such a reference to $\mathbb{R}^m$ is not strictly
necessary and may be discarded at the cost of certain topological
assumptions\footnote{The general definition, that uses the Hausdorff
and second countability properties, is equivalent to the ambient
Euclidean space definition by Whitney's embedding theorem.
Nevertheless, it is considerably more elegant to do away with the
excess baggage of an ambient space.} \cite{Lee2000}. Unless
otherwise noted, for the remainder of this section we assume that
all manifolds are of dimension $n$.

\paragraph{}
An $n$-dimensional topological manifold with a boundary is defined similarly to an
$n$-dimensional topological manifold, except that each point has a neighborhood that
is homeomorphic to an open subset of the upper half plane
\begin{align*} \label{eq:upperHalfPlane}
\H^n \deff \left\{x\in\mathbb{R}^n: x_n\geq 0\right\}.
\end{align*}
It is possible to show that in this case some points $x\in\cM$ have
neighborhoods homeomorphic to $U\subset \mathbb{H}^n$ such that
$\forall y\in U, y_n>0$ while other points are homeomorphic to a
subset $U\subset \mathbb{H}^n$ that intersects the line $y_n=0$.
These two sets of points are disjoint and are called the interior
and boundary of the manifold $\cM$  and are denoted by
$\text{Int}\,\cM$ and  $\partial \cM$ respectively \cite{Lee2000}.

\paragraph{}
Figure \ref{fig:manifoldWithBoundary} illustrates the concepts
associated with a manifold with a boundary. Note that a manifold is
a manifold with a boundary but the converse does not hold in
general. However, if $\cM$ is an $n$-dimensional manifold with a
boundary then $\text{Int}\cM$ is an $n$ dimensional manifold and
$\partial \cM$ is an $n-1$ dimensional manifold. The above
definition of boundary and interior of a manifold may differ from
the topological notions of boundary and interior, associated with an
ambient topological space. When in doubt, we will specify whether we
refer to the manifold or topological interior and boundary.
\begin{figure}
\centering
{
\psfrag{m}{$\cM$}
\psfrag{g}{$x$}
\psfrag{h}{$y$}
\psfrag{f}{$\mathbb{H}^2$}
\includegraphics[scale=0.57]{RiemannianGeomFigures/manifoldWithBoundary.eps}}
\caption{A 2-dimensional manifold with a boundary.
The boundary $\partial\cM$ is marked by a black contour.
For example, $x$ is a boundary point $x\in\partial \cM$
while $y\in\text{Int}\,\cM$ is an interior point.}
\label{fig:manifoldWithBoundary}
\end{figure}
We return to manifolds with boundary at the end of this Section.

\paragraph{}
We are now in a position to introduce the differentiable structure.
First recall that a mapping between two open sets of Euclidean spaces
$f:U\subset \mathbb{R}^k\to V\subset\mathbb{R}^l$ is infinitely
differentiable, denoted by $f\in C^{\infty}(\mathbb{R}^k,\mathbb{R}^l)$
if $f$ has continuous partial derivatives of all orders.
If for every pair of charts $\phi_U,\phi_V$
the transition function defined by
\[\psi:\phi_V(U\cap V)\subset \mathbb{R}^n\to\mathbb{R}^n,
\quad \psi=\phi_U\circ\phi_V^{-1}\]
(when $U\cap V\neq \emptyset$)
is a $C^{\infty}(\mathbb{R}^n,\mathbb{R}^n)$ differentiable map
then $\cM$ is called an $n$-dimensional differentiable manifold.
The charts and transition function for a 2-dimensional manifold
are illustrated in Figure~\ref{fig:transitionFunction}.

\begin{figure}
\centering
{
\psfrag{m}{$\cM$}
\psfrag{h}{$\phi_V:V\to\mathbb{R}^2$}
\psfrag{g}{$\phi_U:U\to\mathbb{R}^2$}
\psfrag{f}{$\text{dom}\,\psi=\phi_V(U\cap V)$}
\psfrag{k}{$\psi=\phi_U\circ\phi_V^{-1}$}
\psfrag{u}{$U$}
\psfrag{v}{$V$}
\psfrag{l}{$\text{range}\,\psi=\phi_U(U\cap V)$}
\includegraphics[scale=0.57]{RiemannianGeomFigures/transitionFunction.eps}}
\caption{Two neighborhoods $U,V$ in a
 2-dimensional manifold $\cM$, the coordinate charts
$\phi_U,\phi_V$ and the transition function $\psi$ between
them.}
\label{fig:transitionFunction}
\end{figure}

\paragraph{}
Differentiable manifolds of dimensions 1 and 2 may be visualized
as smooth curves and surfaces in Euclidean space.
Examples of $n$-dimensional differentiable manifolds are
the Euclidean space $\mathbb{R}^n$, the $n$-sphere
\begin{align}\label{eq:sphereDef}
\Sn \deff \left\{x\in\mathbb{R}^{n+1}:\sum_{i=1}^n x_i^2=1\right\},
\end{align}
its positive orthant
\begin{align}\label{eq:posSphereDef}
\Snp \deff \left\{x\in\mathbb{R}^{n+1}:\sum_{i=1}^n x_i^2=1, \,\, \forall i\,\, x_i>0\right\},
\end{align}
and the $n$-simplex
\begin{align}\label{eq:simplexDef}
\mathbb{P}_n \deff \left\{x\in\mathbb{R}^{n+1}:
\sum_{i=1}^n x_i=1, \,\, \forall i\,\, x_i>0\right\}.
\end{align}
We will keep these examples in mind as they will keep appearing throughout
the thesis.

\paragraph{}
Using the charts, we can extend the definition of
differentiable maps to real valued functions on manifolds
$f:\cM\to\mathbb{R}$ and functions from one manifold to another
$f:\cM\to\cN$.
A continuous function $f:\cM\to\mathbb{R}$ is said
to be $C^{\infty}(\cM,\mathbb{R})$ differentiable if
for every chart $\phi_U$ the function
$f\circ \phi_U^{-1}\in C^{\infty}(\mathbb{R}^n,\mathbb{R})$.
A continuous mapping between two differentiable manifolds
$f:\cM\to\cN$ is said to be
$C^{\infty}(\mathcal{M,N})$ differentiable if
\[\forall r\in C^{\infty}(\cN,\mathbb{R}),
\qquad r\circ f\in C^{\infty}(\cM,\mathbb{R}).\]
A diffeomorphism between two manifolds $\mathcal{M,N}$ is a
bijection $f:\cM\to\cN$
such that $f\in C^{\infty}(\mathcal{M,N})$
and $f^{-1}\in C^{\infty}(\mathcal{N,M})$.

\subsection{The Tangent Space}
For every point $x\in\cM$, we define an $n$-dimensional real vector space
$T_x\cM$, isomorphic to $\R^n$, called the tangent space.
The elements of the tangent space, the tangent vectors $v\in T_x \cM$,
are usually defined as directional derivatives at $x$ operating
on $C^{\infty}(\cM,\mathbb{R})$ differentiable functions
or as equivalence classes of curves having the same velocity vectors
at $x$ \cite{Spivak1975,Lee2002}.
Intuitively, tangent spaces and tangent vectors are a generalization
of geometric tangent vectors and spaces for smooth curves and
two dimensional surfaces in the ambient $\mathbb{R}^3$.
For an $n$-dimensional manifold $\cM$ embedded in an ambient $\R^m$
the tangent space $T_x \cM$ is a copy of $\R^n$ translated so
that its origin is positioned at $x$. See Figure~\ref{fig:tangentSpaces}
for an illustration of this concept for two dimensional manifolds in $\R^3$.

\begin{figure}
\centering
{
\psfrag{p}{$\mathbb{P}_2$}
\psfrag{s}{$\mathbb{S}_2$}
\psfrag{x}{$T_x\mathbb{P}_2$}
\psfrag{t}{$T_x\mathbb{S}_2$}
\includegraphics[scale=0.7]{RiemannianGeomFigures/tangentSpaces.eps}}
\caption{Tangent spaces of the 2-simplex $T_x\mathbb{P}_2$
and the 2-sphere $T_x\mathbb{S}_2$.}
\label{fig:tangentSpaces}
\end{figure}


\paragraph{}
In many cases the manifold $\cM$  is a
submanifold of an $m$-dimensional manifold $\cN$, $m\geq n$.
Considering $\cM$ and its ambient space $\mathbb{R}^m, m\geq n$
is one special case of this phenomenon.
For example, both $\Pn$ and $\Sn$ defined in
\eqref{eq:sphereDef},\eqref{eq:simplexDef}
are submanifolds of $\mathbb{R}^{n+1}$.
In these cases,
the tangent space of the submanifold $T_x\cM$
is a vector subspace of $T_x\cN\cong \mathbb{R}^m$  and we may represent
tangent vectors $v\in T_x\cM$ in the standard basis
$\{\partial_i\}_{i=1}^m$  of the embedding tangent
space $T_x\mathbb{R}^m$ as $v=\sum_{i=1}^m v_i \partial_i$.
For example, for the simplex and the sphere we have (see  Figure~\ref{fig:tangentSpaces})
\begin{align}\label{eq:simplexTangentSpace}
T_x\mathbb{P}_n=\left\{ v\in\mathbb{R}^{n+1}:\sum_{i=1}^{n+1}
v_i=0\right\} \qquad \qquad T_x\mathbb{S}_n=\left\{
v\in\mathbb{R}^{n+1}:\sum_{i=1}^{n+1} v_ix_i=0\right\}.
\end{align}


\paragraph{}
A $C^{\infty}$ vector field\footnote{The precise definition
of a $C^{\infty}$ vector field
requires the definition of the tangent bundle. We do not give this definition
since it is somewhat technical and does not contribute much to our discussion.}
 $X$ on $\cM$ is a smooth assignment of
tangent vectors to each point of $\cM$. We denote the set of vector
fields on $\cM$ as $\mathfrak{X}(\cM)$ and $X_p$ is the value of the
vector field $X$ at $p\in\cM$.  Given a function $f\in
C^{\infty}(\cM,\R)$ we define the action of $X\in \mathfrak{X}(\cM)$
on $f$ as
\[ Xf\in C^{\infty}(\cM,\R)  \qquad (Xf)(p) = X_p(f)\]
in accordance with our definition of tangent vectors as directional
derivatives of functions.

\subsection{Riemannian Manifolds}
A Riemannian manifold $(\cM,g)$ is a differentiable manifold $\cM$
equipped with a Riemannian metric $g$. The metric $g$ is defined by
a local inner product on tangent vectors
\[g_x(\cdot,\cdot):
T_x\cM\times T_x\cM\to\mathbb{R}, \quad x\in\cM\] that is symmetric,
bi-linear and positive definite\\
\begin{align*}
g_x(u,v)&=g_x(v,u)\\
g_x\left(\sum_{i=1}^n u_i,\sum_{i=1}^n v_i\right)&=\sum_{i=1}^n\sum_{j=1}^n g_x(u_i,v_j)\\
g_x(u,u) &\geq 0\\
g_x(u,u) &=0 \Leftrightarrow u=0
\end{align*}
and is also  $C^{\infty}$ differentiable in $x$. By the bi-linearity
of the inner product $g$, for every $u,v\in T_x\cM$
\[ g_x(v,u) = \sum_{i=1}^n\sum_{j=1}^n v_i u_j g_x(\partial_i,\partial_j)\]
and $g_x$ is completely described by
$\{g_x(\partial_i,\partial_j):1\leq i,j\leq n\}
$ -- the set of inner products between
the basis elements $\{\partial_i\}_{i=1}^n$ of $T_x\cM$.
The Gram matrix $[G(x)]_{ij}=g_x(\partial_i,\partial_j)$ is a symmetric and
positive definite matrix that completely describes the metric $g_x$.

\paragraph{}
The metric enables us to define lengths of
tangent vectors $v\in T_x \cM$
by $\sqrt{g_x(v,v)}$ and
lengths of curves
$\gamma:[a,b]\to\cM$
by
\[L(\gamma)=\int_a^b
    \sqrt{g_x(
        \dot{\gamma}(t),
        \dot{\gamma}(t))} %\|\dotp{\gamma}(t)\|
\text{d}t\]
 where $\dot{\gamma}(t)$ is the velocity
vector of the curve $\gamma$ at time $t$.
Using the above definition of lengths of curves, we can define the distance
$d_{g}(x,y)$
between two points $x,y\in\cM$ as
\[ d_{g}(x,y) = \inf_{\gamma\in \Gamma(x,y)} \int_a^b
\sqrt{g_x(
        \dot{\gamma}(t),
        \dot{\gamma}(t))} %\|\dotp{\gamma}(t)\|
\text{d}t\] where $\Gamma(x,y)$ is the set of piecewise
differentiable curves connecting $x$ and $y$. The distance $d_{g}$
is called geodesic distance and the minimal curve achieving it is
called a geodesic curve\footnote{It is also common to define
geodesics as curves satisfying certain differential equations. The
above definition, however, is more intuitive and appropriate for our
needs.}. Geodesic distance satisfies the usual requirements of a
distance and is compatible with the topological structure of $\cM$
as a topological manifold. If the manifold in question is clear from
the context, we will remove the subscript and use $d$ for the
geodesic distance.

\paragraph{}
A manifold is said to be geodesically complete if any geodesic curve
$c(t)$, $t\in [a,b]$, can be extended to be defined for all $t\in
\R$.  It can be shown, that the following are equivalent
\begin{itemize} \item $(\cM,g)$ is geodesically complete
\item $d_g$ is a complete metric on $\cM$ \item closed and bounded
subsets of $\cM$ are compact. \end{itemize} In particular, compact
manifolds are geodesically complete. The Hopf-Rinow theorem asserts
that if $\cM$ is geodesically complete, then any two points can be
joined by a geodesic.

\paragraph{}
Given two Riemannian manifolds $(\cM,g)$,
$(\cN,h)$ and a diffeomorphism
between them $f:\cM\to\cN$ we define
the push-forward and pull-back maps below\footnote{
The push-forward and pull-back maps may be defined more generally
using category theory as covariant and contravariant functors \cite{Lee2000}.}
\begin{defn}
The push-forward map $f_*:T_x\cM\to T_{f(x)}\cN$,
associated with the diffeomorphism $f:\cM\to\cN$ is
the mapping that satisfies
\[ v(r\circ f)=(f_*v)r, \qquad \forall r\in C^{\infty}(\cN,\mathbb{R}).\]
\end{defn}
The push-forward is none other than a coordinate free version of the Jacobian
matrix $J$  or the total derivative operator associated with the local
chart representation of $f$.
In other words, if we define the coordinate version of  $f:\cM\to\cN$
\[\tilde f=\phi\circ f\circ \psi^{-1}:\mathbb{R}^n\to\mathbb{R}^m\]
where $\phi,\psi$ are local charts of $\cN,\cM$
then the push-forward map is
\[
f_*u = Ju = \sum_i \left(\sum_j \frac{\partial \tilde f_i}{\partial x_j} u_j
\right) e_i
\]
where $J$ is the Jacobian of $\tilde f$ and
$\tilde f_i$ is the $i$-component function of
$\tilde f:\R^m\to\R^n$.
Intuitively, as illustrated in Figure~\ref{fig:push-forward},
the push-forward transforms  velocity vectors of curves
$\gamma$ to velocity vectors of transformed curves $f(\gamma)$.
\begin{figure}
\centering
{
\psfrag{m}{$\cM$}
\psfrag{n}{$\cN$}
\psfrag{f}{$f$}
\psfrag{p}{$T_x\cM$}
\psfrag{q}{$T_{f(x)}\cN$}
\includegraphics[scale=0.6]{RiemannianGeomFigures/push-forward.eps}
\caption{The map $f:\cM\to\cN$
defines a push forward map $f_*:T_x\cM\to T_{f(x)}\cN$
that transforms velocity vectors of curves to velocity vectors of
the transformed curves.}
\label{fig:push-forward}}
\end{figure}

\begin{defn} \label{def:pull-back}
Given  $(\cN,h)$ and a diffeomorphism $f:\cM\to\cN$
we define a metric $f^*h$
on $\cM$ called the pull-back metric by the relation
\[(f^* h)_x(u,v)=h_{f(x)}(f_*u,f_*v).\]
\end{defn}
\begin{defn}
An isometry is a diffeomorphism  $f:\cM\to \cN$
between two Riemannian manifolds $(\cM,g),(\cN,h)$
for which
\[g_x(u,v)=(f^*h)_x(u,v) \qquad \forall x\in\cM,\qquad
\forall u,v\in T_x\cM.\]
\end{defn}
Isometries, as defined above, identify two
Riemannian manifolds as identical in terms of their Riemannian structure.
Accordingly, isometries preserve all the geometric properties including the
geodesic distance function  $d_{g}(x,y)=d_h(f(x),f(y))$.
Note that the above definition of
an isometry is defined through the local metric
in contrast to the global definition of isometry in
other branches of mathematical analysis.

\paragraph{}
A smooth and Riemannian structure may be defined over a topological manifold
with a boundary as well. The definition is a relatively straightforward
extension using the notion of differentiability of maps between non-open sets
in $\mathbb{R}^n$  \cite{Lee2002}.
Manifolds with a boundary are important for integration and
boundary value problems; our use of them will be restricted to
Section~\ref{sec:diffusion}.

\paragraph{}
In the following section we discuss some previous work
and then proceed to examine manifolds of probability distributions
and their Fisher geometry.

%----------------------------------------------------------------
\section{Previous Work}\label{sec:previousWork}
Connections between statistics and Riemannian geometry have been
discovered and studied for over fifty years. We give below a roughly
chronological overview of this line of research. The overview
presented here is not exhaustive. It is intended to review some of
the influential research that is closely related to this thesis. For
more information on this research area consult the monographs
\cite{Murray1993,Kass1997,Amari2000}. Additional previous work that
is related to a specific section of this thesis will be discussed
inside that section.

\paragraph{}
\emcite{Rao1945} was the first to point out that a statistical
family can be considered as a Riemannian manifold with the metric
determined by the Fisher information quantity. \emcite{Efron1975}
found a relationship between the geometric curvature of a
statistical model and Fisher and Rao's theory of second order
efficiency. In this sense, a model with small curvature enjoys nice
asymptotic properties and a model with high curvature implies a
break-down of these properties. Since Efron's result marks a
historic breakthrough of geometry in statistics we describe it below
in some detail.

\paragraph{}
Recall that by Cram\'er-Rao lower bound, the variance of unbiased
estimators is bounded from below by the inverse Fisher information.
More precisely, in matrix form we have that $\forall \theta$,
$V(\theta)-G^{-1}(\theta)$ is positive semi-definite, where $V$ is
the covariance matrix of the estimator and $G$ is the Fisher
information matrix\footnote{The matrix form of the Cram\'er-Rao
lower bound may be written as $V(\theta)-G^{-1}(\theta)\succeq 0$
with equality $V(\theta)=G^{-1}(\theta)$ if the bound is attained.}.
Estimators that achieve this bound asymptotically, for all $\theta$,
are called asymptotically efficient, or first order efficient. The
prime example for such estimators, assuming some regularity
conditions, is the maximum likelihood estimator. The most common
example of statistical families for which the regularity conditions
hold is the exponential family. Furthermore, in this case, the MLE
is a sufficient statistic. These nice properties of efficiency and
sufficiency of the MLE do not hold in general for non-exponential
families.

\paragraph{}
The term second order efficiency was coined by Rao, and refers to a
subset of consistent and first order efficient estimators
$\hat\theta(x_1,\ldots,x_n)$ that attain equality in the following
general inequality
\begin{align} \label{eq:secOrder}
\lim_{n\to\infty}
n(i(\theta^{\text{true}})-i(\hat\theta(x_1,\ldots,x_n))) \geq
i(\theta^{\text{true}}) \gamma^2(\theta^{\text{true}})
\end{align}
where $i$ is the one-dimensional Fisher information and $\gamma$
some function that depends on $\theta$. Here $x_1,\ldots,x_n$ are
sampled iid from $p(x\,;\theta^{\text{true}})$ and
$\hat\theta(x_1,\ldots,x_n)$ is an estimate of
$\theta^{\text{true}}$ generated by the estimator $\hat\theta$. The
left hand side of \eqref{eq:secOrder} may be interpreted as the
asymptotic rate of the loss of information incurred by using the
estimated parameter rather than the true parameter. It turns out
that the only consistent asymptotically efficient estimator that
achieves equality in \eqref{eq:secOrder} is the MLE, thereby giving
it a preferred place among the class of first order efficient
estimators.

\paragraph{}
 The significance of Efron's result is that he identified the
function $\gamma$ in \eqref{eq:secOrder} as the Riemannian curvature
of the statistical manifold with respect to the exponential
connection. Under this connection, exponential families are flat and
their curvature is 0. For non-exponential families the curvature
$\gamma$ may be interpreted as measuring the breakdown of asymptotic
properties which is surprisingly similar to the interpretation of
curvature as measuring the deviation from flatness, expressing in
our case an exponential family. Furthermore, Efron showed that the
variance of the MLE in non-exponential families exceeds the
Cram\'er-Rao lower bound in approximate proportion to
$\gamma^2(\theta)$.


\paragraph{}
\emcite{Dawid1975} points out that Efron's notion of curvature is
based on a connection that is not the natural one with respect to
the Fisher geometry. A similar notion of curvature may be defined
for other connections, and in particular for the Riemannian
connection that is compatible with the Fisher information metric.
Dawid's comment about the possible statistical significance of
curvature with respect to the metric connection remains a largely
open question, although some results were obtained by
\emcite{Brody1998}. \emcite{Chentsov1982} introduced a family of
connections, later parameterized by $\alpha$, that include as
special cases the exponential connection and the metric connection.
Using Amari's parametrization of this family, $\alpha=1$ corresponds
to the exponential connection, $\alpha=0$ corresponds to the metric
connection and the $\alpha=-1$ corresponds to the mixture
connection, under which mixture families enjoy 0 curvature
\cite{Amari2000}.

\paragraph{}
\emcite{Chentsov1982} proved that the Fisher information metric is
the only Riemannian metric that is preserved under basic
probabilistic transformations. These transformations, called
congruent embeddings by a Markov morphism, represent transformations
of the event space that is equivalent to extracting a sufficient
statistic. Later on, \emcite{Campbell86} extended \v{C}encov's
result to non-normalized positive models, thus axiomatically
characterizing a set of metrics on the positive cone
$\mathbb{R}^n_{+}$.

\paragraph{}
In his short note \emcite{Dawid1977} extended these ideas to
infinite dimensional manifolds representing non-parametric sets of
densities. More rigorous studies include \cite{Lafferty1988} that
models the manifold of densities on a Hilbert space and
\cite{Pistone1995} that models the same manifold on non-reflexive
Banach spaces called Orlicz spaces. This latter approach, that does
not admit a Riemannian structure, was further extended by Pistone
and his collaborators and by \emcite{Grasselli2001}.


\paragraph{}
Additional research by Barndorff-Nielsen and others considered the
connection between geometry and statistics from a different angle.
Below is a brief description of some these results. In
\cite{Barndorff1986} the expected Fisher information is replaced
with the observed Fisher information to provide an alternative
geometry for a family of statistical models. The observed geometry
metric is useful in obtaining approximate expansion of the
distribution of the MLE, conditioned on an ancillary statistic. This
result continues previous research by Efron and Amari that provides
a geometric interpretation for various terms appearing in the
Edgeworth expansion of the distribution of the MLE for curved
exponential models. \emcite{Barndorff1983} studied the relation
between certain partitions of an exponential family of models and
geometric constructs. The partitions, termed affine dual foliations,
refers to a geometric variant of the standard division of $\R^n$
into copies of $\R^{n-1}$. Based on differential geometry,
\emcite{Barndorff1993} define a set of models called orthogeodesic
models that enjoy nice higher-order asymptotic properties.
Orthogeodesic models include exponential models with dual affine
foliations as well as transformation models and provides an abstract
unifying framework for such models.


\paragraph{}
The geodesic distance under the Fisher metric has been examined in
various statistical studies. It is used in statistical testing and
estimation as an alternative to Kullback Leibler or Jeffreys
divergence \cite{Atkinson1981}. It is also essentially equivalent to
the popular Hellinger distance \cite{Beran1977} that plays a strong
role in the field of statistical robustness
\cite{Tamura1986,Lindsay1994,Cutler1996}.

\paragraph{}
In a somewhat different line of research, \emcite{Csiszar1975}
studied the geometry of probability distributions through the notion
of $I$-divergence. In a later paper \emcite{Csiszar1991} showed that
$I$-divergence estimation, along with least squares, enjoy nice
axiomatic frameworks. While not a distance measure, the
$I$-divergence bears close connection to the geodesic distance under
the Fisher information metric \cite{Kullback1968}. This fact,
together with the prevalence of the $I$-divergence and its special
case the Kullback-Leibler divergence, brings these research
directions together under the umbrella of information geometry.
\emcite{Amari2000} contains some further details on the connection
between $I$-divergence and Fisher geometry.

\paragraph{}
Information geometry arrived somewhat later in the machine learning
literature. Most of the studies in this context were done by Amari's
group. Amari examines several geometric learning algorithms for
neural networks \cite{Amari1995} and shows how to adapt the gradient
descent algorithm to information geometry in the context of neural
networks \cite{Amari1998} and independent component analysis (ICA)
\cite{Amari1999}. \emcite{Ikeda2004} interprets several learning
algorithms in graphical models such as belief propagation using
information geometry and introduces new variations on them. Further
information on Amari's effort in different applications including
information theory and quantum estimation theory may be obtained
from \cite{Amari2000}. \emcite{Saul1997} interpolates between
different models based on differential geometric principles and
\emcite{Jaakkola1998} use the Fisher information to enhance a
discriminative classifier with generative qualities.
\emcite{Gous1998} and \emcite{Hall2000}  use information geometry to
represent text documents by affine subfamilies of multinomial
models.

\paragraph{}
In the next section we apply the mathematical framework developed in
Section~\ref{sec:RiemannianGeometry}
to manifolds of probability models.

%----------------------------------------------------------------
\section{Geometry of Spaces of Probability Models}\label{sec:probSimplexGeom}
Parametric inference in statistics is concerned with
a parametric family of distributions
$\{p(x\,;\theta):\theta\in\Theta\subset\mathbb{R}^n\}$
over the event space $\cX$.
If the parameter space $\Theta$ is a differentiable manifold
and the mapping $\theta\mapsto p(x\,;\theta)$ is a diffeomorphism
we can identify statistical models in the family
as points on the manifold $\Theta$.
The Fisher information matrix $E\{s s^{\top}\}$ where
$s$ is the gradient of the score
$[s]_i=\partial \log p(x\,;\theta)/\partial \theta_i$
may be used to endow $\Theta$ with the following Riemannian metric
\begin{align} \nonumber
\cJ_{\theta}(u,v) &\deff \sum_{i,j}u_i v_j
\int p(x\,;\theta) \frac{\partial}{\partial \theta_i} \log p(x\,;\theta)
\frac{\partial}{\partial \theta_j} \log p(x\,;\theta) \text{d}x\\
&=  \sum_{i,j}u_i v_j \,\, E\left\{\frac{\partial \log p(x\,;\theta)}{\partial\theta_i}
\frac{\partial \log p(x\,;\theta)}{\partial\theta_j}\right\}.\label{eq:FisherInformationMetric}
\end{align}
If $\cX$ is discrete the above integral is replaced with a sum.
An equivalent form of \eqref{eq:FisherInformationMetric}
for normalized distributions that is sometimes
easier to compute is
\begin{align} \nonumber
 \cJ_{\theta}(u,v) &=  \cJ_{\theta}(u,v) - \sum_{ij} u_i v_j
\frac{\partial^2}{\partial\theta_i\partial\theta_j}
\int  p(x\,; \theta)  \dd d x \\ \nonumber
&= \sum_{ij} u_i v_j\int p(x\,;\theta)
\left(
\left(\frac{1}{p(x\given \theta)}
\frac{\partial p(x\,; \theta)}{\partial \theta_j}\right)
\left(\frac{1}{p(x\given \theta)}
\frac{\partial p(x\,; \theta)}{\partial \theta_i}\right)
-\frac{1}{p(x\,;\theta)}
\frac{\partial^2}{\partial\theta_i\partial\theta_j} p(x\,; \theta)
\right)
\, \dd d x \\ \nonumber
&= -\sum_{ij} u_i v_j\int p(x\,;\theta)
\frac{\partial }{\partial \theta_j}\frac{1}{p(x\given \theta)}\frac{\partial p(x\,; \theta)}{\partial \theta_i}  \, \dd d x \\
&=-\sum_{ij} u_i v_j\int p(x\,;\theta) \frac{\partial^2 }{\partial
\theta_j \partial \theta_i}\log p(x\,; \theta)  \, \dd d x\nonumber \\
&=\sum_{ij} u_i v_j E\left\{-\frac{\partial^2 }{\partial \theta_j
\partial \theta_i}\log p(x\,; \theta)\right\}
\label{eq:secDerivFish}
\end{align}
assuming that we can change the order of
the integration and differentiation operators.

\paragraph{}
In the remainder of this section
we examine in detail a few important Fisher geometries.
The Fisher geometries of finite dimensional non-parametric
space, finite dimensional conditional non-parametric space
and spherical normal space are studied next.

\subsection{Geometry of Non-Parametric Spaces}
In the finite non-parametric setting, the event space $\cX$ is a
finite set with $|\cX|=n$ and $\Theta=\P_{n-1}$, defined in
\eqref{eq:simplexDef}, which represents the manifold of all positive
probability models over $\cX$. The positivity constraint is
necessary for $\Theta=\P_{n-1}$ to be a manifold. If zero
probabilities are admitted, the appropriate framework for the
parameter space $\Theta=\overline{\P_{n-1}}$ is a manifold with
corners \cite{Lee2002}. Note that the above space $\Theta$ is
precisely the parametric space of the multinomial family. Hence, the
results of this section may be interpreted with respect to the space
of all positive distributions on a finite event space, or with
respect to the parametric space of the multinomial distribution.


\paragraph{}
The finiteness of $\cX$ is necessary for $\Theta$ to be a finite
dimensional manifold. Relaxing the finiteness assumption results in
a manifold where each neighborhood is homeomorphic to an infinite
dimensional vector space called the model space. Such manifolds are
called Frechet, Banach or Hilbert manifolds (depending on the model
space) and are  the topic of a branch of geometry called global
analysis \cite{Lang1999}. \emcite{Dawid1977} remarked that an
infinite dimensional non-parametric space may be endowed with
multinomial geometry leading to spherical geometry on a Hilbert
manifold. More rigorous modeling attempts were made by
\emcite{Lafferty1988} that models the manifold of densities on a
Hilbert space and by \emcite{Pistone1995} that model it on a
non-reflexive Banach space. See also the brief discussion on
infinite dimensional manifolds representing densities by
\emcite{Amari2000} pp. 44-45.



\paragraph{}
Considering $\mathbb{P}_{n-1}$ as a submanifold of $\mathbb{R}^n$, we
represent tangent vectors $v\in T_{\theta}\mathbb{P}_{n-1}$
in the standard basis of $T_{\theta}\mathbb{R}^{n}$.
As mentioned earlier \eqref{eq:simplexTangentSpace}, this
results in the following representation of $v\in T_{\theta}\mathbb{P}_{n-1}$
\[v=\sum_{i=1}^n v_i\partial_i \quad \text{ subject to } \quad \sum_{i=1}^n v_i=0.\]
Using this representation, the loglikelihood and its derivatives are
\begin{align*}
\log p(x\,; \theta) &= \sum_{i=1}^{n} x_i \log \theta_i \\
\frac{\partial \log p(x\,; \theta)}{\partial \theta_i}
&= \frac{x_i}{\theta_i} \\
\frac{\partial^2 \log p(x\,; \theta)}{\partial \theta_i\partial\theta_j}
 &= -\frac{x_i}{\theta_i^2}\,\delta_{ij}
\end{align*}
and using equation  \eqref{eq:secDerivFish}
 the Fisher information metric on $\P_{n-1}$ becomes
\begin{align*}
\cJ_{\theta}(u,v) &= - \sum_{i=1}^{n}\sum_{j=1}^{n} u_i v_j E\left[
\frac{\partial^2 \log p(x\given \theta)} {\partial \theta_i\partial
\theta_j} \right] = - \sum_{i=1}^{n} u_i v_i  E\left\{
-x_i/\theta_i^2\right\}  = \sum_{i=1}^{n} \frac{u_i v_i}{\theta_i}
\end{align*}
since $E x_i=\theta_i$. Note that the Fisher metric emphasizes
coordinates that correspond to low probabilities. The fact that the
metric $\cJ_{\theta}(u,v)\to \infty$ when $\theta_i\to 0$ is not
problematic since length of curves that involves integrals over $g$
converge.


\paragraph{}
While geodesic distances are difficult to compute in general,
in the present case we can easily compute the geodesics by observing
that the standard Euclidean metric on the surface of the positive $n$-sphere
is the pull-back of the Fisher information metric on the simplex.
More precisely, the transformation
$F(\theta_1,\ldots,\theta_{n+1})=(2\sqrt{\theta_1},\ldots,2\sqrt{\theta_{n+1}})$
is a diffeomorphism of the $n$-simplex $\Pn$ onto the
positive portion of the $n$-sphere of radius~2
\[\tilde{\mathbb{S}}_{+}^n = \left\{ \theta\in\R^{n+1} \;:\;
\sum_{i=1}^{n+1}\theta_i^2=4,\; \theta_i > 0 \right\}.\]
The inverse transformation is
\[F^{-1}:\tilde{\mathbb{S}}_{+}^n\to\Pn,
\qquad F^{-1}(\theta)=\left(\frac{\theta_1^2}{4},\ldots,\frac{\theta_{n+1}^2}{4}\right)\]
and its push-forward is
\[ F^{-1}_*(u) = \left(\frac{u_1}{2},\ldots,\frac{u_{n+1}}{2}\right).\]
The metric on $\tilde{\mathbb{S}}_{+}^n$ obtained
by pulling back the Fisher information on $\Pn$ through $F^{-1}$ is
\begin{align*}
h_{\theta}(u,v)&=
 \cJ_{\theta^2/4}\left(F^{-1}_*\sum_{k=1}^{n+1} u_k e_k,F^{-1}_*\sum_{l=1}^{n+1} v_l e_l\right)
=\sum_{k=1}^{n+1}\sum_{l=1}^{n+1} u_k v_l\, g_{\theta^2/4}(F^{-1}_*e_k,F^{-1}_*e_l)\\
&= \sum_{k=1}^{n+1} \sum_{l=1}^{n+1}  u_k v_l \sum_i \frac{4}{\theta_i^2}\,
(F_*^{-1}e_k)_i\,(F_*^{-1}e_l)_i
= \sum_{k=1}^{n+1}\sum_{l=1}^{n+1}  u_k  v_l \sum_i
\frac{4}{\theta_i^{2}}\,\frac{\theta_k \delta_{ki}}{2}\,
\frac{\theta_l\delta_{li}}{2}
=\sum_{i=1}^{n+1} u_i v_i\\
&= \delta_{\theta}(u,v)
\end{align*}
the Euclidean metric on $\tilde{\mathbb{S}}_{+}^n$
inherited from the embedding Euclidean space $\R^{n+1}$.


\paragraph{}
Since the transformation $F:(\Pn,\cJ)\to(\tilde{\mathbb{S}}_{+}^n,\delta)$
is an isometry,  the geodesic distance $d_{\cJ}(\theta,\theta')$ on
$\Pn$ may be
computed as the shortest curve on $\tilde{\mathbb{S}}_{+}^n$
connecting $F(\theta)$ and $F(\theta')$.
These shortest curves are  portions of great circles -- the intersection of a
two dimensional subspace and
$\tilde{\mathbb{S}}_{+}^n$ whose lengths are
\begin{equation}
\label{eq:arccos} d_{\cJ}(\theta,\theta') =
d_{\delta}(F(\theta),F(\theta')) = 2 \arccos \left(\sum_{i=1}^{n+1}
\sqrt{\theta_i\, \theta'_i}\right)\,.
\end{equation}
We illustrate these geodesic distances in Figures~\ref{fig:vis}-\ref{fig:equalDist2}.
Figure~\ref{fig:vis} shows how to picture $\P_2$ as a triangle in $\R^2$
and Figure~\ref{fig:equalDist2} shows the equal distant contours
for both Euclidean and Fisher geometries.
Will often ignore the factor of 2 in \eqref{eq:arccos} to obtain
a more compact notation for the geodesic distance.

\begin{figure}
\begin{center}
{\psfrag{x}{$(1,0,0)$}
\psfrag{y}{$(0,1,0)$}
\psfrag{z}{$(0,0,1)$}
\includegraphics[scale=0.6]{probSpacesFigures/simplex2D-3D.eps}}
\end{center}
\caption{The 2-simplex $\P_2$ may be visualized as a surface in
$\R^3$ (left) or as a triangle in $\R^2$ (right).}
\label{fig:vis}
\end{figure}


\begin{figure}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=0.4]{diffusionFigures/heat1}&
\includegraphics[scale=0.4]{diffusionFigures/rbf1}\\[16pt]
\includegraphics[scale=0.4]{diffusionFigures/heat2}&
\includegraphics[scale=0.4]{diffusionFigures/rbf2}\\[16pt]
\includegraphics[scale=0.4]{diffusionFigures/heat3} &
\includegraphics[scale=0.4]{diffusionFigures/rbf3}
\end{tabular}
\end{center}
\caption{Equal distance contours on $\mathbb{P}_2$ from the upper right edge (top row),
the center (center row), and lower right corner (bottom row). The distances are computed
using the Fisher information metric (left column) or the Euclidean metric (right column).}
\label{fig:equalDist2}
\end{figure}

\paragraph{}
The geodesic distances $d_{\cJ}(\theta,\theta')$ under the Fisher geometry and the Kullback-Leibler
divergence $D(\theta,\theta')$ agree up to second order as
$\theta\to\theta'$ \cite{Kullback1968}.
Similarly, the Hellinger distance \cite{Beran1977}
\begin{equation}
d_H(\theta,\theta') = \sqrt{\sum_i \left( \sqrt{\theta_i} - \sqrt{\theta'_i}\right)^2}
\end{equation}
is related to $d_{\cJ}(\theta,\theta')$ by
\begin{equation}
d_H(\theta,\theta') = 2 \sin\left(d_{\cJ}(\theta,\theta')/4\right)
\end{equation}
and thus also agrees with the distance up to second order
as $\theta'\rightarrow \theta$.



\subsection{Geometry of Non-Parametric Conditional Spaces}
Given two finite event sets $\cX,\cY$ of sizes
$k$ and $m$ respectively, a
conditional probability model $p(y|x)$ reduces to an element
of $\mathbb{P}_{m-1}$ for each $x\in\cX$.
We may thus identify the space of conditional probability models
associated with $\cX$ and $\cY$ as the product space
\[\P_{m-1}\times\cdots\times\P_{m-1}=\P_{m-1}^{k}.\]

\paragraph{}
For our purposes, it will be sometimes more convenient to work with
the more general case of positive non-normalized conditional models.
Dropping the normalization constraints $\sum_i p(y_i|x_j)=1$
we obtain conditional models
in the cone of $k\times m$ matrices with positive entries,
denoted by $\mathbb{R}_{+}^{k\times m}$.
Since a normalized conditional model is also a non-normalized one,
we can consider $\P_{m-1}^{k}$ to be a subset of $\mathbb{R}_{+}^{k \times m}$.
Results obtained for non-normalized models apply then to normalized models
as a special case. In addition, some of the notation and formulation
is simplified by working with non-normalized models.

\paragraph{}
In the interest of simplicity,
we will often use matrix notation instead of the standard probabilistic
notation.
A conditional model (either normalized or non-normalized) is described
by a positive matrix $M$ such that $M_{ij}=p(y_j|x_i)$.
Matrices that correspond to normalized models are (row) stochastic matrices.
We denote tangent vectors to
$\mathbb{R}_{+}^{k\times m}$ using the standard basis
\[T_M\mathbb{R}_{+}^{k\times m}=\text{span}
\{\partial_{ij}:i=1,\ldots,k, j=1,\ldots,m\}.\] Tangent vectors to
$\P_{m-1}^{k}$, when expressed using the basis of the embedding
tangent space $T_M\mathbb{R}_{+}^{k\times m}$ are linear
combinations of $\{\partial_{ij}\}$ such that the sums of the
combination coefficients over each row are 0, e.g.
\begin{align*}
\frac{1}{2}\partial_{11}+\frac{1}{2}\partial_{12}-\partial_{13}+\frac{1}{3}\partial_{21}-\frac{1}{3}
\partial_{22}&\in T_M\P_{2}^{3}\\
\frac{1}{2}\partial_{11}+\frac{1}{2}\partial_{12}-\partial_{13}+\frac{1}{3}\partial_{21}-\frac{1}{2}
\partial_{22}&\not\in T_M\P_{2}^{3}.
\end{align*}

\paragraph{}
The identification of the space of conditional models as a product
of simplexes demonstrates the topological and differentiable
structure. In particular, we do not assume that the metric has a
product form. However, it is instructive to consider as a special
case the product Fisher information metric on $\P_{n-1}^k$ and
$\mathbb{R}_{+}^{k\times m}$. Using the above representation of
tangent vectors $u,v\in T_M\mathbb{R}_{+}^{k\times m}$ or $u,v\in
T_M\mathbb{P}_{m-1}^{k}$ the product Fisher information
\begin{align*}
\cJ^k_M(u_1\oplus\cdots\oplus u_k,v_1\oplus\cdots\oplus v_k)&\deff
(\cJ\otimes \cdots \otimes\cJ)_M(u_1\oplus\cdots\oplus
u_k,v_1\oplus\cdots\oplus v_k)\\
 &\deff \sum_{i=1}^k
\cJ_{[M]_i}(u_i,v_i),
\end{align*} where $[A]_i$ is the
$i$-row of the matrix $A$, $\otimes$ is the tensor product and
$\oplus$ is the direct sum decomposition of vectors, reduces to
\begin{align}\label{eq:productFisherMetric1}
\cJ^k_M(u,v) &= \sum_{i=1}^k\sum_{j=1}^m \frac{u_{ij}
v_{ij}}{M_{ij}}.
\end{align}
A different way of expressing \eqref{eq:productFisherMetric1} is by
specifying the values of the metric on pairs of basis elements
\begin{align}\label{eq:productFisherMetric2}
g_M(\partial_{ab},\partial_{cd})
&=\delta_{ac}\delta_{bd}\frac{1}{M_{ab}}
\end{align}
where $\delta_{ab}=1$ if $a=b$ and 0 otherwise.

%----------------------
\subsection{Geometry of Spherical Normal Spaces}
\label{sec:geomSphericalNormal}
Given a restricted parametric family
$\Theta\subset \Pn$ the Fisher information metric on $\Theta$ agrees
with the induced metric from the Fisher information metric on $\Pn$.
When $\cX$ is infinite and $\Theta$ is a finite dimensional
parametric family, we can still define the Fisher information metric
$\cJ$ on $\Theta$, however without a reference to an embedding
non-parametric space. We use this approach to consider the Fisher
geometry of the spherical normal distributions on $\cX=\R^{n-1}$
\[ \{\cN(\mu,\sigma I): \mu\in\R^{n-1},\sigma\in \R_{+}\}\]
parameterized by the upper half plane $\Theta=\H^n\cong
\R^{n-1}\times \R_{+}$.

\paragraph{}
To compute the Fisher information metric for this family, it is
convenient to use the  expression given by equation
\eqref{eq:secDerivFish}. Then simple calculations yield, for $1 \leq
i,j \leq n-1$
\begin{align*}
[G(\theta)]_{ij} &= -\int_{\R^{n-1}} \frac{\partial^2}{\partial
{\mu_i}\partial {\mu_j}}
\left( - \sum_{k=1}^{n-1} \frac{(x_k-\mu_k)^2}{2\sigma^2}\right) p(x\given\theta) \, dx = \frac{1}{\sigma^2} \,\delta_{ij} \\
[G(\theta)]_{ni} &= -\int_{\R^{n-1}} \frac{\partial^2}{\partial
\sigma\partial {\mu_i}} \left( - \sum_{k=1}^{n-1}
\frac{(x_k-\mu_k)^2}{2\sigma^2}\right) p(x\given\theta) \, dx
= \frac{2}{\sigma^3} \int_{\R^{n-1}} (x_i - \mu_i) \, p(x\given\theta)\, dx = 0 \\
[G(\theta)]_{nn} &= -\int_{\R^{n-1}} \frac{\partial^2}{\partial
\sigma\partial \sigma} \left( - \sum_{k=1}^{n-1}
\frac{(x_k-\mu_k)^2}{2\sigma^2}
   - (n-1)\log \sigma\right) p(x\given\theta) \, dx \\
&= \frac{3}{\sigma^4} \int_{\R^{n-1}} \sum_{k=1}^{n-1}(x_k - \mu_k)^2 \, p(x\given\theta)\, dx
- \frac{n-1}{\sigma^2} = \frac{2(n-1)}{\sigma^2}.
\end{align*}
\paragraph{}
Letting $\theta'$ be new coordinates defined by $\theta'_i = \mu_i$ for $1\leq i\leq n-1$
and $\theta'_n = \sqrt{2(n-1)}\, \sigma$, we see that the Gram
matrix is given by
\begin{equation}
[G(\theta')]_{ij} = \frac{1}{\sigma^2} \,\delta_{ij}
\end{equation}
and the Fisher information metric gives
the spherical normal manifold a hyperbolic geometry\footnote{The
manifold $\H^n$ with with hyperbolic geometry is often referred to as Poincar\'{e}'s
upper half plane and is a space of constant negative curvature.}.
It is shown by \emcite{Kass1997} that
any location-scale family of densities
\begin{equation*}
p(x\,; \mu,\sigma) = \frac{1}{\sigma} f\left(\frac{x-\mu}{\sigma}\right)
\quad (\mu,\sigma)\in\R\times\R_+ \quad f:\R\rightarrow \R
\end{equation*}
have a similar hyperbolic geometry. The geodesic curves in the two
dimensional hyperbolic space are circles whose centers lie on the
line $x_2=0$ or vertical lines (considered as circles whose centers
lie on the line $x_2=0$ with infinite radius) \cite{Lee1997}. An
illustration of these curves appear in
Figure~\ref{fig:hypGeodesics}.
\begin{figure}
\begin{center}
\includegraphics[scale=0.5]{probSpacesFigures/hyperbolic-Geodesics.eps}
\caption{
Geodesic curves in $\H^2$ with the hyperbolic metric are circles
whose centers lie on the line $x_2=0$ or vertical lines.}
\label{fig:hypGeodesics}
\end{center}
\end{figure}
To compute the geodesic distance on $\H^n$ we transform points in
$\H^n$ to an isometric manifold known as Poincar\'e's ball. We first
define the sphere inversion of $x$ with respect to a sphere $S$ with
center $a$ and radius $r$ as
\[I_S(x)=\frac{r^2}{\|x-a\|^2}(x-a)+a.\]
The Cayley transform is the sphere inversion with respect to a
sphere with center $(0,\ldots,0,-1)$ and radius $\sqrt{2}$. We
denote by $\eta$ the inverse of the Cayley's transform that maps the
hyperbolic half plane to Poincar\'e's ball.
\[
\eta(x) = -I_{S'}(-x)\qquad x\in\H^n
\]
where $S'$ is a sphere with center at $(0,\ldots,0,1)$ and radius
$\sqrt{2}$.  The geodesic distance in $\H^n$ is then given by
\begin{align} \label{eq:hypDist}
d(x,y) &= \text{acosh}\,
\left(1+2\frac{\|\eta(x)-\eta(y)\|^2}{(1-\|\eta(x)\|^2)(1-\|\eta(y)\|^2)}\right)
\qquad x,y\in\H^n.
\end{align}
For more details see \cite{Bridson1999} pages 86--90.

\paragraph{}
The following sections describe the main contributions of the
thesis. The next two sections deal with the geometry of the model
space $\Theta$ in the context of estimation of conditional models.
The later sections study the geometry of the data space $\cX$.

%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Geometry of Conditional Exponential Models and AdaBoost}
\label{sec:boostMl}

Several recent papers in statistics and machine learning have been
devoted to the relationship between boosting and more standard
statistical procedures such as logistic regression.  In spite of this
activity, an easy-to-understand and clean connection between these
different techniques has not emerged.
\emcite{Friedman2000} note
the similarity between boosting and stepwise logistic regression
procedures, and suggest a least-squares alternative, but view the loss
functions of the two problems as different, leaving the precise
relationship between boosting and maximum likelihood unresolved.
\emcite{Kivinen1999} note that boosting is a form of ``entropy
projection,'' and \emcite{Lafferty1999} suggests the use of
Bregman distances to approximate the exponential loss.  \emcite{Mason:99}
consider boosting algorithms as functional gradient descent
and \emcite{Duffy+Helmbold:00} study various loss functions
with respect to the PAC boosting property.
More recently, \emcite{Collins2002} show how
different Bregman distances precisely account for boosting and logistic
regression, and use this framework to give the first convergence proof
of AdaBoost.  However, in this work the two methods
are viewed as minimizing different loss functions.
Moreover, the optimization problems are formulated in terms
of a reference distribution consisting of the zero vector, rather than
the empirical distribution of the data, making the interpretation
of this use of Bregman distances problematic from a statistical point of view.

\paragraph{}
In this section we present a very basic connection between boosting
and maximum likelihood for exponential models through a simple convex
optimization problem.  In this setting, it is seen that the only
difference between AdaBoost and maximum likelihood for exponential
models, in particular logistic regression, is that the latter requires
the model to be normalized to form a probability distribution.  The
two methods minimize the same $I$-divergence objective
function subject to the same feature constraints.  Using information
geometry, we show that projecting the exponential loss model onto the
simplex of conditional probability distributions gives precisely the
maximum likelihood exponential model with the specified sufficient
statistics.  In many cases of practical interest, the resulting models
will be identical; in particular, as the number of features increases
to fit the training data the two methods will give the same
classifiers.
We note that throughout the thesis we view boosting as a
procedure for minimizing the exponential loss, using either parallel
or sequential update algorithms as presented by  \emcite{Collins2002}, rather than
as a forward stepwise procedure as presented by \emcite{Friedman2000}
and \emcite{Freund1996}.

\paragraph{}
Given the recent interest in these techniques, it is striking that this connection has
gone unobserved until now.  However in general, there may be many ways
of writing the constraints for a convex optimization problem, and many
different settings of the Lagrange multipliers
that represent identical solutions.  The key to the
connection we present here lies in the use of a particular
non-standard presentation of the constraints.  When viewed in this
way, there is no need for special-purpose Bregman divergence as in \cite{Collins2002}
to give a unified account of boosting and maximum likelihood, and we only make
use of the standard $I$-divergence.  But our analysis
gives more than a formal framework for understanding old algorithms;
it also leads to new algorithms for regularizing AdaBoost, which is
required when the training data is noisy.  In particular, we derive a
regularization procedure for AdaBoost that directly corresponds to
penalized maximum likelihood using a Gaussian prior.  Experiments on
UCI data support our theoretical analysis, demonstrate the
effectiveness of the new regularization method, and give further
insight into the relationship between boosting and maximum likelihood
exponential models.

\paragraph{}
The next section describes an axiomatic characterization of metrics over
the space of conditional models and the relationship between the characterized
metric and the $I$-divergence. In this sense this section and the next one should
be viewed as one unit as they provide an axiomatic characterization
of the geometry underlying conditional exponential models such as logistic
regression and AdaBoost.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Definitions}
Let $\cX$ and $\cY$ be finite sets of sizes $k$ and $m$
and $\P_{m-1}^k,\mathbb{R}_{+}^{k\times m}$
be the sets of normalized and non-normalized conditional models
as defined in Section~\ref{sec:probSimplexGeom}.
Their closures $\overline{\P_{m-1}^k},\overline{\mathbb{R}_{+}^{k\times m}}$
represent the sets of non-negative conditional models.
Let
\[f=(f_1,\ldots,f_l), \qquad f_j:\cX\times \cY \to \R\]
be a sequence functions, which we
will refer to as a feature vector.  These functions
correspond to the weak learners in boosting,
and to the sufficient statistics in an exponential model.
The empirical distribution associated with
a training set  $\{(x_i,y_i)\}_{i=1}^n$ is
${\tilde{p}}(x,y) = \frac{1}{n} \sum_{i=1}^{n} \delta_{x_i, x} \, \delta_{y_i, y}.$
Based on ${\tilde{p}}(x,y)$ we define marginal and conditional
distributions ${\tilde{p}}(x),{\tilde{p}}(y|x)$ as usual.
We assume that ${\tilde{p}}(x)>0$ and that for all $x$
there is a unique $y\in\cY$, denoted by $\tilde{y}(x)$,
for which  ${\tilde{p}}(y\given x) > 0$.
This assumption, referred to as the consistent data assumption,
is made to obtain notation that corresponds to the conventions used to
present boosting algorithms; it is not essential to the correspondence
between AdaBoost and conditional exponential models presented here.
We will use the notation $f(x,y)$ to represent the
real vector $(f_1(x,y),\ldots,f_l(x,y))$
and $\langle \cdot,\cdot\rangle$ to be the usual scalar or dot product between
real vectors.

\paragraph{}
The conditional exponential model $q(y\given x\,;\theta)$ associated
with the feature vector $f$ is defined by
\begin{equation}
q(y\given x\,;\theta) = \frac{e^{\dotp{\theta}{f(x,y)}}}
{\sum_{y} e^{\dotp{\theta}{f(x,y)}}} \qquad \theta\in\R^l
\end{equation}
where $\langle \cdot,\cdot\rangle$ denotes the standard scalar product between
two vectors.
The maximum likelihood estimation problem is
to determine a parameter vector $\theta$ that maximize the conditional log-likelihood
\[\ell(\theta) \deff \sum_{x,y} {\tilde{p}}(x,y) \log q(y\given x\,;\theta).\]


\paragraph{}
The objective function to be minimized in the multi-label boosting algorithm
AdaBoost.M2 \cite{Collins2002}
is the exponential loss given by
\begin{equation}
\cE(\theta) \deff \sum_{i=1}^n \sum_{y\neq y_i}
e^{\,\dotp{\theta}{f(x_i, y) - f(x_i, y_i)}}.
\end{equation}
In the binary case $\cY=\{-1,+1\}$ and taking $f_j(x,y) = \frac{1}{2} y f_j(x)$
the exponential loss becomes
\begin{equation}
\cE(\theta) = \sum_{i=1}^n e^{-y_i \dotp{\theta}{f(x_i)}},
\label{eq:exploss}
\end{equation}
the conditional exponential model becomes the logistic model
\begin{equation}
q(y\given x\,;\theta) \seq \frac{1}{1+e^{-y\dotp{\theta}{f(x)}}},
\end{equation}
for which the maximum likelihood problem becomes equivalent
to minimizing the logistic loss function
\begin{equation}
-\ell(\theta)=\sum_{i=1}^n \log\left(1+e^{-y_i \dotp{\theta}{f(x_i)}}
\right).
\label{eq:logloss}
\end{equation}
As has been often noted, the log-loss (\ref{eq:logloss})
and the exponential loss (\ref{eq:exploss})
are qualitatively different.  The exponential loss (\ref{eq:exploss})
grows exponentially with increasing negative
margin $-y \dotp{\theta}{f(x)}$, while the log-loss grows linearly.


%-------------------------------------------------------------------------------
\subsection{Correspondence Between AdaBoost and Maximum Likelihood}
We define the conditional $I$-divergence with respect
to a distribution $r$ over $\mathcal{X}$ as
\begin{equation} \label{eq:I-div}
D_{r}(p, q) \deff \sum_{x} r(x) \sum_y \left(
p(y\given x) \log
\frac{p(y\given x)}{q(y\given x)} - p(y\given x) + q(y\given x) \right).
\end{equation}
It is a non-negative measure of discrepancy between two conditional models
$p,q\in\overline{\R_{+}^{k\times m}}$. If
$q\not\in \R_{+}^{k\times m}$, $D_r(p,q)$ may be $\infty$.
The $I$ divergence is not a formal distance function as it does not satisfy
symmetry and the triangle inequality.
In this section we will always take $r(x)={\tilde{p}}(x)$ and hence we omit
it from the notation and write $D(p,q)=D_{{\tilde{p}}}(p,q)$.
For normalized conditional models, the $I$-divergence $D(p,q)$
is equal to the Kullback-Leibler divergence \cite{Kullback1968}.
The formula presented here \eqref{eq:I-div} is a straightforward
adaptation of the non-conditional form of the $I$-divergence studied by
\emcite{Csiszar1991}.
The $I$-divergence comes up in many applications of statistics and machine learning.
See \cite{Kullback1968,OSullivan:98,Amari2000} for
many examples of such connections.

\paragraph{}
We define the feasible set $\cF({\tilde{p}}, f)$ of conditional models
associated with $f=(f_1,\ldots,f_l)$ and an empirical distribution
${\tilde{p}}(x,y)$ as
\begin{equation}
\label{eq:constraints}
\cF({\tilde{p}},f) \deff \left\{ p\in\overline{\R_{+}^{k\times m}} :
\sum_x {\tilde{p}}(x) \sum_{y} p(y\given x)\, \left(f_j(x,y) - E_{{\tilde{p}}}[f_j\given x]\right)
= 0,\; j=1,\ldots,l\right\}.
\end{equation}
Note that this set is non-empty since ${\tilde{p}}\in\cF({\tilde{p}},f)$ and
that under the consistent data assumption
$E_{{\tilde{p}}}[f\given x] = f(x,\tilde{y}(x))$.
The feasible set represents conditional models that agree with ${\tilde{p}}$
on the conditional expectation of the features.


\paragraph{}
Consider now the following two convex optimization problems, labeled $P_1$ and $P_2$.
\begin{center}
\begin{tabular}{cc}
\begin{tabular}{rrl}
$(P_1)$ & {\it minimize} & $D(p,q_{0})$ \\[2pt]
        & {\it subject to} & $p \in \cF({\tilde{p}}, f)$ \\
        & &
\end{tabular}
&\hskip20pt
\begin{tabular}{rrl}
$(P_2)$ & {\it minimize} & $D(p,q_{0})$ \\[2pt]
        & {\it subject to} & $p \in \cF({\tilde{p}}, f)$ \\
        &                  & $p \in \overline{\P_{m-1}^k}$.
\end{tabular}
\end{tabular}
\end{center}
Thus, problem $P_2$ differs from $P_1$ only in that the solution
is required to be normalized.  As we will show, the dual problem
$P_1^*$ corresponds to AdaBoost,
and the dual problem
$P_2^*$ corresponds to maximum likelihood for exponential models.

\paragraph{}
This presentation of the constraints is the key to making the
correspondence between AdaBoost and maximum likelihood. The
constraint $\sum_{x}{\tilde{p}}(x)\sum_y p(y\given x)\, f(x,y) =
E_{{\tilde{p}}}[f]$, which is the usual presentation of the
constraints for maximum likelihood (as dual to maximum entropy),
doesn't make sense for non-normalized models, since the two sides of
the equation may not be on the same scale. Note further that
attempting to re-scale by dividing by the mass of $p$ to get
\[\sum_{x}{\tilde{p}}(x)\frac{\sum_y p(y\given x)\, f(x,y)}{\sum_y p(y\given x)}
= E_{{\tilde{p}}}[f]\] would yield nonlinear constraints.

\paragraph{}
Before we continue,
we recall the dual formulation from convex optimization. For mode details
refer for example to Section 5 of \cite{Boyd2004}.
Given a convex optimization problem
\begin{align}\label{eq:primalProb}
\min_{x\in\R^n} f_0(x) \quad \text{subject to} \quad h_i(x)=0 \quad i=1,\ldots,r
\end{align}
the Lagrangian is defined as
\begin{align} \label{eq:Lagrangian}
\cL(x,\theta)\deff f_0(x)-\sum_{i=1}^r \theta_i h_i(x).
\end{align}
The vector $\theta\in\mathbb{R}^r$ is called the dual variable
or the Lagrange multiplier
vector. The Lagrange dual function is defined as $h(\theta)=\inf_x \cL(x,\theta)$
and the dual problem of the original problem \eqref{eq:primalProb},
is $\max_{\theta} h(\theta)$.
The dual problem and the original problem, called the primal problem, are
equivalent to each other and typically, the easier of the two problems
is solved. Both problems are useful, however, as they provide alternative
views of the optimization  problem.


\subsubsection{The Dual Problem $(P_1^*)$}
Applying the above definitions to $(P_1)$,
and noting that the term $q(y\given x)$ in \eqref{eq:I-div} does not play a role
in the minimization problem, the Lagrangian is
\begin{align*}
\cL_1(p,\theta) &= \sum_x {\tilde{p}}(x) \sum_y p(y\given x)
\left(\log \frac{p(y\given x)}{q_{0}(y\given x)} - 1\right)
- \sum_i \theta_i \sum_x {\tilde{p}}(x) \sum_y p(y\given x)  (f_i(x,y) - E_{{\tilde{p}}}[f_i\given x])\\
&= \sum_x {\tilde{p}}(x) \sum_y p(y\given x)
\left(\log \frac{p(y\given x)}{q_{0}(y\given x)} - 1 -
\dotp{\theta}{f(x,y) - E_{{\tilde{p}}}[f\given x]}\right).
\end{align*}
The first step is to minimize the Lagrangian with respect to $p$, which will allow
us to express the dual function.
Equating the partial derivatives
$\frac{\partial \cL_1(p,\theta)}{\partial p(y\given x)}$  to 0 gives
\begin{align*}
0&= {\tilde{p}}(x)\left(\log \frac{p(y\given x)}{q_{0}(y\given x)} - 1 -
\dotp{\theta}{f(x,y) - E_{{\tilde{p}}}[f\given x]}+p(y\given x)\frac{1}{p(y\given x)}\right)\\
&= {\tilde{p}}(x)\left(\log \frac{p(y\given x)}{q_{0}(y\given x)}  -
\dotp{\theta}{f(x,y) - E_{{\tilde{p}}}[f\given x]}\right)
\end{align*}
and we deduce that
$z(y\given x\,;\theta) \deff \mbox{arg}\min_{p} \cL_1(p, \theta)$,  is
\begin{align*}
z(y\given x\,;\theta) = q_{0}(y\given x)\,
\exp\left(\sum_{j} \theta_j  \left(f_j(x,y)-E_{{\tilde{p}}}[f_j\given x]\right)\right).
\end{align*}
Thus, the dual function $h_1(\theta) =
\cL_1(z(y\given x\,;\theta), \theta)$ is given by
\begin{align*}
h_1(\theta) &= \sum_x {\tilde{p}}(x)\sum_y
    q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)-E_{{\tilde{p}}}[f\given x]}}
\left(\dotp{\theta}{f(x,y)-E_{{\tilde{p}}}[f\given x]}-1-\dotp{\theta}{f(x,y)-E_{{\tilde{p}}}[f\given x]}\right) \\
 &=- \sum_x {\tilde{p}}(x) \sum_{y} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)-E_{{\tilde{p}}}[f\given x]}}.
\end{align*}
The dual problem $(P_1^*)$ is to determine
\begin{align}
\theta^\star &= \argmax_\theta h_1(\theta) \nonumber
=\argmin_{\theta}\sum_x {\tilde{p}}(x) \sum_{y} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)-E_{{\tilde{p}}}[f\given x]}}\\
&= \argmin_{\theta}\sum_x {\tilde{p}}(x) \sum_{y} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)-f(x,\tilde{y}(x)}}\nonumber \\
&= \argmin_{\theta}\sum_x {\tilde{p}}(x) \sum_{y\neq \tilde{y}(x)}
    q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)-f(x,\tilde{y}(x)}}. \label{eq:dualProb1}
\end{align}

\subsubsection{The Dual Problem $(P_2^*)$}
To derive the dual for $P_2$, we add additional Lagrange
multipliers $\mu_x$ for the constraints $\sum_{y} p(y\given x) = 1$
and note that if
the normalization constraints are satisfied then the other constraints take the form
\begin{align*}
\sum_{x} {\tilde{p}}(x) \sum_y p(y\given x) f_j(x,y) =
\sum_{x, y} {\tilde{p}}(x, y) f_j(x,y).
\end{align*}

The Lagrangian becomes
\begin{align*}
\cL_2(p,\theta,\mu)
=& D(p,q_0)-\sum_j \theta_j
\left(\sum_{x} {\tilde{p}}(x) \sum_y p(y\given x) f_j(x,y)-\sum_{x, y} {\tilde{p}}(x, y) f_j(x,y)\right)\\
&-\sum_x \mu_x\left(1-\sum_y p(y\given x)\right).
\end{align*}

Setting the partial derivatives $\frac{\partial \cL_2(p,\theta)}{\partial p(y|x)}$
to 0 and noting that in the normalized case we can ignore the last two terms
in the $I$-divergence, we get
\begin{align*}
0&= {\tilde{p}}(x)\left(\log \frac{p(y|x)}{q_0(y|x)}+1-\dotp{\theta}{f(x,y)}\right)+ \mu_x
\end{align*}
from which the minimizer
$z(y\given x\,;\theta) \deff \mbox{arg}\min_{p} \cL_2(p, \theta)$ is seen to be
\begin{align*}
z(y\given x\,;\theta) = q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)}-1-\mu_x/{\tilde{p}}(x)}.
\end{align*}
Substituting $z(y\given x\,;\theta)$ in $\cL_2$ we obtain the dual function
$h_2(\theta,\mu)$. Maximizing the dual function with respect to $\mu$ results in
a choice of $\mu_x$ that ensure the normalization of $z$
\[z(y\given x\,;\theta) = \frac{1}{Z_x} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)}}\]
and maximizing $h_2$ with respect to $\theta$ we get the following dual problem
\begin{align} \nonumber
\theta^*&=\argmax_{\theta}
\sum_x {\tilde{p}}(x)\sum_y \frac{1}{Z_x(\theta)} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)}}
(\dotp{\theta}{f(x,y)}- \log Z_x(\theta))\\ \nonumber
&-\sum_x {\tilde{p}}(x)\sum_y \dotp{\theta}{f(x,y)} \frac{1}{Z_x(\theta)} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)}}
+\sum_{xy} {\tilde{p}}(x,y)\dotp{\theta}{f(x,y)}-0 \sum_x \mu_x\\ \nonumber
&= \argmax_{\theta}
-\sum_x {\tilde{p}}(x)\log Z_x(\theta) \sum_y \frac{1}{Z_x(\theta)} q_{0}(y\given x) e^{\dotp{\theta}{f(x,y)}}
+\sum_{xy} {\tilde{p}}(x,y)\dotp{\theta}{f(x,y)}\\ \nonumber
&=\argmax_{\theta} \sum_{x,y} {\tilde{p}}(x,y) \dotp{\theta}{f(x,y)}
-\sum_{x,y} {\tilde{p}}(x,y) \log Z_x(\theta)\\ \nonumber
&=\argmax_{\theta}\sum_{x,y} {\tilde{p}}(x,y) \log \frac{1}{Z_x(\theta)} q_{0}(y \given x) e^{\dotp{\theta}{f(x,y)}}\\
&=\argmax_{\theta}\sum_{x} {\tilde{p}}(x) \log \frac{1}{Z_x(\theta)} q_{0}(\tilde y(x) \given x) e^{\dotp{\theta}{f(x,\tilde y(x))}} \label{eq:dualProb2}.
\end{align}


\subsubsection{Special cases}
It is now straightforward to derive various boosting
and conditional exponential models problems
as special cases of the dual problems \eqref{eq:dualProb1} and \eqref{eq:dualProb2}.
 \vspace{0.1in} \newline
\emph{Case 1: AdaBoost.M2}.\enspace
The dual problem $(P_1^*)$ with  $q_{0}(y\given x) = 1$ is
 the optimization problem of AdaBoost.M2
\begin{align*}
\theta^\star  &=
\argmin_{\theta} \sum_{x}{\tilde{p}}(x) \sum_{y \neq y_i} e^{\dotp{\theta}{f(x_i,y)-f(x_i, y_i)}}=\argmin_{\theta} \cE(\theta).
\end{align*}
 \vspace{0.1in} \newline \emph{Case 2: Binary AdaBoost}.\enspace
The dual problem $(P_1^*)$ with  $q_{0}(y\given x) = 1$, $\mathcal{Y}=\{-1,1\}$
and $f_j(x,y) = \frac{1}{2} y\,f_j(x)$
is the optimization problem of binary AdaBoost
\begin{align*}
\theta^\star &= \argmin_{\theta}  \sum_{x}{\tilde{p}}(x) e^{-y_i \dotp{\theta}{f(x_i)}}.
\end{align*}
\vspace{0.1in} \newline
\emph{Case 3: Maximum Likelihood for Exponential Models}.\enspace
The dual problem $(P_2^*)$ with  $q_{0}(y\given x) = 1$ is
is maximum (conditional)
likelihood for a conditional exponential model with sufficient
statistics $f_j(x,y)$
\begin{align*}
\theta^\star
&= \argmax_{\theta}\sum_{x} {\tilde{p}}(x) \log \frac{1}{Z_x}e^{\dotp{\theta}{f(x,\tilde y(x))}}
= \argmax_{\theta} \ell(\theta).
\end{align*}
\vspace{0.1in} \newline
\emph{Case 4: Logistic Regression}.\enspace
The dual problem $(P_2^*)$ with  $q_{0}(y\given x) = 1$, $\mathcal{Y}=\{-1,1\}$
and $f_j(x,y) = \frac{1}{2} y\,f_j(x)$
is maximum (conditional) likelihood for binary logistic regression.
\begin{align*}
\theta^\star =
\argmax_{\theta}\sum_{x} {\tilde{p}}(x)
\frac{1}{1 + e^{-{\tilde{y}(x)\dotp{\theta}{f(x)}}}}.
\end{align*}
We note that it is not necessary to scale the features by
a constant factor here, as in \cite{Friedman2000}; the
correspondence between logistic regression and boosting
is direct.
\vspace{0.1in} \newline
\emph{Case 5: Exponential Models with Carrier Density}.\enspace
Taking $q_0(y|x)\neq 1$ to be a non-parametric density estimator
in $(P_2^*)$ results in maximum likelihood for exponential
models with a carrier density $q_0$.
Such semi-parametric
models have been proposed by \emcite{Efron1996} for integrating between
parametric and nonparametric statistical modeling
and by \emcite{DellaPietra1992} and
\emcite{Rosenfeld1996} for integrating exponential models and
$n$-gram estimators in language modeling.

\paragraph{}
Making the Lagrangian duality argument
rigorous requires care, because of the possibility that the solution may lie on
the (topological)
boundary of $\overline{\mathbb{R}_{+}^{k\times m}}$ or
$\overline{\P_{m-1}^k}$.

\paragraph{}
Let $\cQ_1$ and $\cQ_2$ be
\begin{align*}
\cQ_1(q_{0}, f) &= \left\{ q\in\overline{\mathbb{R}_{+}^{k\times m}} \;\given\; q(y\given x) \,=\,
q_{0}(y\given x)\, e^{\dotp{\theta}{f(x,y)-f(x,\tilde{y}(x))}}
,\,\theta\in\R^l\right\} \\
\cQ_2(q_{0}, f) &= \left\{ q\in\overline{\P_{m-1}^k} \;\given\; q(y\given x) \,\propto\,
{q_{0}(y\given x)\, e^{\dotp{\theta}{f(x,y)}}}
,\,\theta\in\R^l\right\}
\end{align*}
and the boosting solution $q^\star_{\mbox{\small\it boost}}$
and maximum likelihood solution $q^\star_{\mbox{\small\it ml}}$ be
\begin{align*}
q^\star_{\mbox{\small\it boost}} &= \argmin_{q\in \overline{\cQ_1}} \sum_{x}{\tilde{p}}(x) \sum_y q(y\given x) \\
q^\star_{\mbox{\small\it ml}} &= \argmax_{q\in \overline{\cQ_2}} \sum_{x}{\tilde{p}}(x) \log q(\tilde{y}(x)\given x).
\end{align*}

\paragraph{}
The following proposition corresponds to Proposition~4 of
\cite{DellaPietra1997} for the usual Kullback-Leibler divergence;
the proof for the $I$-divergence carries over with only minor changes.
In \cite{DellaPietra2001} the duality theorem is proved for a class of
Bregman divergences, including the extended Kullback-Leibler divergence as a special case.
Note that we do not  require divergences such as $D(0, q)$ as
in \cite{Collins2002},
but rather $D({\tilde{p}}, q)$, which is more natural and interpretable from a
statistical point-of-view.
\begin{prop}
Suppose that $D({\tilde{p}},q_{0}) < \infty$.  Then $q^\star_{\mbox{\small\it boost}}$ and $q^\star_{\mbox{\small\it ml}}$ exist,
are unique, and satisfy
\begin{eqnarray}
q^\star_{\mbox{\small\it boost}} &=& \argmin_{p\in\cF} D(p,q_{0})  \seq
  \argmin_{q\in\overline{\cQ_1}} D({\tilde{p}}, q) \\
q^\star_{\mbox{\small\it ml}}    &=& \argmin_{p\in\cF\cap\P_{m-1}^k} D(p,q_{0}) \seq
\argmin_{q\in\overline{\cQ_2}} D({\tilde{p}}, q)
\end{eqnarray}
Moreover, $q^\star_{\mbox{\small\it ml}}$ is computed in terms of
$q^\star_{\mbox{\small\it boost}}$ as
$ q^\star_{\mbox{\small\it ml}} = \displaystyle \argmin_{p\in\cF\cap\P_{m-1}^k} D(p,q^\star_{\mbox{\small\it boost}}) $.
\end{prop}
\paragraph{}
This result has a simple geometric interpretation.
The non-normalized exponential family $\cQ_1$ intersects the
feasible set of measures $\cF$ satisfying the constraints (\ref{eq:constraints})
at a single point. The algorithms
presented in \cite{Collins2002} determine this point, which
is the exponential loss solution
$q^\star_{\mbox{\small\it boost}} = \argmin_{q\in\overline{\cQ_1}} D({\tilde{p}}, q)$ (see Figure~\ref{fig:geom}, left).
On the other hand, maximum conditional likelihood estimation for an
exponential model with the same features is equivalent to
the problem $q^\star_{\mbox{\small\it ml}} = \argmin_{q\in\overline{\cQ_1'}} D({\tilde{p}}, q)$
where $\cQ_1'$ is the exponential family with additional Lagrange
multipliers, one for each normalization constraint.  The feasible
set for this problem is $\cF \cap \overline{\P_{m-1}^k}$.
Since
$\cF \cap\overline{\P_{m-1}^k} \subset \cF$, by the Pythagorean equality we have
that $q^\star_{\mbox{\small\it ml}} = \argmin_{p\in\cF\cap\overline{\P_{m-1}^k}}
D(p, q^\star_{\mbox{\small\it boost}})$ (see Figure \ref{fig:geom}, right).

\begin{figure}
\centering
\psfrag{f}{$\cF$}
\psfrag{qb}{$q^\star_{\mbox{\small\it boost}}$}
\psfrag{qm}{$q^\star_{\mbox{\small\it ml}}$}
\psfrag{g}{$\mathcal{G}$}
{\psfrag{q}{$\mathcal{Q}_1$}
\includegraphics[scale=0.45]{boostingLogRegFigures/geom1.eps}}
\hskip10pt
\psfrag{f}{$\cF$}
\psfrag{qb}{$q^\star_{\mbox{\small\it boost}}$}
\psfrag{qm}{$q^\star_{\mbox{\small\it ml}}$}
\psfrag{g}{$\cF\cap \overline{\P_{m-1}^k}$}
{\psfrag{q}{$\mathcal{Q}_1'$}
\includegraphics[scale=0.45]{boostingLogRegFigures/geom2.eps}}
\caption{\small Geometric view of duality.  Minimizing the exponential loss finds the
member of $\cQ_1$ that intersects the feasible set of measures
satisfying the moment constraints (left).  When we impose the
additional constraint that each conditional distribution
$q_\theta(y\given x)$ must be normalized, we introduce a Lagrange
multiplier for each training example $x$, giving a higher-dimensional
family $\cQ_1'$.  By the duality theorem, projecting the exponential
loss solution onto the intersection of the feasible set with the
simplex of conditional probabilities, $\cF\cap\overline{\P_{m-1}^k}$, we obtain the
maximum likelihood solution.  In many practical cases this projection
is obtained by simply normalizing by a constant, resulting in an
identical model.}
\label{fig:geom}
\vskip.4in
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Regularization}
Minimizing the exponential loss or the log-loss on real data often fails
to produce finite parameters. Specifically, this happens when for some feature $f_j$
\begin{eqnarray}
\label{eq:cond0}
&& f_j(x,y) - f_j(x,\tilde{y}(x)) \geq 0 \;
\mbox{for all $y$ and $x$ with ${\tilde{p}}(x)>0$} \\
\mbox{or}
&& f_j(x,y) - f_j(x,\tilde{y}(x)) \leq 0 \;
\mbox{for all $y$ and $x$ with ${\tilde{p}}(x)>0$}
\nonumber
\end{eqnarray}
This is especially harmful since often the features for which (\ref{eq:cond0}) holds
are the most important for the purpose of discrimination.
The parallel update in \cite{Collins2002} breaks down in such cases, resulting
in parameters going to $\infty$ or $-\infty$.
On the other hand, iterative scaling algorithms work in principle for
such features. In practice however,
either the parameters $\theta$ need to be artificially capped or the features
need to be thrown out altogether, resulting in a partial and less discriminating set of features.
Of course, even when (\ref{eq:cond0}) does not hold, models trained by maximizing likelihood or
minimizing exponential loss can overfit the training data.  The standard regularization technique
in the case of maximum likelihood employs parameter priors in a Bayesian framework.

\paragraph{}
In terms of convex duality, a parameter prior for the dual problem
corresponds to a ``potential'' on the constraint values in the
primal problem.  The case of a Gaussian prior on $\theta$, for
example, corresponds to a quadratic potential on the constraint
values in the primal problem. Using this correspondence, the
connection between boosting and maximum likelihood presented in the
previous section indicates how to regularize AdaBoost using Bayesian
MAP estimation for non-normalized models, as explained below.

\paragraph{}
We now consider primal problems
over $(p,c)$ where $p\in\R_{+}^{k\times m}$ and $c\in\R^m$ is a parameter vector
that relaxes the original constraints.
The feasible set $\cF({\tilde{p}}, f, c)\subset \R_{+}^{k\times m}$ allows
a violation of the expectation constraints, represented by the vector $c$
\begin{equation}
\label{eq:constraintsreg}
\cF({\tilde{p}},f,c) \seq \left\{ p\in\R_{+}^{k\times m} \;\given\;
\sum_x {\tilde{p}}(x) \sum_{y} p(y\given x)\, \left(f_j(x,y) - E_{{\tilde{p}}}[f_j\given x]\right) = c_j\right\}.
\end{equation}
A regularized problem for non-normalized models is defined by
\begin{center}
\begin{tabular}{rrl}
$(P_{1,\mbox{\small reg}})$ & {\it minimize} & $D(p,q_{0}) + U(c)$ \\[2pt]
        & {\it subject to} & $p \in \cF({\tilde{p}}, f, c)$ \\
\end{tabular}
\end{center}
where $U:\mathbb{R}^l\to \mathbb{R}$ is a convex function whose
minimum is at $0\in\R^l$.
Intuitively $(P_{1,\mbox{\small reg}})$ allows some trade-off between
achieving low $I$ divergence to $q_0$ and some constraint violation,
with the exact form of the trade-off represented by the function $U$.
Note that it is  possible to choose $U$ in a way
that considers some feature constraints more important than others.
This may be useful when the values of the features $(f_1,\ldots,f_l)$
are known to be  corrupted by noise, where the noise intensity
varies among the features.

\paragraph{}
The dual function of the regularized problem $(P_{1,\text{\small reg}})$,
as derived in Appendix~\ref{sec:regulLossFunc}, is
\[h_{1,\mbox{\small reg}}(\theta) = h_{1}(\theta) + U^*(\theta)\]
where $U^*(\theta)$ is the convex conjugate of $U$.
For a quadratic penalty $U(c)=\sum_j \frac{1}{2}\sigma_j^2\,c_j^2 $,
we have ${U}^*(\theta)=-\sum_j\frac{1}{2}\sigma_j^{-2}\,\theta_j^{2}$
and the dual function becomes
\begin{eqnarray} \label{eq:dual}
h_{1,\mbox{\small reg}}(\theta) \seq
- \sum_x {\tilde{p}}(x) \sum_{y}
q_{0}(y\given x)\, e^{\dotp{\theta}{f(x,y)-f(x,\tilde{y}(x))}} -
\sum_j\frac{\theta_j^2}{2\sigma_j^2}.
\end{eqnarray}
A sequential update rule for (\ref{eq:dual}) incurs the small additional cost of
solving a nonlinear equation by Newton's method every iteration.
See Appendix~\ref{sec:regulLossFunc} for more details.
\emcite{Chen2000} contains a similar regularization for
maximum likelihood for exponential models in the
context of statistical language modeling.

\subsection{Experiments}
We performed experiments on some of the UCI datasets \cite{UCI} in order
to investigate the relationship between boosting and maximum likelihood empirically.
The weak learner was
\texttt{FindAttrTest} as described in \cite{Freund1996}, and the training set consisted of
a randomly chosen 90\% of the data.
Table~\ref{table:reg} shows experiments with regularized boosting.
Two boosting models are compared. The first model $q_1$
was trained for 10 features generated by \texttt{FindAttrTest}, excluding
features satisfying condition (\ref{eq:cond0}). Training was carried out
 using the parallel update method described in \cite{Collins2002}.
The second model, $q_2$, was trained using the exponential loss
with quadratic regularization. The performance was measured using the
conditional log-likelihood of the (normalized) models
over the training and test set, denoted
$\ell_{\ss{train}}$ and $\ell_{\ss{test}}$,
as well as using the test error rate $\epsilon_{\ss{test}}$.
The table entries were averaged by 10-fold cross validation.

\paragraph{}
For the weak learner \texttt{FindAttrTest}, only the Iris dataset produced
features that satisfy (\ref{eq:cond0}). On average, 4 out of the 10 features
were removed.  As the flexibility of the weak learner is increased,
(\ref{eq:cond0}) is expected to hold more often.
On this dataset regularization
improves both the test set log-likelihood and error rate considerably.
In datasets where $q_1$ shows significant over-fitting, regularization improves
both the log-likelihood measure and the error rate.
In cases of little over-fitting (according to the log-likelihood measure),
regularization only improves the test set log-likelihood at the expense of the training
set log-likelihood, however without affecting much the test set error.


\begin{table}
\centering
\begin{tabular}{|l|c|c|c||c|c|c|}
\multicolumn{1}{c}{} & \multicolumn{3}{c}{\it Unregularized} & \multicolumn{3}{c}{\it Regularized} \\
\hline
Data & $\ell_{\ss{train}}(q_1)$ & $\ell_{\ss{test}}(q_1)$&$\epsilon_{\ss{test}}(q_1)$
& $\ell_{\ss{train}}(q_2)$ & $\ell_{\ss{test}}(q_2)$ & $\epsilon_{\ss{test}}(q_2)$ \\
\hline\hline
Promoters& -0.29 &-0.60 &0.28 &-0.32 &-0.50 &0.26\\
\hline
Iris&-0.29 &-1.16 &0.21 &-0.10 &-0.20&0.09\\
\hline
Sonar& -0.22 &-0.58 &0.25 &-0.26 &-0.48&0.19\\
\hline
Glass&-0.82 &-0.90&0.36& -0.84 &-0.90  &0.36  \\
\hline
Ionosphere&-0.18 &-0.36 &0.13  &-0.21  &-0.28 &0.10 \\
\hline
Hepatitis&-0.28 &-0.42 &0.19 &-0.28 &-0.39&0.19 \\
\hline
Breast Cancer Wisconsin&-0.12 &-0.14 &0.04  &-0.12 &-0.14 &0.04 \\
\hline
Pima-Indians& -0.48&-0.53 &0.26 &-0.48  &-0.52  &0.25 \\
\hline
\end{tabular}
\caption{\small Comparison of unregularized to regularized boosting.
For both the regularized and unregularized cases, the first column
shows training log-likelihood, the second column shows
test log-likelihood, and the third column shows test
error rate.  Regularization reduces
error rate in some cases while it consistently improves
the test set log-likelihood measure on all datasets. All entries were
averaged using 10-fold cross validation.}
\label{table:reg}
\end{table}

\begin{figure}
\begin{center}
\vbox to 4.0in{
\def\yleft{\lower.4ex\hbox{\tiny{$\ell_{\st{train}}(q^{\star}_{\st{boost}})$}}}
\def\yright{\lower.3ex\hbox{\tiny{$D_{\st{train}}(q^{\star}_{\st{ml}},q^{\star}_{\st{boost}})$}}}
\def\xleft{\raise.8ex\hbox{\tiny{$\ell_{\st{train}}(q^{\star}_{\st{ml}})$}}}
\def\xright{\lower1ex\hbox{\tiny{$\ell_{\st{train}}(q^{\star}_{\st{ml}})$}}}
\begin{tabular}{ccl}
\hskip.2in
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/hepatitisD.eps}
}
&&\hskip0.3in\mbox{\ }
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/hepatitis2D.eps}
}
\\[10pt]
\hskip.2in
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/promotersD.eps}
}
&&\hskip0.3in\mbox{\ }
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/promoters2D.eps}
}
\\[10pt]
\hskip.2in
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/sonarD.eps}
}
&&\hskip0.3in\mbox{\ }
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/sonar2D.eps}
}
\\[10pt]
\hskip.2in
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/glassD.eps}
}
&& \hskip0.3in\mbox{\ }
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/glass2D.eps}
}
\end{tabular}
}
\end{center}
\vskip5pt
\caption{\small Comparison of AdaBoost and maximum likelihood on four UCI datasets:
Hepatitis (top row), Promoters (second row), Sonar (third row) and Glass (bottom row).
The left column compares $\ell_{\ss{train}}(q^{\star}_{\ss{ml}})$ to $\ell_{\ss{train}}(q^{\star}_{\ss{boost}})$, and the right column compares
$\ell_{\ss{train}}(q^{\star}_{\ss{ml}})$ to $D_{\ss{train}}(q^{\star}_{\ss{ml}},q^{\star}_{\ss{boost}})$.}
\label{fig:ll}
\end{figure}


\begin{figure}
\begin{center}
\vbox to 3.8in {
\def\yleft{\raise1ex\hbox{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{boost}})$}}}
\def\xright{\lower.4ex\hbox{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{ml}})$}}}
\def\xleft{\raise.2ex\hbox{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{ml}})$}}}
\def\yright{\raise1ex\hbox{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{boost}})$}}}

\begin{tabular}{ccl}
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/hepatitis3D.eps}
}
&&
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/hepatitis4D.eps}
}
\\[4pt]
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/promoters3D.eps}
}
&&
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/promoters4D.eps}
}
\\[4pt]
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/sonar3D.eps}
}
&&
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/sonar4D.eps}
}
\\[4pt]
{\psfrag{xl}{\xleft}
 \psfrag{yl}{\yleft}
\includegraphics[scale=0.35]{boostingLogRegFigures/glass3D.eps}
}
&&
{\psfrag{xl}{\xright}
\psfrag{yl}{\yright}
\includegraphics[scale=0.35]{boostingLogRegFigures/glass4D.eps}
}
\end{tabular}
}
\end{center}
\vskip20pt
\caption{\small Comparison of AdaBoost and maximum likelihood on the same
UCI datasets as in the previous figure.  The left column compares the
test likelihoods, $\ell_{\ss{test}}(q^{\star}_{\ss{ml}})$ to
$\ell_{\ss{test}}(q^{\star}_{\ss{boost}})$, and the right column
compares test error rates, $\epsilon_{\ss{test}}(q^{\star}_{\ss{ml}})$
to $\epsilon_{\ss{test}}(q^{\star}_{\ss{boost}})$.  In each plot, the
color represents the {\it training\/} likelihood
$\ell_{\ss{train}}(q^{\star}_{\ss{ml}})$; red corresponds to fitting
the training data well.}
\label{fig:color}
\end{figure}

\ignore{
\begin{figure}
\begin{center}
\begin{tabular}{ccl}
{\psfrag{yl}{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{ml}})$}}
 \psfrag{xl}{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{boost}})$}}
\includegraphics[scale=0.35]{boostingLogRegFigures/ionoshpere3D.eps}
}
&&
{\psfrag{xl}{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{ml}})$}}
\psfrag{yl}{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{boost}})$}}
\includegraphics[scale=0.35]{boostingLogRegFigures/ionoshpere4D.eps}
}
\\[3pt]
{\psfrag{yl}{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{ml}})$}}
 \psfrag{xl}{\tiny{$\ell_{\st{test}}(q^{\star}_{\st{boost}})$}}
\includegraphics[scale=0.35]{boostingLogRegFigures/pima3D.eps}
}
&&
{\psfrag{xl}{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{ml}})$}}
\psfrag{yl}{\tiny{$\epsilon_{\st{test}}(q^{\star}_{\st{boost}})$}}
\includegraphics[scale=0.35]{boostingLogRegFigures/pima4D.eps}
}
\end{tabular}
\end{center}
\caption{\small Comparison of AdaBoost and maximum likelihood on two  UCI datasets:
Ionosphere (top row) and  Pima-Indians (second row).
Each row has two plots: $\ell_{\ss{test}}(q^{\star}_{\ss{ml}})$, $\ell_{\ss{test}}(q^{\star}_{\ss{boost}})$ (left) and test error rates
$\epsilon_{\ss{test}}(q^{\star}_{\ss{ml}})$, $\epsilon_{\ss{test}}(q^{\star}_{\ss{boost}})$ (right). In all figures the color represent $\ell_{\ss{train}}(q^{\star}_{\ss{ml}})$.}
\label{fig:color2}
\end{figure}
}

\paragraph{}
Next we performed a set of experiments to test how much
$q_{\ss{boost}}^{\star}$
differs from  $q_{\ss{ml}}^{\star}\,$,
where the boosting model is normalized (after training)
to form a conditional probability distribution.
For different experiments, \texttt{FindAttrTest} generated a different number of features
(10--100), and the training set was selected randomly.
The plots in Figure~\ref{fig:ll} show for different datasets
the relationship between  $\ell_{\ss{train}}(q_{\ss{ml}}^{\star})$ and $\ell_{\ss{train}}(q_{\ss{boost}}^{\star})$ as well as between
$\ell_{\ss{train}}(q_{\ss{ml}}^{\star})$ and
$D_{\ss{train}}(q^{\star}_{\ss{ml}},q^{\star}_{\ss{boost}})$.
The trend is the same in each data set: as the number of features increases
so that the training data is more closely fit
($\ell_{\ss{train}}(q_{\ss{ml}}) \rightarrow 0$),
the boosting and maximum likelihood models become more similar, as measured
by the $I$-divergence.

\paragraph{}
The plots in Figure~\ref{fig:color} show the relationship between the
test set log-likelihoods, $\ell_{\ss{test}}(q_{\ss{ml}}^{\star})$ to
$\ell_{\ss{test}}(q_{\ss{boost}}^{\star})$, together with the test set
error rates $\epsilon_{\ss{test}}(q_{\ss{ml}}^{\star})$ and
$\epsilon_{\ss{test}}(q_{\ss{boost}}^{\star})$.  In these figures the
testing set was chosen to be 50\% of the total data. The color represents
the {\it training\/} data log-likelihood, $\ell_{\ss{train}}(q_{\ss{ml}}^{\star})$,
with the color red corresponding to high likelihood.  In order to indicate
the number of points at each error rate, each circle
was shifted by a small random value to avoid points falling
on top of each other.

\paragraph{}
While the
plots in Figure~\ref{fig:ll} indicate that
$\ell_{\ss{train}}(q^{\star}_{\ss{ml}}) > \ell_{\ss{train}}(q^{\star}_{\ss{boost}})$, as expected,
on the test data the linear trend is reversed, so that
$\ell_{\ss{test}}(q^{\star}_{\ss{ml}}) < \ell_{\ss{test}}(q^{\star}_{\ss{boost}})$.
This is a result of the fact that for the above data-sets and features,
little over-fitting occurs and the more aggressive exponential loss criterion
is superior to the more relaxed log-loss criterion.
However, as $\ell(q^{\star}_{\ss{ml}})\longrightarrow 0$, the two models
come to agree.
Appendix~\ref{sec:divExpModels} shows that
for any exponential model $q_{\theta}\in\mathcal{Q}_2,$
\begin{equation} \label{eq:KLeqll}
D_{\ss{train}}(q^{\star}_{\ss{ml}},q_{\theta}) \;\;=\;\;
\ell(q^{\star}_{\ss{ml}})-\ell(q_{\theta}).
\end{equation}
By taking $q_{\theta}=q^{\star}_{\ss{boost}}$ it is seen that as the
difference between $\ell(q^{\star}_{\ss{ml}})$ and
$\ell(q^{\star}_{\ss{boost}})$ gets smaller, the divergence between the
two models also gets smaller.
%Furthermore, since the correlation
%coefficient $\rho$ is close to 1, we can use the approximation
%$\ell(q^{\star}_{\ss{boost}})\approx \alpha\, \ell(q^{\star}_{\ss{ml}})$
%to obtain
%\begin{equation}
%D_{\ss{train}}(q^{\star}_{\ss{ml}},q^{\star}_{\ss{boost}})\;\;\approx\;\;
%(1-\alpha)\,\ell(q^{\star}_{\ss{ml}}).
%\end{equation}

\paragraph{}
The results are consistent with the theoretical analysis.  As the number of
features is increased so that the training data is fit more closely, the
model matches the empirical distribution ${\tilde{p}}$ and the normalizing term
$Z(x)$ becomes a constant.  In this case, normalizing the
boosting model $q^{\star}_{\ss{boost}}$ does not violate the constraints,
and results in the maximum likelihood model.

\paragraph{}
In Appendix \ref{sec:parUpdate},\ref{sec:secUpdate} we
derive update rules for exponential loss minimization.
These update rules are
derived by minimizing an auxiliary function that bounds from above the
reduction in loss. See \cite{Collins2002} for the definition of
an auxiliary function and proofs that these functions are
indeed auxiliary functions.
The derived update rules are similar to the ones derived by
\emcite{Collins2002}, except that we do not
assume that $M=\mbox{max}_{i,y} \sum_j|f_j(x,y)-f_j(x,\tilde{y})|<1$.
In Appendix~\ref{sec:regulLossFunc} the regularized
formulation is shown in detail and a sequential update rule is
derived.  Appendix~\ref{sec:divExpModels} contains a proof for (\ref{eq:KLeqll}).

\paragraph{}
The next section derives an axiomatic characterization of
the geometry of conditional models. It then builds on the
results of this section to given an axiomatic characterization
of the geometry underlying conditional exponential models
and AdaBoost.
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Axiomatic Geometry for Conditional Models}
\label{sec:axiomaticGeometry}
A fundamental assumption in the information geometric framework,
is the choice of the Fisher information as the
metric that underlies the geometry of probability distributions.
The choice of the Fisher information metric may be motivated
in several ways the strongest of which is \v{C}encov's
characterization theorem (Lemma 11.3, \cite{Chentsov1982}).
In his theorem, \v{C}encov
proves that the Fisher information metric is the only metric
that is invariant under a family of probabilistically
meaningful mappings termed congruent embeddings by a Markov morphism.
Later on, Campbell
extended \v{C}encov's result to include non-normalized positive models
\cite{Campbell86}.
\paragraph{}
The theorems of \v{C}encov and Campbell are particularly interesting
since the Fisher information is pervasive in statistics and machine learning.
It is the asymptotic variance of the maximum likelihood estimators
under some regularity conditions. Cramer and Rao
used it to compute a lower bound on the variance of
arbitrary unbiased estimators.
In Bayesian statistics, it was used by Jeffreys
to define non-informative prior.
It is tightly connected to the Kullback-Leibler divergence
which the cornerstone of maximum likelihood estimation for exponential models
as well as various aspects of information theory.

\paragraph{}
While the geometric approach to statistical inference
has attracted considerable attention, little research was
conducted on the geometric approach to conditional inference.
The characterization theorems of \v{C}encov and Campbell no longer
apply in this setting and the different ways of choosing
a geometry for the space of conditional distributions,
in contrast to the non-conditional case, are not supported
by theoretical considerations.
\paragraph{}
In this section we extend the results of \v{C}encov and Campbell to
provide an axiomatic characterization of conditional information
geometry. We derive the characterization theorem in the setting
of non-normalized conditional models from which the geometry for
normalized models is obtained as a special case.
In addition, we demonstrate a close connection between the
characterized geometry and the conditional $I$-divergence which leads
to a new axiomatic interpretation of the geometry underlying
the primal problems of logistic regression and AdaBoost.
This interpretation builds on the
connection between AdaBoost and constrained minimization of $I$-divergence
described in Section~\ref{sec:boostMl}.
\paragraph{}
Throughout the section we consider spaces of strictly positive
conditional models where the
sample spaces of the explanatory and response variable are finite.
Moving to the infinite case presents some serious difficulties.
The positivity constraint on the other hand does not play a crucial role
and may by discarded at some notational cost.

\paragraph{}

In the characterization theorem we will make use of the fact that
$\P_{m-1}^k\cap \mathbb{Q}^{k\times m}$
and  $\mathbb{R}_{+}^{k\times m}\cap \mathbb{Q}^{k\times m}=
\mathbb{Q}_{+}^{k\times m}$ are dense in
$\P_{m-1}^k$ and $\mathbb{R}_{+}^{k\times m}$ respectively.
Since continuous functions are uniquely characterized by their
values on dense sets, it is enough to compute the metric
for positive rational models $\mathbb{Q}_{+}^{k\times m}$.
The value of the metric on non-rational
models follows from its continuous extension to
$\mathbb{R}_{+}^{k\times m}$.

\paragraph{}
In Section~\ref{sec:congEmbed} we define a class of transformations
called congruent embeddings by a Markov morphism. These
transformations set the stage for the axioms
in the characterization theorem of Section~\ref{sec:charTheorem}.


%-----------------------------------------------------------
\subsection{Congruent Embeddings by Markov Morphisms
of Conditional Models} \label{sec:congEmbed}
The characterization result of Section~\ref{sec:charTheorem} is based on axioms that
require geometric invariance through a set of transformations
between conditional models. These transformations
are a generalization of the transformations
underlying \v{C}encov's theorem. For consistency
with the terminology of \emcite{Chentsov1982} and \emcite{Campbell86}
we refer to these transformations as Congruent embeddings by Markov morphisms
of conditional models.

\begin{defn}
Let $\mathcal{A}=\{A_1,\ldots,A_{m}\}$ be a set partition
of $\{1,\ldots,n\}$  with $0<m\leq n$.
A matrix  $Q\in\mathbb{R}^{m\times n}$ is called $\mathcal{A}$-stochastic
if
\begin{align*}
\forall i\,\,\, \sum_{j=1}^n Q_{ij}=1,\quad \text{and}
\quad
Q_{ij}=\begin{cases}
c_{ij}>0 & j\in A_i\\
0 & j\not\in A_i
\end{cases}.
\end{align*}
\end{defn}
In other words, $\mathcal{A}$-stochastic matrices are stochastic matrices
whose rows are concentrated on the sets of the partition $\mathcal{A}$.
For example, if $\mathcal{A}=\{\{1,3\},\{2,4\},\{5\}\}$ then
the following matrix is $\mathcal{A}$-stochastic
\begin{align} \label{eq:matrixExample}
\begin{pmatrix}
1/3&0  &2/3&0  &0\\
0  &1/2&0  &1/2&0\\
0  & 0 &0  &0  &1
\end{pmatrix}.
\end{align}
Obviously, the columns of any $\mathcal{A}$-stochastic matrix
have precisely one non-zero element. If $m=n$ then
an $\mathcal{A}$-stochastic matrix is a permutation matrix.

\paragraph{}
Multiplying a row probability vector $p\in\mathbb{R}_{+}^{1\times m}$
with an $\mathcal{A}$-stochastic matrix $Q\in\mathbb{R}^{m\times n}$
results in a row probability vector $q\in\mathbb{R}_{+}^{1\times n}$.
The mapping $p\mapsto pQ$ has the following statistical interpretation.
The event $x_i$ is split into $|A_i|$ distinct events stochastically,
with the splitting probabilities given by the $i$-row of $Q$.
The new event space, denoted by $\mathcal{Z}=\{z_1,\ldots,z_n\}$
may be considered a refinement of $\mathcal{X}=\{x_1,\ldots,x_m\}$
(if $m<n$) and the model $q(z)$ is a consistent refinement of $p(x)$.
For example, multiplying $p=(1/2,1/4,1/4)$ with the matrix $Q$
in \eqref{eq:matrixExample} yields
\[ q=pQ=(1/6,1/8,2/6,1/8,1/4).\]
In this transformation, $x_1$ was split into $\{z_1,z_3\}$
with unequal probabilities, $x_2$ was split into $\{z_2,z_4\}$
with equal probabilities and $x_3$ was relabeled $z_5$ (Figure 3)
\begin{figure}
\centering
{
\psfrag{p}{$p$}
\psfrag{q}{$q=pQ$}
\includegraphics[scale=0.5]{congruentEmbeddingFigures/congEmbed1.eps}}
\caption{Congruent embedding by a Markov morphism of $p=(1/2,1/4,1/4)$.}
\end{figure}

\paragraph{}
The transformation $q\mapsto qQ$ is injective and therefore invertible.
For example, the inverse transformation to $Q$ in \eqref{eq:matrixExample} is
\begin{align*}
p(x_1)&=q(z_1)+q(z_3)\\p(x_2)&=q(z_2)+q(z_4)\\ p(x_3)&=q(z_5).
\end{align*}
The inverse transformation may be interpreted as extracting a sufficient statistic
$T$ from $\mathcal{Z}$. The sufficient statistic
joins events in $\mathcal{Z}$ to create the event space $\mathcal{X}$,
hence transforming models on $\mathcal{Z}$ to corresponding models on $\mathcal{X}$.

\paragraph{}
So far we have considered transformations of non-conditional models.
The straightforward generalization to conditional models involves
performing a similar transformation on the response space $\mathcal{Y}$
for every non-conditional model $p(\cdot|x_i)$ followed by
transforming the explanatory space $\mathcal{X}$. It is formalized in the definitions below
and illustrated in Figure 4.
\begin{defn}
Let $M\in\mathbb{R}^{k\times m}$ and $Q=\{Q^{(i)}\}_{i=1}^k$
be a set of  matrices in $\mathbb{R}^{m\times n}$.
We define the
row product $M\otimes Q\in\mathbb{R}^{k\times n}$  as
\begin{align}
[M\otimes Q]_{ij} = \sum_{s=1}^m M_{is}Q^{(i)}_{sj}
= [MQ^{(i)}]_{ij}.
\end{align}
\end{defn}
In other words, the $i$-row of $M\otimes Q$ is the $i$-row of the
matrix product $MQ^{(i)}$.

\paragraph{}
\begin{defn} \label{def:cess}
Let $\mathcal{B}$ be a $k$ sized partition of $\{1,\ldots,l\}$ and
 $\{\mathcal{A}^{(i)}\}_{i=1}^k$ be a set of $m$ sized partitions
of $\{1,\ldots,n\}$.
Furthermore, let $R\in\mathbb{R}_{+}^{k\times l}$
be a $\mathcal{B}$-stochastic matrix
and $Q=\{Q^{(i)}\}_{i=1}^k$ a sequence of $\mathcal{A}^{(i)}$-stochastic
matrices in $\mathbb{R}_{+}^{m\times n}$.
Then the map
\begin{align} \label{eq:ss-map}
f:\mathbb{R}_{+}^{k\times m}\to\mathbb{R}_{+}^{l\times n}
\quad f(M)=R^{\top}(M\otimes Q)
\end{align}
is termed a congruent
embeddings by a Markov morphism of $\mathbb{R}_{+}^{k\times m}$
into $\mathbb{R}_{+}^{l\times n}$ and
the set of all such maps is denoted by $\mathfrak{F}_{k,m}^{l,n}$.
\end{defn}

\paragraph{}
Congruent embeddings by a Markov morphism $f$ are injective and if restricted to the
space of normalized  models $\P_{m-1}^k$ they produce a normalized model as well
i.e. $f(\P_{m-1}^k)\subset \P_{n-1}^l$.
\begin{figure*}
\centering
{
\psfrag{p}{$M$}
\psfrag{r}{$M\otimes Q$}
\psfrag{q}{$R^{\top}(M\otimes Q)$}
\includegraphics[scale=0.5]{congruentEmbeddingFigures/congEmbed2.eps}}
\caption{Congruent embedding by a Markov morphism of $\mathbb{R}_{+}^{2\times 3}$
into $\mathbb{R}_{+}^{5\times 6}$.}
\end{figure*}
The component-wise version of equation \eqref{eq:ss-map} is
\begin{align} \label{eq:ss-map-component}
[f(M)]_{ij} &= \sum_{s=1}^k \sum_{t=1}^m R_{si} Q^{(s)}_{tj} M_{st}
\end{align}
with the above sum containing precisely one non-zero term since
every column of $Q^{(s)}$ and $R$ contains only one non-zero entry.
The push-forward map
$f_*:T_M\mathbb{R}_{+}^{k\times m}\to T_{f(M)}\mathbb{R}_{+}^{l\times n}$
associated with $f$ is
\begin{align}
f_*(\partial_{ab})&=
\sum_{i=1}^l \sum_{j=1}^{n} R_{ai} Q^{(a)}_{bj} \partial_{ij}'
\label{eq:push-forward}
\end{align}
where $\{\partial_{ab}\}_{a,b}$ and $\{\partial_{ij}'\}_{ij}$
are the bases of $T_M\mathbb{R}_{+}^{k\times m}$ and
$\partial_{ij}'\in T_{f(M)}\mathbb{R}_{+}^{l\times n}$
respectively.
Using definition \ref{def:pull-back} and
equation \eqref{eq:push-forward},
the pull-back of a metric $g$ on $\mathbb{R}_{+}^{l\times n}$
through $f\in\mathfrak{F}_{k,m}^{l,n}$ is
\begin{align}
(f^*g)_M(\partial_{ab},\partial_{cd}) \label{eq:pull-back}
=g_{f(M)}(f_*\partial_{ab},f_*\partial_{cd})
&= \sum_{i=1}^l\sum_{j=1}^n\sum_{s=1}^l\sum_{t=1}^n
    R_{ai}R_{cs}Q^{(a)}_{bj}Q^{(c)}_{dt}
g_{f(M)}(\partial_{ij}',\partial_{st}').
\end{align}

\paragraph{}
An important special case of a congruent embedding by a Markov morphism
is specified by uniform $\mathcal{A}$-stochastic matrices defined next.
\begin{defn}\label{def:uniformMatrix}
An $\mathcal{A}$-stochastic matrix
is called uniform if every row has the same number of non-zero elements
and if all its positive entries are identical.
\end{defn}
For example, the following matrix is a uniform $\mathcal{A}$-stochastic
matrix for $\mathcal{A}=\{\{1,3\},\{2,4\},\{5,6\}\}$
\[
\begin{pmatrix}
1/2&0  &1/2&0  &0&0\\
0  &1/2&0  &1/2&0&0\\
0  & 0 &0  &0  &1/2&1/2
\end{pmatrix}.
\]
\paragraph{}
We proceed in the next section to state and prove the characterization theorem.
%--------------------------------------------------------------
\subsection{A Characterization of Metrics on Conditional Manifolds} \label{sec:charTheorem}
As mentioned in the previous section, congruent embeddings
by a Markov morphism have a strong probabilistic
interpretation. Such maps transform conditional models
to other conditional models in a manner consistent with
changing the granularity of the event spaces.
Moving to a finer or coarser description of the event space
should not have an effect on the models if such a move may
be expressed as a sufficient statistic.
It makes sense then to require that the geometry of a space
of conditional models be invariant under such transformations.
Such geometrical invariance is obtained by requiring
maps $f\in\mathfrak{F}_{k,m}^{l,n}$ to be isometries.
The main results of the section are Theorems \ref{thrm:axiom1} and \ref{thrm:axiom1} below
followed by Corollary \ref{corr:axiomNormalized}. The proof of Theorem 1 bears some
similarity to the proof of Campbell's theorem \cite{Campbell86}
which in turn is related to the proof technique used in
Khinchin's characterization of the entropy \cite{Khinchin1957}.
Throughout the section we avoid \v{C}encov's style of
using category theory and use only
standard techniques in differential geometry.

%-------------------
\subsubsection{Three Useful Transformation}
Before we turn to the characterization theorem we show that
congruent embeddings by a Markov morphisms are norm preserving
and examine three special cases that will be useful later on.

\paragraph{}
We denote by $M_i$ the $i$th row of the matrix
$M$ and by $|\cdot|$ the $L^1$ norm applied to vectors or matrices
\[ |v|=\sum_i |v_i|\quad |M|=\sum_i |M_i| = \sum_{ij}|M_{ij}|.\]
\begin{prop}
Maps in $\mathfrak{F}_{k,m}^{l,n}$ are norm preserving:
\[ |M|=|f(M)| \qquad \forall f\in\mathfrak{F}_{k,m}^{l,n},\,\,
\forall M\in \mathbb{R}_{+}^{k\times m}.\]
\end{prop}
\begin{proof}
Multiplying a positive row vector $v$  by an
$\mathcal{A}$-stochastic matrix $T$ is norm preserving
\[|v T|=\sum_i [vT]_i=\sum_{j} v_j\sum_i T_{ji}
=\sum_j v_j = |v|.\]
As a result, $|[MQ^{(i)}]_i|=|M_i|$ for any positive matrix $M$
and hence
\[|M|=\sum_i|M_i|=\sum_i|[MQ^{(i)}]_i|=|M\otimes Q|.\]
A map $f\in\mathfrak{F}_{k,m}^{l,n}$ is norm preserving since
\begin{align*}
|M|&=|M\otimes Q|=|(M\otimes Q)^{\top}|=|(M\otimes Q)^{\top}R|
=|R^{\top}(M\otimes Q)|=|f(M)|.
\end{align*}
\end{proof}

We denote the symmetric group of permutations
over $k$ letters by  $\mathfrak{S}_k$. The first transformation
$\mathfrak{h}_{\sigma}^{\Pi}\in\mathfrak{F}_{k,m}^{k,m}$,
parameterized by a $\sigma\in\mathfrak{S}_k$ and
\[\Pi=(\pi^{(1)},\ldots,\pi^{(k)}) \quad \pi^{(i)}\in\mathfrak{S}_m,\]
is defined by $Q^{(i)}$ being the permutation
matrix that corresponds to $\pi^{(i)}$ and $R$ being the permutation
matrix that corresponds to $\sigma$.
The push forward is
\begin{align}
\mathfrak{h}_{\sigma*}^{\Pi}(\partial_{ab})
=\partial_{\sigma(a)\pi^{(a)}(b)}'
\end{align}
and requiring $\mathfrak{h}_{\sigma}^{\Pi}$ to be an isometry
from $(\mathbb{R}_{+}^{k\times m},g)$ to itself amounts to
\begin{align} \label{eq:pull-back1}
g_M(\partial_{ab},\partial_{cd})
&=g_{\mathfrak{h}_{\sigma}^{\Pi}(M)}(
    \partial_{\sigma(a)\pi^{(a)}(b)},
    \partial_{\sigma(c)\pi^{(c)}(d)})
\end{align}
for all $M\in\mathbb{R}_{+}^{k\times m}$ and
for every pair of basis vectors
$\partial_{ab},\partial_{cd}$ in $T_M\mathbb{R}_{+}^{k\times m}$.


The usefulness of $\mathfrak{h}_{\sigma}^{\Pi}$
stems in part from the following proposition.
\begin{prop} \label{prop:map1}
Given $\partial_{a_1b_1},\partial_{a_2b_2},\partial_{c_1d_1},\partial_{c_2d_2}$
with $a_1\neq c_1$ and $a_2\neq c_2$ there exists $\sigma,\Pi$ such that
\begin{align} \label{eq:f1class}
%\exists \mathfrak{h}_{\sigma}^{\Pi}\quad
\mathfrak{h}_{\sigma*}^{\Pi}(\partial_{a_1b_1})=\partial_{a_2b_2}\qquad
\mathfrak{h}_{\sigma*}^{\Pi}(\partial_{c_1d_1})=\partial_{c_2d_2}.
\end{align}
\end{prop}
\begin{proof}
The desired map may be obtained by
selecting $\Pi,\sigma$ such that
$\sigma(a_1)=a_2,\sigma(c_1)=c_2$
and
$\pi^{(a_1)}(b_1)=b_2$, $\pi^{(c_1)}(d_1)=d_2$.
\end{proof}


The second transformation $\mathfrak{r}_{zw}\in\mathfrak{F}_{k,m}^{kz,mw}$,
parameterized by $z,w\in\mathbb{N}$,
is defined by $Q^{(1)}=\cdots=Q^{(k)}\in\mathbb{R}^{m\times mw}$
and $R\in\mathbb{R}^{k\times kz}$ being uniform matrices
(in the sense of Definition \ref{def:uniformMatrix}).
Note that each row of $Q^{(i)}$ has precisely $m$ non-zero entries
of value $1/m$ and each row of $R$ has precisely $z$ non-zero entries
of value $1/z$.
The exact forms of  $\{Q^{(i)}\}$ and $R$ are immaterial for our purposes
and any uniform matrices of the above sizes will suffice.
By equation \eqref{eq:push-forward} the push-forward is
\[ \mathfrak{r}_{zw*}(\partial_{st}) = \frac{1}{zw}\sum_{i=1}^z\sum_{j=1}^w
\partial_{\pi(i)\sigma(j)}'\]
for some permutations $\pi,\sigma$ that depend on $s,t$
and the precise shape of $\{Q^{(i)}\}$ and $R$.
The pull-back of $g$ is
\begin{align}
&(\mathfrak{r}_{zw}^*g)_M(\partial_{ab},\partial_{cd})= \label{eq:pull-back2}
\frac{1}{(zw)^2}\sum_{i=1}^z \sum_{j=1}^w \sum_{s=1}^z \sum_{t=1}^w
g_{\mathfrak{r}_{zw}(M)}(\partial_{\pi_1(i),\sigma_1(j)}',
\partial_{\pi_2(s),\sigma_2(t)}'),
\end{align}
again, for some permutations $\pi_1,\pi_2,\sigma_1,\sigma_2$.

We will often express rational conditional models
$M\in\mathbb{Q}_{+}^{k\times m}$  as
\[M=\frac{1}{z} \tilde M,
\qquad \tilde M\in\mathbb{N}^{k\times m}\,\,\, z\in\mathbb{N}\]
where $\mathbb{N}$ is the natural numbers.
Given a rational model $M$, the third mapping
\[\mathfrak{y}_M\in\mathfrak{F}_{k,m}^{|\tilde M|,\prod_i |\tilde M_i|}
\quad \text{where}\quad M=\frac{1}{z} \tilde M\in\mathbb{Q}_{+}^{k\times m}\]
is associated with $Q^{(i)}\in\mathbb{R}^{m\times \prod_s |\tilde M_s|}$ and
$R\in\mathbb{R}^{k\times |\tilde M|}$ which are defined as follows.
The $i$-row of $R\in\mathbb{R}^{k\times |\tilde M|}$ is required to have
$|\tilde M_{i}|$ non-zero elements of value $|\tilde M_{i}|^{-1}$.
Since the number of columns equals the number of positive entries it is possible
to arrange the entries such that each columns will have precisely
one positive entry. $R$ then is an $\mathcal{A}$-stochastic matrix
for some partition $\mathcal{A}$.
The $j$th row of $Q^{(i)}\in\mathbb{R}^{m\times \prod_s |\tilde M_s|}$
is required to have $\tilde M_{ij}\prod_{s\neq i}|\tilde M_{s}|$
non-zero elements of value $(\tilde M_{ij}\prod_{s\neq i}|M_{s}|)^{-1}$.
Again, the number of positive entries
\[\sum_j \tilde M_{ij}\prod_{s\neq i}|\tilde M_{s}|
= \prod_s |\tilde M_s|\]
is equal to the number of columns and hence $Q^{(i)}$ is a legal
$\mathcal{A}$ stochastic matrix for some $\mathcal{A}$.
Note that the number of positive entries, and also
columns of $Q^{(i)}$ does not depend on $i$ hence $\{Q^{(i)}\}$ are of the
same size.
The exact forms of $\{Q^{(i)}\}$ and $R$ do not matter for our purposes as long
as the above restriction and the requirements for $\mathcal{A}$-stochasticity
 apply (Definition \ref{def:cess}).

The usefulness of $\mathfrak{y}_M$ comes from the fact that it transforms
the rational models $M$ into a constant matrix.
\begin{prop}
For $M=\frac{1}{z}\tilde M\in\mathbb{Q}_{+}^{k\times m}$,
\[
\mathfrak{y}_M(M)=\left(z\prod_{s}|\tilde M_{s}|\right)^{-1}
\mathbf{1}\]
where $\mathbf{1}$ is a matrix of ones of size
$|\tilde M|\times\prod_{s}|\tilde M_{s}|$.
\end{prop}
\begin{proof}
$[M\otimes Q]_i$ is a row vector of size
$\prod_{s}|\tilde M_{s}|$
whose elements are
\begin{align*}
[M\otimes Q]_{ij}&=[MQ^{(i)}]_{ij}
= \frac{1}{z} \tilde M_{ir}
\frac{1}{\tilde M_{ir} \prod_{s\neq i} |\tilde M_s|}
=\left(z\prod_{s\neq i}|\tilde M_{s}|\right)^{-1}
\end{align*}
for some $r$ that depends on $i,j$.
Multiplying on the left by $R$
results in
\begin{align*}
[R^{\top}(M\otimes Q)]_{ij}&=R_{ri}[M\otimes Q]_{rj}
=\frac{1}{|\tilde M_r|}\frac{1}{z \prod_{s\neq r}|\tilde M_s|}
=\left(z\prod_{s}|\tilde M_{s}|\right)^{-1}
\end{align*}
for some $r$ that depends on $i,j$.
\end{proof}
A straightforward calculation using equation \eqref{eq:push-forward}
and the definition of $\mathfrak{y}_{M*}$ above shows
that the push-forward of $\mathfrak{y}_M$ is
\begin{align} \label{eq:map3}
\mathfrak{y}_{M*}(\partial_{ab}) =
\frac{\sum_{i=1}^{|\tilde M_{a}|}\,\,\,
\sum_{j=1}^{\tilde M_{ab}
    \prod_{l\neq a}|\tilde M_l|}\partial_{\pi(i)\sigma(j)}'}{
    \tilde M_{ab}\prod_{i}|\tilde M_{i}|}.
\end{align}
for some permutations $\pi,\sigma$ that depend on $M,s,t$.
Substituting equation \eqref{eq:map3} in equation \eqref{eq:pull-back}
gives the pull-back
\begin{align} \label{eq:pull-back3}
(\mathfrak{y}_M^*g)_M(\partial_{ab},\partial_{cd})=
\frac{\sum_i\sum_s\sum_j\sum_t
    g_{\mathfrak{y}_M(M)}(
    \partial_{\pi_1(i)\sigma_1(j)},\partial_{\pi_2(s)\sigma_2(t)})}
{\tilde M_{ab}\tilde M_{cd}\prod_s |\tilde M_s|^2}
\end{align}
where the first two summations are over $1,\ldots,|\tilde M_a|$
and $1,\ldots,|\tilde M_c|$ and the last two summations are over
$1,\ldots,\tilde M_{ab}\prod_{l\neq a}|\tilde M_{l}|$ and
$1,\ldots,\tilde M_{cd}\prod_{l\neq c}|\tilde M_{l}|$.

\subsubsection{The Characterization Theorem}
Theorems \ref{thrm:axiom1} and \ref{thrm:axiom2} below are the main result of
Section~\ref{sec:axiomaticGeometry}.
\begin{thrm} \label{thrm:axiom1}
Let $\{(\mathbb{R}_{+}^{k\times m}, g^{(k,m)}): k\geq 1,m\geq 2\}$
be a sequence of Riemannian manifolds with the property that
every congruent embedding
by a Markov morphism is an isometry. Then
\begin{align}
&g_M^{(k,m)}(\partial_{ab},\partial_{cd})=\nonumber \\
&A(|M|)+\delta_{ac} \left(\frac{|M|}{|M_{a}|} B(|M|)
+\delta_{bd} \frac{|M|}{M_{ab}}C(|M|)\right) \label{eq:metricForm}
\end{align}
for some $A,B,C\in C^{\infty}(\mathbb{R}_{+},\R)$.
\end{thrm}
\begin{proof}
The proof below uses the isometry requirement
to obtain restrictions on $g^{(k,m)}_M(\partial_{ab},\partial_{cd})$
first for
$a\neq c$, followed by the case of $a=c,b\neq d$ and finally
for the case $a=c,b=d$. In each of these
cases, we first characterize the metric at constant matrices $U$
and then compute it for rational models $M$ by pulling back the metric at
$U$ through $\mathfrak{y}_M$. The value of the metric
at non-rational models follows from the rational case by the denseness
of $\mathbb{Q}_{+}^{k\times m}$ in $\mathbb{R}_{+}^{k\times m}$
and the continuity of the metric.
%---------------------------------------
\paragraph{}
\textit{Part I:} $g_M^{(k,m)}(\partial_{ab},\partial_{cd})$
\textit{ for } $a\neq c$\newline
We start by computing the metric at constant matrices
$U$.
Given  $\partial_{a_1b_1},\partial_{c_1d_1}, a_1\neq c_1$
and  $\partial_{a_2b_2},\partial_{c_2d_2}, a_2\neq c_2$
we can use Proposition~\ref{prop:map1} and equation \eqref{eq:pull-back1}
to pull back
through a corresponding $\mathfrak{h}_{\sigma}^{\Pi}$ to obtain
\begin{align}\label{eq:proof0}
g_U^{(k,m)}(\partial_{a_1b_1},\partial_{c_1d_1})&=
g_{\mathfrak{h}_{\sigma}^{\Pi}(U)}^{(k,m)}
    (\partial_{a_2b_2},\partial_{c_2d_2})
=g_U^{(k,m)}(\partial_{a_2b_2},\partial_{c_2d_2}).
\end{align}
Since \eqref{eq:proof0} holds for all $a_1,a_2,b_1,b_2$ with
$a_1\neq c_1,a_2,\neq c_2$ we have that
$g_U^{(k,m)}(\partial_{ab},\partial_{cd}), a\neq c$
depends only on $k$, $m$ and $|U|$ and we denote it
temporarily by $\hat A(k,m,|U|)$.
\paragraph{}
A key observation, illustrated in Figure \ref{fig:congEmbed2-a},
is the fact that pushing forward
$\partial_{a,b},\partial_{c,d}$ for $a\neq c$
through any $f\in\mathfrak{F}_{k,m}^{l,n}$ results in two sets
of basis vectors whose pairs have disjoint rows.
As a result, in the pull-back equation \eqref{eq:pull-back},
 all the terms in the sum represent metrics between two basis
vectors with different rows.
\begin{figure}
\centering
{
\psfrag{p}{$\partial_{ab}$}
\psfrag{q}{$\partial_{cd}$}
\psfrag{r}{$\mathcal{S}_1$}
\psfrag{t}{$\mathcal{S}_2$}
\includegraphics[scale=0.3]{congruentEmbeddingFigures/congEmbed2-a.eps}}
\caption{Pushing forward
$\partial_{ab},\partial_{cd}$ for $a\neq c$
through any $f\in\mathfrak{F}_{k,m}^{l,n}$ results in two sets
of basis vectors $\mathcal{S}_1$ (black) and $\mathcal{S}_2$ (gray)
for which every pair of vectors
$\{(v,u):v\in\mathcal{S}_1,u\in\mathcal{S}_2\}$
are in disjoint rows.}
\label{fig:congEmbed2-a}
\end{figure}
As a result of the above observation, in computing
the pull back $g^{(kz,mw)}$ through $\mathfrak{r}_{zw}$ \eqref{eq:pull-back2}
we have a sum of $z^2w^2$ metrics between vectors of disjoint rows
\begin{align}
\hat A(k,m,|U|) &= g_U^{(k,m)}(\partial_{ab},\partial_{cd})
= \frac{(zw)^2}{(zw)^2}\hat A(kz,mw,|\mathfrak{r}_{zw}(U)|)=
\hat A(kz,mw,|U|) \label{eq:proof1}
\end{align}
since $\mathfrak{r}_{zw}(U)$ is a constant matrix with the same
norm as $U$.
Equation \eqref{eq:proof1} holds for any $z,w\in\mathbb{N}$ and
hence $g_U^{(k,m)}(\partial_{ab},\partial_{cd})$ does not depend
on $k,m$ and we write
\[
 g_U^{(k,m)}(\partial_{ab},\partial_{cd})
=A(|U|)\quad \text{for some} \quad A\in C^{\infty}(\mathbb{R}_{+},\R).\]

\paragraph{}
We turn now to computing
$g_M^{(k,m)}(\partial_{ab},\partial_{cd}), a\neq c$ for
rational models $M=\frac{1}{z}\tilde M$. Pulling back through
$\mathfrak{y}_M$ according to equation \eqref{eq:pull-back3} we have
\begin{align}
g_M^{(k,m)}(\partial_{ab},\partial_{cd}) &=
\frac{{\tilde M_{ab}\tilde M_{cd}\prod_s |\tilde M_{s}|^2}}{{\tilde M_{ab}\tilde M_{cd}\prod_s |\tilde M_{s}|^2}}A(|\mathfrak{y}_M(M)|)
=A(|M|). \label{eq:proof2}
\end{align}
Again, we made use of the fact that in the pull-back equation
\eqref{eq:pull-back3} all the terms in the sum are metrics between
vectors of different rows.
\paragraph{}
Finally, since $\mathbb{Q}_{+}^{k\times m}$ is dense in $\mathbb{R}_{+}^{k\times m}$
and $g^{(k,m)}_M$ is continuous in $M$,
equation \eqref{eq:proof2} holds for all models
in $\mathbb{R}_{+}^{k\times m}$.
\paragraph{}
%--------------------------------
\textit{Part II:} $g_M^{(k,m)}(\partial_{ab},\partial_{cd})$
\textit{ for } $a=c,b\neq d$\newline
As before we start with constant matrices $U$.
Given $\partial_{a_1,b_1},\partial_{c_1,d_1}$ with $a_1=c_1,b_1\neq d_1$
and  $\partial_{a_2,b_2},\partial_{c_2,d_2}$ with $a_2=c_2,b_2\neq d_2$
we can pull-back through $\mathfrak{h}_{\sigma}^{\Pi}$
with $\sigma(a_1)=a_2, \pi^{(a_1)}(b_1)=b_2$ and
$\pi^{(a_1)}(d_1)=d_2$ to obtain
\begin{align*}
 g^{(k,m)}_U(\partial_{a_1b_1},\partial_{c_1d_1})
=g^{(k,m)}_{\mathfrak{h}_{\sigma}^{\Pi}(U)}(\partial_{a_2b_2},\partial_{c_2d_2})
=g^{(k,m)}_U(\partial_{a_2b_2},\partial_{c_2d_2}).
\end{align*}
It follows that
$g_U^{(k,m)}(\partial_{ab},\partial_{ad})$ depends only on $k,m,|U|$
and we temporarily denote
\[g_U^{(k,m)}(\partial_{ab},\partial_{ad})=\hat B(k,m,|U|).\]
\paragraph{}
As in Part I, we stop to make an important observation,
illustrated in Figure  \ref{fig:congEmbed2-b}.
Assume that $f_*$ pushes forward
$\partial_{a,b}$ to a set of vectors $\mathcal{S}_1$
organized in $z$ rows and $w_1$ columns
and $\partial_{a,d},b\neq d$ to a set of vectors
$\mathcal{S}_2$ organized in $z$ rows and $w_2$ columns.
Then counting the pairs of vectors $\mathcal{S}_1\times\mathcal{S}_2$
we obtain $zw_1w_2$ pairs of vectors that have the same rows but
different columns and $zw_1(z-1)w_2$ pairs of vectors that have different rows
and different columns.
\begin{figure}
\centering
{
\psfrag{p}{$\partial_{ab}$}
\psfrag{q}{$\partial_{cd}$}
\psfrag{r}{$\mathcal{S}_1$}
\psfrag{t}{$\mathcal{S}_2$}
\psfrag{z}{$z$}
\psfrag{w}{$w_1$}
\psfrag{x}{$w_2$}
\includegraphics[scale=0.3]{congruentEmbeddingFigures/congEmbed2-b.eps}}
\caption{
Let $f_*$ push forward
$\partial_{ab}$ to a set of vectors $\mathcal{S}_1$ (black)
organized in $z$ rows and $w_1$ columns
and $\partial_{ab},b\neq d$ to a set of vectors (gray)
$\mathcal{S}_2$ organized in $z$ rows and $w_2$ columns.
Then counting the pairs of vectors $\mathcal{S}_1\times\mathcal{S}_2$
we obtain $zw_1w_2$ pairs of vectors that have the same rows but
different columns and $zw_1(z-1)w_2$ pairs of vectors that have different rows
and different columns.}\label{fig:congEmbed2-b}
\end{figure}

Applying the above observation to the push-forward of
$\mathfrak{r}_{k,m}^{kz,mw}$ we have among the set of pairs
$\mathcal{S}_1\times\mathcal{S}_2$,
$zw^2$ pairs of vectors with the same rows but different columns
and $zw^2(z-1)$ pairs of vectors with different rows and different columns.
\paragraph{}
Pulling back through $\mathfrak{r}_{zw}$ according
to equation \eqref{eq:pull-back2} and the above observation we obtain
\begin{align*}
    \hat B(k,m,|U|) &=
    \frac{zw^2\hat B(kz,mw,|U|)}{(zw)^2}
    +\frac{z(z-1)w^2A(|U|)}{(zw)^2}
= \frac{1}{z}\hat B(kz,mw,|U|) + \frac{z-1}{z}A(|U|)
\end{align*}
where the first term corresponds to the
 $zw^2$ pairs of vectors with the same rows but different columns
and the second term corresponds to the
$zw^2(z-1)$ pairs of vectors with different rows and different columns.
Rearranging and dividing by $k$ results in
\begin{align*}
    \frac{\hat B(k,m,|U|)-A(|U|)}{k}
    &= \frac{\hat B(kz,mw,|U|)-A(|U|)}{kz}.
\end{align*}
It follows that the above quantity is independent of $k,m$ and
we write $\frac{\hat B(k,m,|U|)-A(|U|)}{k}=B(|U|)$ for some
$B\in C^{\infty}(\mathbb{R}_{+},\R)$ which after rearrangement
gives us
\begin{align}
g_U^{(k,m)}(\partial_{ab},\partial_{ad}) = A(|U|) + k B(|U|).
\end{align}
\paragraph{}
We compute next the metric for positive rational matrices
$M=\frac{1}{z}\tilde M$ by pulling back through $\mathfrak{y}_M$.
We use again the observation in Figure \ref{fig:congEmbed2-b},
but now with $z=|\tilde M_a|$, $w_1=\tilde M_{ab}\prod_{l\neq a}|\tilde M_l|$
and $w_2=\tilde M_{ad}\prod_{l\neq a}|\tilde M_l|$.
Using \eqref{eq:pull-back3} the pull-back through
$\mathfrak{y}_M$ is
\begin{align} \label{eq:proof3}
g_M^{(k,m)}(\partial_{ab},\partial_{ad})
=&\frac{|\tilde M_a|\tilde M_{ab}\prod_{l\neq a}|\tilde M_l|(|\tilde M_a|-1)
\tilde M_{ad}\prod_{l\neq a}|\tilde M_l|}{\tilde M_{ab}\tilde M_{ad}\prod_i |\tilde M_{i}|^2}
A(|M|)\nonumber \\
&+ \frac{|\tilde M_a|\tilde M_{ab}\tilde M_{ad}\prod_{l\neq a}|\tilde M_l|^2}{\tilde M_{ab}\tilde M_{ad}\prod_i |\tilde M_{i}|^2}
\Big(A(|M|)+B(|M|)\sum_i |\tilde M_{i}|\Big)
\nonumber \\
=&\frac{|\tilde M_{a}|-1}{|\tilde M_{a}|}A(|M|)
 + \frac{1}{|\tilde M_{a}|}\Big(A(|M|)
 +B(|M|)\sum_i |\tilde M_{i}|\Big)\\ \nonumber
=& A(|M|) + \frac{|\tilde M|}{|\tilde M_{a}|}B(|M|)
= A(|M|) + \frac{|M|}{|M_{a}|}B(|M|).\nonumber
\end{align}
The first term in the sums above corresponds to the
$zw_1(z-1)w_2$ pairs of vectors that have different rows and
different columns and the second term corresponds
to the $zw_1w_2$ pairs of vectors that have different columns but the
same row.
As previously, by denseness of $\mathbb{Q}_{+}^{k\times m}$ in
 $\mathbb{R}_{+}^{k\times m}$
and continuity of $g^{(k,m)}$
equation \eqref{eq:proof3} holds for all
$M\in \mathbb{R}_{+}^{k\times m}$.
\paragraph{}
%-----------------------------
\textit{Part III:} $g_M^{(k,m)}(\partial_{ab},\partial_{cd})$
\textit{ for } $a=c,b= d$\newline
As before, we start by computing the metric for
constant matrices $U$.
Given $a_1,b_1,a_2,b_2$ we pull back through
$\mathfrak{h}_{\sigma}^{\Pi}$ with
$\sigma(a_1)=a_2,\pi^{(a_1)}(b_1)=b_2$ to obtain
\[g^{(k,m)}_U(\partial_{a_1b_1},\partial_{a_1b_1})
=g^{(k,m)}_U(\partial_{a_2b_2},\partial_{a_2b_2}).\]
It follows that $g_U^{(k,m)}(\partial_{ab},\partial_{ab})$
does not depend on $a,b$ and we temporarily denote
\[g^{(k,m)}_U(\partial_{ab},\partial_{ab})=\hat C(k,m,|U|).\]
\paragraph{}

In the present case, pushing forward two identical vectors
$\partial_{a,b},\partial_{a,b}$ by a congruent embedding $f$
results in two identical sets of vectors $\mathcal{S},\mathcal{S}$
that we assume are organized in $z$ rows and $k$ columns.
Counting the pairs in $\mathcal{S}\times \mathcal{S}$ we obtain
$zw$ pairs of identical vectors, $zw(w-1)$ pairs of vectors
of the identical rows but different columns and
$zw^2(z-1)$ pairs of vectors of different rows and columns.
These three sets of pairs allow us to organize the terms
in the pull-back summation \eqref{eq:pull-back}
into the three cases under considerations.
\paragraph{}

Pulling back through $\mathfrak{r}_{zw}$ \eqref{eq:pull-back2}
we obtain
\begin{align*}
\hat C(k,m,|U|) &=
    \frac{zw\hat C(kz,mw,|U|)}{(zw)^2}+
    \frac{z(z-1)w^2A(|U|)}{(zw)^2}
+\frac{zw(w-1)(A(|U|)+kzB(|U|))}{(zw)^2}\\
    =&\frac{\hat C(kz,mw,|U|)}{zw}
+\Big(1-\frac{1}{zw}\Big)A(|U|)
+ \Big(k-\frac{zk}{zw}\Big)B(|U|)
\end{align*}
which after rearrangement and dividing by $km$
gives
\begin{align} \label{eq:proofC}
\frac{\hat C(k,m,|U|) - A(|U|) - kB(|U|)}{km} =
 \frac{\hat C(kz,mw,|U|) -A(|U|) -kzB(|U|)}{kzmw}.
\end{align}
It follows that the left side of \eqref{eq:proofC}
equals a function $C(|U|)$ for some $C\in C^{\infty}(\mathbb{R}_{+},\R)$
independent of $k$ and $m$ resulting in
\begin{align*}
g_U^{(k,m)}(\partial_{ab},\partial_{ab}) &=
A(|U|) + k B(|U|) + km C(|U|).
\end{align*}
\paragraph{}
Finally, we compute $g_M^{(k,m)}(\partial_{ab},\partial_{ab})$ for
positive rational matrices $M=\frac{1}{z}\tilde M$.
Pulling back through $\mathfrak{y}_M$ \eqref{eq:pull-back3}
and using the above division of $\mathcal{S}\times\mathcal{S}$
with $z=\tilde M_a,w=\tilde M_{ab}\prod_{l\neq a}|\tilde M_l|$ we obtain
\begin{align}\label{eq:proof5}
g_M^{(k,m)}(\partial_{ab},\partial_{ab})
=&\frac{|\tilde M_{a}|-1}{|\tilde M_{a}|}A(|M|)
+ \Big(\frac{1}{|\tilde M_a|}-\frac{1}{\tilde M_{ab}\prod_i |\tilde M_i|}\Big)\Big(A(|M|) +B(|M|)\sum_i |\tilde M_{i}|\Big)\\
&+ \frac{A(|M|)+B(|M|)\sum_i |\tilde M_i|+C(|M|)
\prod_j |\tilde M_j|\sum_i |\tilde M_i|}{\tilde M_{ab}\prod_i|\tilde M_i|}\nonumber \\
=& A(|M|) + \frac{|\tilde M|}{|\tilde M_{a}|}B(|M|)
+\frac{|\tilde M|}{\tilde M_{ab}}C(|M|)
= A(|M|) + \frac{|M|}{|M_{a}|}B(|M|) +\frac{|M|}{M_{ab}}C(|M|).\nonumber
\end{align}
Since the positive  rational matrices are dense in
$\mathbb{R}_{+}^{k\times m}$
and the metric $g^{(k,m)}_M$ is continuous in $M$, equation \eqref{eq:proof5}
holds for all models $M\in\mathbb{R}_{+}^{k\times m}$.
\end{proof}

%--------------------------
The following theorem is the converse of Theorem \ref{thrm:axiom1}.
\begin{thrm} \label{thrm:axiom2}
Let $\{(\mathbb{R}_{+}^{k\times m},g^{(k,m)})\}$
be a sequence of Riemannian manifolds,
with the metrics $g^{(k,m)}$ given by
\begin{align}\label{eq:metricForm2}
g_M^{(k,m)}(\partial_{ab},\partial_{cd})=
A(|M|)+\delta_{ac} \left(\frac{|M|}{|M_{a}|} B(|M|)
+\delta_{bd} \frac{|M|}{M_{ab}}C(|M|)\right)
\end{align}
for some $A,B,C\in C^{\infty}(\mathbb{R}_{+},\R)$.
Then every congruent embedding by a Markov morphism is an isometry.
\end{thrm}
\begin{proof}
To prove the theorem we need to show that
\begin{align}
&\forall M\in\mathbb{R}_{+}^{k\times m},\quad \forall f\in\mathfrak{F}_{k,m}^{l,n},
\quad \forall u,v\in T_M\mathbb{R}^{k\times m}_{+},
\qquad
g^{(k,m)}_{M}(u,v)=g^{(l,n)}_{f(M)}(f_*u,f_*v).\label{eq:thrm2-3}
\end{align}
\paragraph{}
Considering arbitrary $M\in\mathbb{R}_{+}^{k\times m}$ and
$f\in\mathfrak{F}_{k,m}^{l,n}$ we have by equation \eqref{eq:pull-back}
\begin{align}\label{eq:thrm2}
g^{(l,n)}_{f(M)}(f_*\partial_{ab},f_*\partial_{cd})
&= \sum_{i=1}^l\sum_{j=1}^n\sum_{s=1}^l\sum_{t=1}^n
    R_{ai}R_{cs}Q^{(a)}_{bj}Q^{(c)}_{dt}
g^{(l,n)}_{f(M)}(\partial_{ij}',\partial_{st}').
\end{align}
\paragraph{}
For $a\neq c$, using the metric form of equation \eqref{eq:metricForm2},
the right hand side of equation \eqref{eq:thrm2} reduces to
\begin{align*}
A(|f(M)|)\sum_{i=1}^l\sum_{j=1}^n\sum_{s=1}^l\sum_{t=1}^n
    R_{ai}R_{cs}Q^{(a)}_{bj}Q^{(c)}_{dt}
=A(|f(M)|)=A(|M|)=g^{(k,m)}_{M}(\partial_{ab},\partial_{cd})
\end{align*}
since $R$ and $Q^{(i)}$ are stochastic matrices.
\paragraph{}
Similarly, for $a=c,b\neq d$, the right hand side of
equation \eqref{eq:thrm2} reduces to
\begin{align}
&A(|f(M)|)\sum_{i=1}^l\sum_{j=1}^n\sum_{s=1}^l\sum_{t=1}^n
 R_{ai}R_{cs}Q^{(a)}_{bj}Q^{(c)}_{dt}
+ B(|f(M)|) \sum_i\frac{|f(M)|}{|[f(M)]_i|}
    R_{ai}^2 \sum_j\sum_t Q^{(a)}_{bj}Q^{(a)}_{dt}\nonumber \\
&=A(|M|)+ B(|M|)|M|\sum_i \frac{R_{ai}^2}{|[f(M)]_i|}. \label{eq:proof2-1}
\end{align}
Recall from equation \eqref{eq:ss-map-component} that
\[[f(M)]_{ij} = \sum_{s=1}^k \sum_{t=1}^m R_{si} Q^{(s)}_{tj} M_{st}.\]
Summing over $j$ we obtain
\begin{align} \label{eq:proof2-2}
|[f(M)]_i| = \sum_{s=1}^k  R_{si}\sum_{t=1}^m M_{st} \sum_j Q^{(s)}_{tj}
= \sum_{s=1}^k R_{si} |M_s|.
\end{align}
Since every column of $R$ has precisely one non-zero element
it follows from \eqref{eq:proof2-2}
 that $R_{ai}$ is either 0 or $\frac{|[f(M)]_i|}{|M_a|}$ which
turns equation \eqref{eq:proof2-1} into
\begin{align*}
g^{(l,n)}_{f(M)}(f_*\partial_{ab},f_*\partial_{ad})=
A(|M|)+ B(|M|)|M|\sum_{i:R_{ai}\neq 0} \frac{R_{ai}}{|M_a|}
=A(|M|)+ B(|M|)\frac{|M|}{|M_a|}=g^{(k,m)}_{M}(\partial_{ab},\partial_{ad}).
\end{align*}
\paragraph{}
Finally, for the case $a=c,b=d$ the right hand side of
equation \eqref{eq:thrm2} becomes
\begin{align*}
&A(|M|)+ B(|M|)\frac{|M|}{|M_a|}+C(|M|)|M|
\sum_{i=1}^l\sum_{j=1}^n \frac{(R_{ai}Q^{(a)}_{bj})^2}{[f(M)]_{ij}}.
\end{align*}
Since in the double sum of equation \eqref{eq:ss-map-component}
\[[f(M)]_{ij} = \sum_{s=1}^k \sum_{t=1}^m R_{si} Q^{(s)}_{tj} M_{st}\]
there is a unique positive element, $R_{ai}Q^{(a)}_{bj}$ is either $[f(M)]_{ij}/M_{ab}$ or 0.
It follows then that equation \eqref{eq:thrm2} equals
\begin{align*}
g^{(l,n)}_{f(M)}(f_*\partial_{ab},f_*\partial_{ab})
=&A(|M|)+ B(|M|)\frac{|M|}{|M|_a}+C(|M|)|M| \sum_{i:R_{ai}\neq 0}
\sum_{j:Q_{bj}^{(a)}\neq 0}
\frac{R_{ai}Q^{(a)}_{bj}}{M_{ab}}\\
&=A(|M|)+ B(|M|)\frac{|M|}{|M_a|}+C(|M|)\frac{|M|}{M_{ab}}=
g^{(k,m)}_{M}(\partial_{ab},\partial_{ab}).
\end{align*}
\paragraph{}
We have shown  that for arbitrary $M\in\mathbb{R}_{+}^{k\times m}$ and
$f\in\mathfrak{F}_{k,m}^{l,n}$
\[ g^{(k,m)}_M(\partial_{ab},\partial_{cd})
= g^{(l,n)}_{f(M)}(f_*\partial_{ab},f_*\partial_{cd})\]
 for each pair of tangent basis vectors $\partial_{ab},\partial_{cd}$ and hence
the condition in \eqref{eq:thrm2-3} holds, thus proving that
\[f:\left(\mathbb{R}^{(k,m)}_{+},g^{(k,m)}\right)\to
\left(\mathbb{R}^{(l,n)}_{+},g^{(l,n)}\right)\]
is an isometry.
\end{proof}

%------------------------------
\subsubsection{Normalized Conditional Models}
A stronger statement can be said in the case of normalized conditional
models. In this case, it turns out that the choices of $A$ and
$B$ are immaterial and equation \eqref{eq:metricForm} reduces to the
product Fisher information,
scaled by a constant that represents the choice of the function
$C$. The following corollary specializes
the characterization theorem to the normalized manifolds
$\mathbb{P}_{m-1}^k$.
\begin{corr}\label{corr:axiomNormalized}
In the case of the manifold of normalized conditional models,
equation \eqref{eq:metricForm} in
theorem 1 reduces to the product Fisher information
metric up to a multiplicative constant.
\end{corr}
\begin{proof}
For $u,v\in T_M\mathbb{P}_{m-1}^k$ expressed in the coordinates
of the embedding tangent space $T_M\mathbb{R}^{k\times m}_{+}$
\[u=\sum_{ij}u_{ij}\partial_{ij}\quad
v=\sum_{ij}v_{ij}\partial_{ij}\]
we have
\begin{align}
g^{(k,m)}_M(u,v)&=
\left(\sum_{ij}u_{ij}\right)\left(\sum_{ij}v_{ij}\right) A(|M|)
+\sum_i \left(\sum_j u_{ij}\right)\left(\sum_j v_{ij}\right)
\frac{|M|}{|M_i|}B(|M|)\\
&+\sum_{ij}u_{ij}v_{ij}\frac{|M|C(|M|)}{M_{ij}}
=kC(k)\sum_{ij}\frac{u_{ij}v_{ij}}{M_{ij}}
\end{align}
since $|M|=k$ and for $v\in T_M\mathbb{P}_{m-1}^k$ we have
$\sum_j v_{ij}=0$ for all $i$.
We see that the choice of $A$ and $B$ is immaterial
and the resulting metric
is precisely the product Fisher information metric up to
a multiplicative constant $kC(k)$, that corresponds to the choice of $C$.
\end{proof}


%--------------------------------------------------------------
\subsection{A Geometric Interpretation of Logistic Regression
and AdaBoost}
In this section, we use the close relationship between the
product Fisher information metric and conditional $I$-divergence
to study the geometry implicitly assumed by logistic regression
and AdaBoost.
\paragraph{}
Logistic regression is a popular technique
for conditional inference, usually represented by the following
normalized conditional model
\[p(1|x\,;\theta)=\frac{1}{Z} e^{\sum_{i} x_i \theta_i},
\quad x,\theta\in\mathbb{R}^n,\quad
\mathcal{Y}=\{-1,1\}\]
where $Z$ is the normalization factor.
A more general form, demonstrated in Section~\ref{sec:boostMl}
 that is appropriate for  $2\leq |\mathcal{Y}|<\infty$  is
\begin{align} \label{eq:logRegMult}
p(y|x\,;\theta) &= \frac{1}{Z} e^{\sum_i \theta_i f_i(x,y)},
\quad x,\theta\in\mathbb{R}^n, y\in\mathcal{Y}
\end{align}
where $f_i:\mathcal{X}\times\mathcal{Y}\to\mathbb{R}$ are
arbitrary feature functions.
The model \eqref{eq:logRegMult} is a conditional exponential model
and the parameters $\theta$ are normally obtained by maximum likelihood estimation
for a training set $\{(x_j,y_j)\}_{j=1}^N$
\begin{align}\label{eq:logRegMLE}
\argmax_{\theta}
\sum_{j=1}^N \sum_i \theta_i f_i(x_j,y_j) - \sum_{j=1}^N \log
\sum_{y'\in\mathcal{Y}} e^{\sum_i \theta_i f_i(x_j,y')}.
\end{align}

\paragraph{}
AdaBoost is a linear classifier, usually viewed as an incremental
ensemble methods that combines weak learners \cite{Schapire2002}.
The incremental rule that AdaBoost uses to
select the weight vector $\theta$ is known to greedily minimize
the exponential loss
\begin{align}\label{eq:expLoss}
\argmin_{\theta} \sum_j \sum_{y\neq y_j} e^{\sum_i \theta_i (f_i(x_j,y)-f_i(x_j,y_j))}
\end{align}
associated with a non-normalized model
\[ p(y|x\,;\theta) = e^{\sum_i \theta_i f_i(x,y)},
\quad x,\theta\in\mathbb{R}^n,y\in\mathcal{Y}.\]
\paragraph{}
By moving to the convex primal problems that correspond to
maximum likelihood for logistic regression \eqref{eq:logRegMLE}
and minimum exponential loss for AdaBoost \eqref{eq:expLoss}
a close connection between the two algorithms appear cf. Section~\ref{sec:boostMl}.
Both problems selects a model that minimizes the $I$-divergence
\eqref{eq:I-div}
\begin{align}
D_r(p,q) = \sum_x r(x)\sum_y\left( p(y|x)\log \frac{p(y|x)}{q(y|x)} - p(y|x)+q(y|x)\right). \nonumber
\end{align}
to a uniform distribution $q$ where $r$ is the empirical distribution
over the training set $r(x) = \frac{1}{N}\sum_{i=1}^N \delta_{x,x_i}$.

\paragraph{}
The minimization is constrained by expectation equations with the addition of normalization
constraints for logistic regression.
The $I$-divergence above applies to non-normalized conditional models and reduces to
the conditional Kullback-Leibler divergence for normalized models.
The conditional form above \eqref{eq:I-div} is a generalization of
the non-normalized divergence for probability measures
studied by Csisz\'{a}r \cite{Csiszar1991}.
\paragraph{}
Assuming $\epsilon=q-p\to 0$ we may approximate
$D_r(p,q)=D_r(p,p+\epsilon)$ by a second order Taylor approximation
around $\epsilon=0$
\begin{align}
&D_r(p,q) \approx D_r(p,p)+
\sum_{xy} \frac{\partial D(p,p+\epsilon)}{\partial \epsilon(y,x)}\Big|_{\epsilon=0} \epsilon(y,x)
\nonumber \\
&+\frac{1}{2}\sum_{x_1y_1}\sum_{x_2y_2}
    \frac{\partial^2 D(p,p+\epsilon)}{\partial \epsilon(y_1,x_1)\partial\epsilon(y_2,x_2)}
        \Big|_{\epsilon=0}
         \epsilon(y_1,x_1)\epsilon(y_2,x_2).\nonumber
\end{align}
The first order terms
\begin{align}
\frac{\partial D_r(p,p+\epsilon)}{\partial \epsilon(y_1,x_1)} &=
r(x_1)\left(1-\frac{p(y_1|x_1)}{p(y_1|x_1)+\epsilon(y_1,x_1)}\right)\nonumber
\end{align}
zero out for $\epsilon=0$. The second order terms
\begin{align*}
\frac{\partial^2 D_r(p,p+\epsilon)}{\partial \epsilon(y_1,x_1)\partial \epsilon(y_2,x_2)} &=
    \frac{\delta_{y_1y_2}\delta_{x_1x_2} r(x_1) p(y_1|x_1)}{(p(y_1|x_1)+\epsilon(y_1,x_1))^2}
\end{align*}
at $\epsilon=0$ are $\delta_{y_1y_2}\delta_{x_1x_2}\frac{r(x_1)}{p(y_1|x_1)}$.
Substituting these expressions in the Taylor approximation gives
\[ D_r(p,p+\epsilon)\approx \frac{1}{2}\sum_{xy}
    \frac{r(x)\epsilon^2(y,x)}{p(y|x)}
=\frac{1}{2}\sum_{xy}
    \frac{(r(x)\epsilon(y,x))^2}{r(x)p(y|x)}\]
which is the squared length
of $r(x)\epsilon(y,x)\in T_{r(x)p(y|x)} \mathbb{R}^{k\times m}_{+}$
under the metric \eqref{eq:metricForm}
for the choices $A(|M|)=B(|M|)=0$ and $C(|M|)=1/(2\,|M|)$.
\paragraph{}
The $I$ divergence $D_r(p,q)$ which both logistic regression and AdaBoost
minimize is then approximately the squared geodesic distance
between the conditional models $r(x)p(y|x)$ and $r(x)q(y|x)$
under a metric \eqref{eq:metricForm} with the above choices of $A,B,C$.
The fact that the models $r(x)p(y|x)$ and $r(x)q(y|x)$ are not strictly
positive is not problematic,
since by the continuity of the metric, theorems 1 and 2 pertaining to
$\mathbb{R}_{+}^{k\times m}$ apply also to its closure
$\overline{\mathbb{R}_{+}^{k\times m}}$ - the set of all
non-negative conditional models.
\paragraph{}
The above result is not restricted to logistic regression and AdaBoost.
It carries over to any conditional modeling technique that
is based on maximum entropy or minimum Kullback-Leibler divergence.
%-------------------------------------------------------------
\subsection{Discussion}
We formulated and proved an axiomatic characterization of a family
of metrics, the simplest of which is the product Fisher information metric
in the conditional setting for both normalized and non-normalized models.
This result is a strict generalization of Campbell's and \v{C}encov's theorems.
For the case $k=1$, Theorems 1 and 2 reduce to
Campbell's theorem \cite{Campbell86} and corollary 1 reduces
to \v{C}encov's theorem (Lemma 11.3 of \cite{Chentsov1982}).
\paragraph{}
In contrast to \v{C}encov's and Campbell's theorems we do not make
any reference to a joint distribution and our analysis is strictly
discriminative. If one is willing to consider a joint
distribution it may be possible to derive a geometry on the space
of conditional models $p(y|x)$ from Campbell's geometry on the  space of joint
models $p(x,y)$. Such a derivation may be based on the observation
that the conditional manifold is a quotient manifold
of the joint manifold. If such a derivation is carried over,
it is likely that the derived metric would be different from
the metric characterized in this section.

\paragraph{}
As mentioned in Section 3, the proper framework for considering
non-negative models is a manifold with corners \cite{Lee2002}.
The theorem stated here carries over by the continuity of the metric
from the manifold of positive models to its closure.
Extensions to infinite $\mathcal{X}$ or $\mathcal{Y}$ poses considerable
difficulty. For a brief discussion of infinite dimensional manifolds
representing densities see \cite{Amari2000} pp. 44-45.
\paragraph{}
The characterized metric \eqref{eq:metricForm} has three additive components.
The first one represents a components that is independent of the
tangent vectors, but depends on the norm of the model at which it is evaluated.
Such a dependency may be used to produce the effect of giving
higher importance to large models, that represent more confidence.
The second term is non-zero if the two tangent vectors represent
increases in the current model along $p(\cdot|x_a)$.
In this case, the term depends not only on the norm of the model but
also on $|M_a|=\sum_j p(y_j|x_a)$.
This may be useful in dealing with non-normalized conditional models whose
values along the different rows $p(\cdot|x_i)$ are not on the same
numeric scale. Such scale variance may represent different importance
in the predictions made, when conditioning on different $x_i$.
The last component represents the essence of the Fisher information
quantity. It scales up with low values $p(y_j|x_i)$ to represent
a kind of space stretching, or distance enhancing when we are dealing
with points close to the boundary. It captures a similar effect as the
log-likelihood of increased importance given to near-zero erroneous
predictions.
\paragraph{}
Using the characterization theorem we give for the first time,
a differential geometric interpretation of
logistic regression and AdaBoost whose metric is characterized by
natural invariance  properties.
Such a geometry applies not only to the above models, but to any
algorithmic technique that is based on maximum conditional entropy principles.
\paragraph{}
Despite the relationship between the $I$-divergence
$D_r(p,q)$ and the geodesic distance $d(pr,qr)$ there are some
important differences. The geodesic distance not only enjoys the symmetry
and triangle inequality properties, but is also bounded.
In contrast, the $I$-divergence grows to infinity - a fact
that causes it to be extremely non-robust. Indeed, in the
statistical literature, the maximum likelihood estimator is
often replaced by more robust estimators, among them the minimum
Hellinger distance estimator \cite{Beran1977,Lindsay1994}.
Interestingly, the Hellinger distance is extremely similar to the
geodesic distance under the Fisher information metric.
It is likely that new techniques in conditional inference that are based
on minimum geodesic distance in the primal space, will perform better than
maximum entropy or conditional exponential models.

\paragraph{}
Another interesting aspect is that
maximum entropy or conditional exponential models
may be interpreted as transforming models $p$ into
$rp$ where $r$ is the empirical distribution of the training set.
This makes sense since two models $rp,rq$ become identical over
$x_i$ that do not appear in the training set, and indeed the lack
of reference data makes such an embedding workable.
It is conceivable, however, to consider embeddings $p\mapsto rp$
using distributions $r$ different from the empirical training data distribution.
Different $x_i$ may have different
importance associated with their prediction $p(\cdot|x_i)$
and some labels $y_i$ may be known to be corrupted by noise
with a distribution that depends on $i$.

\paragraph{}
So far, we examined the geometry of the model space $\Theta$.
In the remainder of this thesis we turn to studying
machine learning algorithms in the context of geometric assumption
on the data space $\cX$.

%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Data Geometry Through the Embedding Principle}
\label{sec:embedding}
The standard practice in machine learning is to represent
data points as vectors in a Euclidean space, and then process
them under the assumptions of Euclidean geometry.
Such an embedding is often done regardless of the origin of
the data. It is a common practice for inherently real valued data,
such as measurements of physical quantities as well as for
categorical data such as text documents or boolean data.

\paragraph{} %Examples of such assumptions
Two classes of learning algorithms, which make such assumptions,
are radial basis machines and linear classifiers.
Radial basis machines are algorithms that are based on the
radial basis or Gaussian function
\[ K(x,y) =
c \exp\left( \frac{1}{\sigma^2}\|x-y\|^2\right)=
c \exp\left( \frac{1}{\sigma^2}\sum_i |x_i-y_i|^2\right)
\]
which is in turn based on the Euclidean normed distance.
Linear classifiers, such as boosting, logistic regression, linear SVM and
the perceptron make an implicit Euclidean assumption by choosing the class of linear
hyperplane separators. Furthermore, the training phase of
many linear classifiers is based on Euclidean arguments such as
the margin. Section~\ref{sec:hyperplane} contains more details
on this point.

\paragraph{}
In accordance with the earlier treatment of the model space geometry, we
would like to investigate a more appropriate geometry for $\cX$
and its implications on practical algorithms.
It is likely that since data come from different sources,
the appropriate geometry should be problem dependent.
More specifically, the data should dictate first the topological space
and then the geometric structure to endow it with.
It seems that selecting a geometry for the data space
is much harder than for the model space, where \v{C}encov's
theorem offered a natural candidate.

\paragraph{}
The first step toward selecting a metric for $\cX$ is assuming that
the data comes from a set of distributions
$\{p(x\,;\theta):\theta\in\Theta\}$. In this case we assume that
every data point is associated with a potentially different
distribution. Since there is already a natural geometric structure
on the model space $\Theta$, we can obtain a geometry for the data
points by identifying them with the corresponding  probabilistic
models and taking the Fisher geometry on $\Theta$.

\paragraph{}
Under a frequentist interpretation, we can associate each data point
with a single model $\theta$. An obvious choice for the
mapping $x\mapsto \theta$ is the maximum likelihood estimator
$\hat\theta:\cX\to\Theta$, which enjoys both nice properties
and often satisfactory performance.
We can then measure geometric quantities such as
distance between two data points $x,y$ by their geometric counterparts
on $\Theta$ with respect to the Fisher geometry
\begin{align} \label{eq:dist-embed-freq}
d(x,y)=d_{\cJ}(\hat\theta(x),\hat\theta(y)).
\end{align}

\paragraph{}
Under a Bayesian interpretation, we can associate a posterior
$p(\theta\given x)$ with each data point $x$. The quantities that correspond
to geometric measurements on $\Theta$ transform then into random variables.
For example, distance between two data points $x,y$ becomes a
random variable whose posterior mean is
\begin{align} \label{eq:dist-embed-Bayes}
E_{\theta|x} d(x,y) = \iint d_{\cJ}(\theta,\eta) p(\theta\given x)
p(\eta\given x) \dd d \theta \dd d \eta.
\end{align}

\paragraph{}
While there is a clearly
defined geometry on $\Theta$, the resulting concepts on $\cX$ may be
different from what is expected by a geometry on $\cX$.
For example, the functions
in \eqref{eq:dist-embed-freq}-\eqref{eq:dist-embed-Bayes} may not be
metric distance function (satisfying positivity, symmetry and triangle inequality)
on $\cX$,
as happens for example when $\hat\theta$ is not injective.
Another possible disparity occurs if $\hat\theta$ is not surjective and
significant areas of $\Theta$ are not represented by the data.

\paragraph{}
As a result, we will be more interested in cases where $\hat\theta$ is
injective and its image $\hat\theta(\cX)$ is dense in $\Theta$.
Such is the case of text document representation, the main
application area of this thesis.
A common assumption for text document representation is to disregard
the order of the words. This assumption, termed bag of words
representation, is almost always used for text classification task.
Under this assumption, a document is represented as a vector
of word counts $x\in\N^{|V|}$ where $V$ is the set of distinct words
commonly known as the dictionary.
In order to treat long and short documents on equal footing, it is
furthermore common to divide the above representation by the length
of the document resulting in a non-negative vector that sum to 1.
This is the representation that we assume in this thesis, and from now
on we assume that text documents
$x\in \cX$ are given in this representation.


\paragraph{}
Assuming that a multinomial model generates the documents, we find a close
relationship between $\cX$ and $\Theta$.
The data space $\cX$ is in fact a subset of the non-negative simplex $\overline{\Pn}$
which is precisely the model space $\Theta$ (in this case, $n+1$
is the size of the dictionary).
More specifically, $\cX$ is the subset of $\overline{\Pn}=\Theta$
with all rational coordinates
\[\cX = \overline{\Pn}\cap \mathbb{Q}^{n+1}.\]
Identifying $\cX$ as a subset of $\overline{\Pn}$,
we see that the maximum likelihood mapping $\hat\theta:\cX\to\Theta$
is the injective inclusion map $\iota:\cX\to\Theta, \iota(x)=x$, and
\[\Theta=\overline{\Pn}=
\overline{\overline{\Pn}\cap \mathbb{Q}^{n+1}}
=\overline{\cX}.\]
In other words, the maximum likelihood mapping is an injective
embedding of $\cX$ onto a dense set in $\Theta$.


\paragraph{}
Since $\cX$ is nowhere dense, it is not suitable for
continuous treatment. It should, at the very least
be embedded
in a complete space in which every Cauchy sequence convergence.
Replacing the document space $\cX$ by its completion $\overline{\Pn}$,
as we propose above,
is the smallest embedding that is sufficient for continuous treatment.

\paragraph{}
Once we assume that the data $x_i$ is sampled from a model
$p(x\,;\theta_i^{\text{true}})$ there is some justification in
replacing the data by points $\{\theta_i^{\text{true}}\}_i$ on a
Riemannian manifold $(\Theta,\cJ)$. Since the models generated the
data, in some sense they contain the essence of the data and the
Fisher geometry is motivated by \v{C}encov's theorem. The weakest
part of the embedding framework, and apparently the most arbitrary,
is the choice of the particular embedding. In the next two
subsections, we address this concern by examining the properties of
different embeddings.

%-------------------------------------------------------------
\subsection{Statistical Analysis of the Embedding Principle}
As mentioned previously, one possible embedding is the maximum
likelihood embedding $\hat\theta^{\text{mle}}:\cX\to\Theta$. The
nice asymptotic properties that the MLE enjoys seem to be irrelevant
to our purposes since it is employed for a single data point. If we
assume that the data parameters are clustered with a finite number
of clusters $\theta_1^{\text{true}},\cdots,\theta_C^{\text{true}}$,
then as the total number of examples increases, we obtain increasing
numbers of data points per cluster. In this case the MLE may enjoy
the asymptotic properties of first order efficiency. If no
clustering exists, then the number of parameters grow proportionally
to the number of data points. Such is the situation in
non-parametric statistics which becomes a more relevant framework
than parametric statistics.

\paragraph{}
Before we continue, we review some definitions from classical
decision theory. For more details, see for example
 \cite{Schervish1995}. Given an estimator $T:\cX^n\to\Theta$, and a
loss function $l:\Theta\times\Theta\to\R$, we can define the risk
\begin{align}
r_T:\Theta\to\R\qquad r_T(\theta) = \int_{\cX^n}
p(x_1,\ldots,x_n\,;\theta)\,\, l(T(x_1,\ldots,x_n),\theta)\, \,
\text{d}x_1\ldots\text{d}x_n.
\end{align}
The risk $r_T(\theta)$ measures the average loss of the estimator
$T$ assuming that the true parameter value is $\theta$. If there
exists another estimator $T'$ that dominates $T$ i.e.
\[ \quad \forall \theta \quad
r_{T'}(\theta)\leq r_T(\theta)\quad \text{ and } \quad \exists
\theta\quad  r_{T'}(\theta)< r_T(\theta)\] we say that the estimator
$T$ is inadmissable. We will assume from now on that the loss
function is the mean squared error function.

\paragraph{}
Assuming no clustering, the admissability of the MLE estimator is
questionable, as was pointed by James and Stein who stunned the
statistics community by proving that the James-Stein estimator
$\hat{\theta}^{\text{JS}}:\R\to\R$ for $N(\theta,1)$ given by
\[\hat{\theta}^{\text{JS}}(x_i)=\left(1-\frac{n-2}{\sum_i x_i^2}\right)x_i\]
dominates\footnote{Interestingly, the James-Stein estimator is
inadmissable as well as it is further dominated by another
estimator.} the MLE in mean squared error for $n\geq 3$
\cite{Stein1955,Efron1977}. Here the data $x_1,\ldots,x_n$ is
sampled from $N(\mu_1,1),\ldots,N(\mu_n,1)$ and the parameter space
is $\Theta=\R$. James-Stein estimator and other shrinkage estimators
are widely studied in statistics. Such estimators, however, depend
on the specific distribution being estimated and are usually
difficult to come up with.

\paragraph{}
A similar result from a different perspective may be obtained by
considering the Bayesian framework. In this framework, $\theta_i$
are iid random variables with a common, but unknown, distribution
$p(\theta|\phi)=\prod_i p(\theta_i|\phi)$. If $\phi$ is unknown our
marginal distribution is a mixture of iid distributions\footnote{the
mixture of iid distribution is also motivated by de Finetti's
theorem that singles it out as the only possibly distribution if
$n\to\infty$ and $\theta_i$ are exchangeable, rather than
independent}
\[p(\theta) = \int\prod_i p(\theta_i|\phi) p(\phi)\, \text{d}\phi.\]

\paragraph{}
Introducing the data $x_i$, we then have the following form for the
posterior \begin{align} \label{eq:bayesianEmbedding}
p(\theta_1,\ldots,\theta_n|x_1,\ldots x_n) \propto \prod_i
p(x_i|\theta_i) \int\prod_i p(\theta_i|\phi) p(\phi)\, \text{d}\phi
= \int\prod_i p(x_i|\theta_i) p(\theta_i|\phi) p(\phi)\,
\text{d}\phi.
\end{align}
Recall that in the Bayesian setting, the embedding becomes a random
variable. Furthermore, as seen from equation
\eqref{eq:bayesianEmbedding} if $\phi$ is unknown, the embedding of
the data $x_1,\ldots,x_n$ cannot be decoupled into independent
random embeddings and the posterior of the entire parameter set has
to be considered.

\paragraph{}
In the Bayesian framework, our distributional assumptions dictate
the form of the posterior and there is no arbitrariness such as a
selection of a specific point estimator. This benefit, as is often
the case in Bayesian statistics, comes at the expense of added
computational complexity. Given a data set $x_1,\ldots,x_n$, the
distance between two data points becomes a random variable. Assuming
we want to use a nearest neighbor classifier, a reasonable thing
would be to consider the posterior mean of the geodesic distance
$E_{\theta|x} d(x_i,x_j)$. If $\phi$ is unknown, equation
\eqref{eq:dist-embed-Bayes} should be replaced with the following
high dimensional integral
\[E_{\theta|x} d(x_i,x_j) \propto
\iint \prod_i p(x_i|\theta_i) p(\theta_i|\phi) p(\phi)\,
\text{d}\phi \, d(\theta_i,\theta_j)\, \text{d}\theta.\]

\paragraph{}
The empirical Bayes framework achieves a compromise between the
frequentist and the Bayesian approaches
\cite{Robbins1955,Casella1985}. In this approach, a deterministic
embedding $\hat\theta$ for $x_i$ is obtained by using the entire
data set $x_1,\ldots,x_n$ to approximate the regression function
$E(\theta_i|x_i)$. For some restricted cases, the empirical Bayes
procedure enjoys asymptotic optimality. However, both the analysis
and the estimation itself depend heavily on the specific case at
hand.


\paragraph{}
In the next sections we use the embedding principle to improve
several classification algorithms. In Section~\ref{sec:embedding} we
propose a generalization of the radial basis function, adapted to
the Fisher geometry of the embedded data space. In
Section~\ref{sec:hyperplane} we define the class of hyperplane
separators and margin quantity to arbitrary data geometries and work
out the detailed generalization of logistic regression to
multinomial geometry. Finally, Section~\ref{sec:metricLearning} goes
beyond the Fisher geometry and attempts to learn a local metric,
adapted to the provided training set. Experimental results  in these
sections show that the above generalizations of known algorithms to
non-Euclidean geometries outperform their Euclidean counterparts.




%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Diffusion Kernels on Statistical Manifolds}
\label{sec:diffusion}
The use of Mercer kernels for transforming linear classification and
regression schemes into nonlinear methods is a fundamental idea, one that
was recognized early in the development of statistical learning algorithms
such as the perceptron, splines, and support vector machines
\cite{Aizerman:64,Kimeldorf:71,Boser:92}.  The recent resurgence
of activity on kernel methods in the machine learning community has
led to the further development of this important technique,
demonstrating how kernels can be key components in tools
for tackling nonlinear data analysis problems, as well as for
integrating data from multiple sources.

\paragraph{}
Kernel methods can typically be viewed either in terms of an implicit
representation of a high dimensional feature space, or in terms of
regularization theory and smoothing \cite{Poggio:90}.  In either
case, most standard Mercer kernels such as the Gaussian or radial
basis function kernel require data points to be represented as vectors
in Euclidean space.  This initial processing of data as real-valued
feature vectors, which is often carried out in an {\it ad hoc\/}
manner, has been called the ``dirty laundry'' of machine learning
\emcite{Dietterich:02}---while the initial Euclidean feature representation
is often crucial, there is little theoretical guidance on how it should be
obtained.  For example in text classification, a standard procedure
for preparing the document collection for the application of learning
algorithms such as support vector machines is to represent each
document as a vector of scores, with each dimension corresponding to a
term, possibly after scaling by an inverse document frequency
weighting that takes into account the distribution of terms in the
collection \emcite{Joachims2000}.  While such a representation has
proven to be effective, the statistical justification of such a
transform of categorical data into Euclidean space is unclear.

\paragraph{}
Recent work by \emcite{KondorLafferty:02} was directly motivated by
this need for kernel methods that can be applied to
discrete, categorical data, in particular when the data lies on a
graph.  \emcite{KondorLafferty:02} propose the use of discrete diffusion
kernels and tools from spectral graph theory for data represented by
graphs.  In this section, we propose a related construction of kernels
based on the heat equation.  The key idea in our approach is to begin
with a statistical family that is natural for the data being analyzed,
and to represent data as points on the statistical manifold associated
with the Fisher information metric of this family. We then exploit the
geometry of the statistical family; specifically, we consider the heat
equation with respect to the Riemannian structure given by the
Fisher metric, leading to a Mercer kernel defined on the appropriate
function spaces.  The result is a family of kernels that generalizes
the familiar Gaussian kernel for Euclidean space, and that includes
new kernels for discrete data by beginning with statistical families
such as the multinomial.  Since the kernels are intimately based on
the geometry of the Fisher information metric and the heat or
diffusion equation on the associated Riemannian manifold, we refer to
them here as information diffusion kernels.

\paragraph{}
One apparent limitation of the discrete diffusion kernels of
\cite{KondorLafferty:02} is the difficulty of analyzing the
associated learning algorithms in the discrete setting.  This stems
from the fact that general bounds on the spectra of finite or even
infinite graphs are difficult to obtain, and research has
concentrated on bounds on the first eigenvalues for special families
of graphs. In contrast, the kernels we investigate here are over
continuous parameter spaces even in the case where the underlying
data is discrete, leading to more amenable spectral analysis.  We
can draw on the considerable body of research in differential
geometry that studies the eigenvalues of the geometric Laplacian,
and thereby apply some of the machinery that has been developed for
analyzing the generalization performance of kernel machines in our
setting.
\paragraph{}
Although the framework proposed is fairly general, in this section we
focus on the application of these ideas to text classification, where
the natural statistical family is the multinomial.  In the simplest
case, the words in a document are modeled as independent draws from a
fixed multinomial; non-independent draws, corresponding to
$n$-grams or more complicated mixture models are also
possible.  For $n$-gram models, the maximum likelihood multinomial
model is obtained simply as normalized counts, and smoothed estimates can
be used to remove the zeros.  This mapping is then used as an embedding
of each document into the statistical family, where the geometric
framework applies.  We remark that the perspective of associating
multinomial models with individual documents has recently been
explored in information retrieval, with promising results
\cite{Ponte:98a,Zhai:01a}.
\paragraph{}
The statistical manifold of the $n$-dimensional multinomial family
comes from an embedding of the multinomial simplex into the
$n$-dimensional sphere which is isometric under the Fisher
information metric.  Thus, the multinomial family can be viewed as a
manifold of constant positive curvature. As discussed below, there
are mathematical technicalities due to corners and edges on the
boundary of the multinomial simplex, but intuitively, the
multinomial family can be viewed in this way as a Riemannian
manifold with boundary; we address the technicalities by a
``rounding'' procedure on the simplex. While the heat kernel for
this manifold does not have a closed form, we can approximate the
kernel in a closed form using the leading term in the parametrix
expansion, a small time asymptotic expansion for the heat kernel
that is of great use in differential geometry.  This results in a
kernel that can be readily applied to text documents, and that is
well motivated mathematically and statistically.
\paragraph{}
We present detailed experiments for text classification, using both
the WebKB and Reuters data sets, which have become standard test
collections.  Our experimental results indicate that the multinomial
information diffusion kernel performs very well empirically.  This
improvement can in part be attributed to the role of the Fisher
information metric, which results in points near the boundary of the
simplex being given relatively more importance than in the flat
Euclidean metric. Viewed differently, effects similar to those
obtained by heuristically designed term weighting schemes such as
inverse document frequency are seen to arise automatically from the
geometry of the statistical manifold.

\paragraph{}
The  section is organized as follows.
In Section~\ref{sec:RiemannianGeom} we define the
relevant concepts from Riemannian geometry,
that have not been described in Section~\ref{sec:RiemannianGeometry}
and then proceed to define the heat kernel for a general manifold, together with its
parametrix expansion.  In Section~\ref{sec:bounds},
we derive bounds on
covering numbers and Rademacher averages for various learning
algorithms that use the new kernels, borrowing results from
differential geometry on bounds for the geometric Laplacian.
Section~\ref{sec:experimentsDiff} describes the results of applying the
multinomial diffusion kernels to text classification, and
we conclude with a discussion of our results in Section~\ref{sec:discuss}.

%----------------------------------------------------------------
\subsection{Riemannian Geometry and the Heat Kernel}
\label{sec:RiemannianGeom}

We begin by briefly reviewing some relevant concepts from
Riemannian geometry that will be used in the construction of
information diffusion kernels.
These concepts complement the ones defined in Section~\ref{sec:RiemannianGeometry}.
Using these concepts,
the heat kernel is defined, and its basic properties are
presented. An
excellent introductory account of this topic is given by
\emcite{Rosenberg:97}, and an authoritative reference for
spectral methods in Riemannian geometry is
\cite{Schoen1994}.


\paragraph{}
The construction of our kernels is based on the Laplacian\footnote{As
described by \emcite{Nelson:68}, ``The Laplace operator in its various
manifestations is the most beautiful and central object in all of
mathematics.  Probability theory, mathematical physics, Fourier
analysis, partial differential equations, the theory of Lie groups,
and differential geometry all revolve around this sun, and its light
even penetrates such obscure regions as number theory and algebraic
geometry.''}.
One way to describe the appropriate generalization of the Euclidean Laplacian
to arbitrary Riemannian manifold is through the notions of
gradient and divergence.
The gradient of a function is defined as the vector field that satisfies
\begin{equation}
\grad: C^{\infty}(\cM,\R)\to \mathfrak{X}(\cM)\qquad
g_p(\grad f|_p, X_p) \;=\; X_p(f)
\end{equation}
for every vector field $X\in\mathfrak{X}(\cM)$ and every point $p\in \cM$.
In local coordinates, the gradient is given by
\begin{equation}
(\grad f|_p)_i = \sum_j [G^{-1}(p)]_{ij} \frac{\partial f(p)}{\partial x_j}
\end{equation}
where $G(p)$ is the gram matrix associated with the metric $g$ (see Section~\ref{sec:RiemannianGeometry}).

\paragraph{}
The divergence operator
\begin{align*}
\div:\mathfrak{X}(\cM)\to C^{\infty}(\cM,\R)
\end{align*}
is defined to be the adjoint of the gradient, allowing ``integration
by parts'' on manifolds with special structure.
In local coordinates, the divergence is the function
\begin{equation}
\div X\, (p) = \frac{1}{\sqrt{\det g(p)}} \sum_i \frac{\partial}{\partial x_i} \left(
\sqrt{\det g(p)}\, (X_p)_i\right)
\end{equation}
where $\det g(p)$ is the determinant\footnote{
Most definition of the divergence, require
the manifold to be oriented.
We ignore this issue because it is not important
to what follows and we will always work with
orientable manifolds.}
of the Gram matrix $G(p)$.


\paragraph{}
Finally, the Laplace-Beltrami operator or the Laplacian is defined by\footnote{There is no general agreement about the sign convention for the Laplacian. Many authors define the Laplacian as the negative of the present definition.}
\begin{equation}
\Delta:C^{\infty}(\cM,\R)\to C^{\infty}(\cM,\R)
\qquad \Delta = \div \circ \grad
\end{equation}
which in local coordinates is given by
\begin{equation}
\Delta f\,(p) =
\frac{1}{\sqrt{\det g(p)}} \sum_{ij} \frac{\partial}{\partial x_i}
\left([G(p)^{-1}]_{ij} \sqrt{\det g(p)} \,\frac{\partial f}{\partial x_j}\right).
\end{equation}
These definitions preserve the familiar intuitive interpretation of
the usual operators in Euclidean geometry; in particular, the gradient
$\grad f$
points in the direction of steepest ascent of $f$
 and the divergence $\div X$ measures
outflow minus inflow of liquid or heat flowing according to the vector field $X$.


\subsubsection{The Heat Kernel}

The Laplacian is used to model how heat
will diffuse throughout a geometric manifold;
the flow $f(x,t)$, at point $x$ and time $t$,
is governed by the following second order partial differential
equation with initial conditions
\begin{eqnarray}
\frac{\partial f}{\partial t} - \Delta f &=& 0 \\
f(x, 0) &=& f_0(x).
\end{eqnarray}
The value $f(x,t)$ describes the heat at
location $x$ and time $t$, beginning from an initial
distribution of heat given by $f_0(x)$ at time zero.
The heat or diffusion
kernel $K_t(x,y)$ is the solution to the heat equation $f(x,t)$ with
initial condition given by Dirac's delta function $\delta_y$.
As a consequence of the linearity of the heat equation,
the heat kernel can be used to generate the solution to the heat equation with
arbitrary initial conditions, according to
\begin{equation}
f(x,t) \;=\; \int_{\cM} K_t(x,y) \, f(y)\, \dd dy.
\end{equation}

\ignore{
As a simple special case, consider heat flow on the circle, or one-dimensional
sphere $M = S^1$.   Parameterizing the manifold by angle $\theta$, and letting
$f(\theta,t) = \sum_{j=0}^{\infty} a_j(t) \,\cos(j\theta)$ be the discrete
cosine transform of the solution to the heat equation, with initial conditions
given by $a_j(0) = a_j$, it is seen that the heat equation leads to the
equation
\begin{equation}
\sum_{j=0}^\infty \left(\frac{d\, }{dt}\,a_j(t) + j^2 a_j(t)\right) \,\cos(j\theta) = 0
\end{equation}
which is easily solved to obtain $a_j(t) = e^{-j^2 t}$ and therefore
$f(\theta,t) = \sum_{j=0}^{\infty} a_j \,e^{-j^2 t} \, \cos(j\theta)$.
As the time parameter $t$ gets large, the solution converges
to $f(\theta,t)\longrightarrow a_0$, which is the average
value of $f$; thus, the heat diffuses until the manifold
is at a uniform temperature.
To express the solution in terms of an integral kernel, note that by
the Fourier inversion formula
\begin{eqnarray}
f(\theta,t) &=& \sum_{j=0}^\infty \langle f, e^{ij\theta}\rangle \, e^{-j^2 t}\,
e^{ij\theta} \\
&=& \frac{1}{2\pi}\int_{S^1} \sum_{j=0}^\infty e^{-j^2t} e^{ij\theta}\,
e^{-ij\phi} \,f(\phi)\,d\phi
\end{eqnarray}
thus expressing the solution as
$f(\theta,t) = \int_{S^1} K_t(\theta, \phi)\, f(\phi)\, d\phi$
for the heat kernel
\begin{equation}
K_t(\phi,\theta) = \frac{1}{2\pi} \sum_{j=0}^\infty e^{-j^2 t} \cos\left(j(\theta-\phi)\right)
\end{equation}
This simple example shows several properties of
the general solution of the heat equation on a (compact) Riemannian
manifold; in particular, note that the eigenvalues of the kernel scale
as $\lambda_j \sim e^{-j^{2/d}}$ where the dimension in
this case is $d=1$.
}
\paragraph{}
When $\cM=\R$ with the Euclidean metric,
the heat kernel is the familiar Gaussian kernel, so that
the solution to the heat equation is expressed as
\begin{equation}
f(x,t) = \frac{1}{\sqrt{4\pi t}} \int_\R e^{-\frac{(x-y)^2}{4t}} f(y) \, \dd dy
\end{equation}
and it is seen that as $t\to\infty$, the heat diffuses
out ``to infinity'' so that $f(x,t)\to 0$.

\paragraph{}
When $\cM$ is compact the Laplacian has discrete eigenvalues
$0=\lambda_0 < \lambda_1 \leq \lambda_2 \cdots$ with
corresponding eigenfunctions $\phi_i$ satisfying
$\Delta\phi_i = -\lambda_i \phi_i$.
When the manifold has a boundary, appropriate
boundary conditions must be imposed in order for $\Delta$
to be self-adjoint.  Dirichlet boundary conditions
set $\left.\phi_i\right|_{\partial \cM} = 0$ and
Neumann boundary conditions require $\left.\frac{\partial\phi_i}{\partial\nu}
\right|_{\partial \cM} = 0$ where $\nu$ is the outer normal direction.
The following theorem summarizes the basic properties for the
kernel of the heat equation on $\cM$; we refer to
\cite{Schoen1994} for a proof.
\begin{thrm}
Let $\cM$ be a complete Riemannian manifold.  Then there exists
a function  $K\in C^\infty(\R_+ \times M\times M)$, called the heat
kernel, which satisfies the following properties
for all $x,y\in M$, with $K_t(\cdot, \cdot) = K(t,\cdot,\cdot)$
\begin{enumerate}
\item $\quad K_t(x,y) = K_t(y,x)$
\item $\quad \lim_{t\rightarrow 0} K_t(x,y) = \delta_x{(y)}$
\item $\quad \left(\Delta - \frac{\partial}{\partial t}\right) K_t(x,y) = 0$
\item $\quad K_t(x,y) = \int_{\cM} K_{t-s}(x,z) K_s(z,y)\, \dd dz$ for any $s > 0$.
\end{enumerate}
If in addition $M$ is compact, then $K_t$ can be expressed in terms
of the eigenvalues and eigenfunctions of the Laplacian as
$K_t(x,y) = \sum_{i=0}^{\infty} e^{-\lambda_i t} \phi_i(x)\, \phi_i(y)$.
\end{thrm}

Properties 2 and 3 imply
that $K_t(x,y)$ solves the heat equation in $x$, starting from
$y$. It follows that
$e^{t\Delta} f(x) = f(x,t) = \int_{M} K_t(x,y)\,f(y)\,\dd dy$ solves the heat
equation with initial conditions $f(x,0) = f(x)$, since
\begin{eqnarray}
\frac{\partial f(x,t)}{\partial t} &=& \int_M \frac{\partial K_t(x,y)}{\partial t}\, f(y)\,\dd dy \\
&=& \int_M \Delta K_t(x,y)\, f(y)\,\dd dy \\
&=& \Delta \int_M K_t(x,y)\, f(y)\, \dd dy \\
&=& \Delta f(x)
\end{eqnarray}
and $\lim_{t\rightarrow 0} f(x,t) = \int_M \lim_{t\rightarrow 0} K_t(x,y)\, dy = f(x)$.
Property~4 implies
that $e^{t\Delta}\kern.1ex e^{s\Delta} = e^{(t+s)\Delta}$, which has the
physically intuitive interpretation that heat diffusion for time $t$
is the composition of heat diffusion up to time~$s$ with heat diffusion for
an additional time $t-s$.
Since $e^{t\Delta}$ is a positive operator,
\begin{eqnarray}
\int_M\int_M K_t(x,y) f(x) f(y) \, \dd dx\, \dd dy &=&
% \int_M f(x) \,e^{t\Delta} f(x) \, \dd dx  \\
\dotp{f}{e^{t\Delta} f} \,\geq\, 0
\end{eqnarray}
and $K_t(x,y)$ is
positive-definite. In the compact case, positive-definiteness follows directly
from the expansion $K_t(x,y) = \sum_{i=0}^{\infty} e^{-\lambda_i t} \phi_i(x)\, \phi_i(y)$,
which shows that the eigenvalues of $K_t$ as
an integral operator are $e^{-\lambda_i t}$.
Together, these properties show that $K_t$ defines
a Mercer kernel.

\paragraph{}
The heat kernel $K_t(x,y)$ is a natural candidate for measuring the
similarity between points between $x,y\in\cM$, while respecting the
geometry encoded in the metric $g$.  Furthermore it is, unlike the
geodesic distance, a Mercer kernel -- a fact that enables its use in
statistical kernel machines.  When this kernel is used for
classification, as in our text classification experiments presented in
Section~\ref{sec:experimentsDiff}, the discriminant function $y_t(x) =
\sum_i \alpha_i \kern.1ex y_i \kern.1ex K_t(x, x_i)$ can be
interpreted as the solution to the heat equation with initial
temperature $y_0(x_i) = \alpha_i\, y_i$ on labeled data points $x_i$,
and $y_0(x)=0$ elsewhere.

\subsubsection{The parametrix expansion}
\label{sec:expansion}

For most geometries, there is no closed form solution for the heat
kernel.  However, the short time behavior of the solutions can be
studied using an asymptotic expansion, called the {\it parametrix
expansion\/}.  In fact, the existence of the heat kernel, as asserted
in the above theorem, is most directly proven by first showing the
existence of the parametrix expansion.  Although it is local, the
parametrix expansion contains a wealth of geometric information, and
indeed much of modern differential geometry, notably index theory, is
based upon this expansion and its generalizations.  In
Section~\ref{sec:experimentsDiff} we will employ the first-order
parametrix expansion for text classification.

\paragraph{}
Recall that the heat kernel on $n$-dimensional Euclidean space is given
by
\begin{equation}
K_t^{\mbox{\tiny Euclid}}(x,y)\; =\; (4\pi t)^{-\frac{n}{2}}
\exp\left( - \frac{\|x - y\|^2}{4t}\right)
\end{equation}
where $\|x-y\|^2 = \sum_{i=1}^n \left| x_i - y_i\right|^2$ is the squared
Euclidean distance between $x$ and $y$.  The parametrix expansion approximates
the heat kernel locally as a correction to this Euclidean heat kernel.
It is given by
\begin{equation}
P_t^{(m)}(x,y)  = (4\pi t)^{-\frac{n}{2}} \exp\left( - \frac{d^2(x,y)}{4t}\right)
\left( \psi_0(x,y)  + \psi_1(x,y) t + \cdots + \psi_m(x,y) t^m\right)
\label{eq:param}
\end{equation}
where $d$ is the geodesic distance and
$\psi_k$ are recursively obtained by solving the
heat equation approximately to order $t^m$, for small diffusion time $t$.
Denoting $K_t^{(m)}(x,y)=P_t^{(m)}(x,y)$ we thus obtain an approximation for
the heat kernel, that converges as  $t\to 0$ and $x\to y$.
For further details refer to \cite{Schoen1994,Rosenberg:97}.




\ignore{
\paragraph{}
Let $r = d(x,y)$ denote the length of the radial geodesic from $x$ to
$y\in V_x$ in the normal coordinates defined by the exponential map.
For any functions $f(r)$ and $h(r)$ of $r$, it can be shown that
\begin{eqnarray}
\Delta f &=& \frac{d^2 f}{d r^2} + \frac{d\left(\log \sqrt{\det g}\right)}{d r}
\frac{d f}{dr} \\
\Delta(f h) &=& f\Delta h + h \Delta f + 2 \frac{df}{dr} \frac{dh}{dr}
\end{eqnarray}
Starting from these basic relations, some calculus shows that
\begin{equation}
\left(\Delta - \frac{\partial}{\partial t}\right) P_t^{(m)} =
\left(t^m \, \Delta \psi_m \right) \left(4\pi t\right)^{-\frac{n}{2}} \exp\left(-\frac{r^2}{4t}\right)
\label{eq:approxheat}
\end{equation}
when $\psi_k$ are defined recursively as
\begin{eqnarray}
\label{eq:psi}
\psi_0 &=& \left(\frac{\sqrt{\det g}}{r^{n-1}}\right)^{-\frac{1}{2}} \\
\psi_k &=& r^{-k}\psi_0 \int_{0}^r \psi_0^{-1} \left(\Delta \phi_{k-1}
\right)\, s^{k-1} ds \kern10pt \mbox{for $k > 0$}
\end{eqnarray}
With this recursive definition of the functions $\psi_k$,
the expansion \eqref{eq:param}, which is defined only locally,
is then extended to all of $M\times M$
by smoothing with a ``cut-off function'' $\eta$, with the specification
that $\eta:\R_+\longrightarrow [0,1]$ is $C^\infty$ and
\begin{equation}
\eta(r) = \begin{cases}
0 & r \geq 1 \\
1 & r \leq \frac{1}{2}
\end{cases}
\end{equation}
Thus, the order-$m$ parametrix is defined as
\begin{equation}
K^{(m)}_t(x,y) = \eta(d(x,y)) \, P_t^{(m)}(x,y)
\end{equation}
As suggested by equation \eqref{eq:approxheat}, $K_t^{(m)}$ is
an approximate solution to the heat equation, and satisfies
$K_t(x,y) = K_t^{(m)}(x,y) + O(t^m)$
for $x$ and $y$ sufficiently close; in particular, the parametrix
is not unique.
For further details
we refer to \emcite{Schoen1994,Rosenberg:97}.
}

\paragraph{}
While the parametrix ${K}_t^{(m)}$ is not in general
positive-definite, and therefore does not define a Mercer kernel, it
is positive-definite for $t$ sufficiently small.  In particular,
define $f(t) = \min\text{spec} \left(K_t^{(m)}\right)$, where
$\min\text{spec}$ denotes the smallest eigenvalue.  Then $f$ is a
continuous function with $f(0) = 1$ since $K_0^{(m)} = I$.  Thus,
there is some time interval $[0,\epsilon)$ for which $K_t^{(m)}$
is positive-definite in case $t\in [0,\epsilon)$.

\paragraph{}
The following two basic examples illustrate the geometry of the Fisher
information metric and the associated diffusion kernel it induces on
a statistical manifold.
Under the Fisher information metric,
the spherical normal family corresponds to
a manifold of constant negative curvature, and the multinomial
corresponds to a manifold of constant positive curvature.  The
multinomial will be the most important example that we develop, and we
will report extensive experiments with the resulting kernels in
Section~\ref{sec:experimentsDiff}.


\paragraph{}
\def\myrho{r}
The heat kernel on the hyperbolic space
$\H^n$ has the following closed form \cite{Grigoryan98}.
For odd $n=2m+1$ it is given by
\begin{equation}
\label{eq:hyper_a}
K_t(x,x') =
\displaystyle
\frac{(-1)^{m}}{2^m\pi^m}\frac{1}{\sqrt{4\pi t}}
 \left(\frac{1}{\sinh \myrho}\frac{\partial}{\partial \myrho}\right)^m \exp\left(-m^2 t - \frac{\myrho^2}{4t}\right)
\end{equation}
and for even $n=2m+2$ it is given by
\begin{equation}
K_t(x,x') =
\label{eq:hyper_b}
\displaystyle
\frac{(-1)^{m}}{2^m\pi^m}\frac{\sqrt{2}}{\sqrt{4\pi t}^3}
 \left(\frac{1}{\sinh \myrho}\frac{\partial}{\partial \myrho}\right)^m
\int_{\myrho}^\infty \displaystyle \frac{s \exp\left(-\frac{(2m+1)^2t}{4}-\frac{s^2}{4t}\right)}{\sqrt{\cosh s - \cosh \myrho}} \, \text{d}s
\end{equation}
where $\myrho = d(x,x')$ is the geodesic distance between the two
points in $\H^n$ given by equation \eqref{eq:hypDist}. If only the
mean $\theta=\mu$ is unspecified, then the associated kernel is the
standard RBF  or Gaussian kernel. The hyperbolic geometry is
illustrated in Figure~\ref{fig:normalexamples} where decision
boundaries of SVM with the diffusion kernel are plotted for both
Euclidean and hyperbolic geometry.

\begin{figure}
\begin{center}
\begin{tabular}{ccc}
\includegraphics[width=2.7in]{diffusionFigures/normalExample2} &&
\includegraphics[width=2.7in]{diffusionFigures/normalExample1}
\end{tabular}
\caption{Example decision boundaries for a kernel-based classifier
using information diffusion kernels for spherical normal
geometry with $d=2$ (right), which has constant negative curvature,
compared with the standard Gaussian kernel for
flat Euclidean space (left).  Two data points are used, simply
to contrast the underlying geometries.  The curved decision boundary for
the diffusion kernel can be interpreted statistically by noting that
as the variance decreases the mean is known with increasing certainty.}
\label{fig:normalexamples}
\end{center}
\end{figure}


\paragraph{}

Unlike the explicit expression for the Gaussian geometry discussed above, there is
no explicit form for the heat kernel on the sphere, nor on the positive orthant
of the sphere.  We will therefore resort to the parametrix expansion to derive
an approximate heat kernel for the multinomial geometry.

\paragraph{}
\ignore{
Recall from Section~\ref{sec:expansion} that the parametrix is obtained according
to the local expansion given in equation \eqref{eq:param},
and then extending this smoothly to zero outside a neighborhood of
the diagonal, as defined by the exponential map.
As we have just derived, this results in the following
parametrix for the multinomial family:
\begin{equation}
P_t^{(m)}(\theta,\theta')  = (4\pi t)^{-\frac{n}{2}} \exp\left( - \frac{\arccos^2(
\sqrt{\theta}\cdot \sqrt{\theta'})}{t}\right)
\left( \psi_0(\theta,\theta')  + \cdots + \psi_m(\theta,\theta') t^m\right)
\end{equation}
The first-order expansion is thus obtained as
\begin{equation}
K_t^{(0)}(\theta,\theta')  = \eta(d(\theta,\theta'))\, P_t^{(0)}(\theta,\theta')
\end{equation}
}

For the $n$-sphere it can be shown \cite{Berger:71} that the function
$\psi_0$ of in the parametrix expansion,
which is the leading order correction of the
Gaussian kernel under the Fisher information metric, is given by
\begin{align*}
\psi_0(r) &= \left(\frac{\sqrt{\det g}}{r^{n-1}}\right)^{-\frac{1}{2}}
= \left(\frac{\sin r }{r}\right)^{-\frac{(n-1)}{2}} \\
&= 1 + \frac{(n-1)}{12}\,r^2 + \frac{(n-1)(5n-1)}{1440}\,r^4 + O(r^6).
%\label{sinexp}
\end{align*}
\ignore{
Thus, the leading order parametrix for the multinomial diffusion kernel is
\begin{equation}
P_t^{(0)}(\theta,\theta')  = (4\pi t)^{-\frac{n}{2}} \exp\left( - \frac{1}{4t} d^2(\theta,\theta')\right)
\left(\frac{\sin d(\theta,\theta')}{d(\theta,\theta')}\right)^{-\frac{(n-1)}{2}}
\end{equation}
}
In our experiments we approximate the diffusion kernel using $\psi_0\equiv 1$
and obtain
\begin{align}
K(\theta,\theta')  = (4\pi t)^{-\frac{n}{2}} e^{-\frac{1}{t}\,
\arccos^2\left(\sum_i \sqrt{\theta_i\theta_i'}\right)}.
\label{nosin}
\end{align}
In Figure~\ref{fig:examples} the kernel \eqref{nosin} is compared with the standard
Euclidean space Gaussian kernel for the case of the trinomial model,
$d=2$, using an SVM classifier.
\paragraph{}
\begin{figure}
\begin{center}
\begin{tabular}{ccc}
\includegraphics[width=2.7in]{diffusionFigures/multExample2} &&
\includegraphics[width=2.7in]{diffusionFigures/multExample1}
\end{tabular}
\end{center}
\caption{Example decision boundaries using support vector
machines with information diffusion kernels for trinomial geometry
on the 2-simplex (top right) compared with the standard Gaussian kernel (left).}
\label{fig:examples}
\end{figure}

\subsection{Rounding the Simplex}

The case of multinomial geometry poses some technical complications
for the analysis of diffusion kernels, due to the fact that the open
simplex is not complete, and moreover, its closure is not a
differentiable manifold with boundary.  Thus, it is technically not possible to
apply several results from differential geometry, such as bounds
on the spectrum of the Laplacian, as adopted in Section~\ref{sec:bounds}.
We now briefly describe a technical ``patch'' that allows us to derive
all of the needed analytical results, without sacrificing in practice any
of the methodology that has been derived so far.
The idea is to ``round the corners'' of $\overline{\Pn}$ to obtain a
compact manifold with boundary, and that closely
approximates the original simplex $\Pn$.

\paragraph{}
For $\epsilon > 0$, let $B_\epsilon(x) = \{ y\given \|x-y\| < \epsilon\}$
be the open Euclidean ball of radius $\epsilon$ centered at~$x$ and
$C_\epsilon(\P_n)$ be
\begin{equation}
C_\epsilon(\P_n) = \left\{ x\in\overline{\Pn} \;:\; B_\epsilon(x) \subset \Pn\right\}
\end{equation}
The  $\epsilon$-rounded simplex  is then defined as
the closure of
\begin{equation}
\P_n^\epsilon = \bigcup_{x\,\in\, C_\epsilon(\P_n)} B_\epsilon(x).
\end{equation}
The above rounding procedure that yields $\overline{\P_2^\epsilon}$ is suggested by
Figure~\ref{fig:rounding}.  Note that in general the
$\epsilon$-rounded simplex $\overline{\P_n^\epsilon}$ will contain points
with a single, but not more than one component having zero
probability and it forms a compact manifold with boundary
whose image under the isometry $F$ described above
is a compact submanifold with boundary of the $n$-sphere.
Since, by choosing $\epsilon$ small enough we can approximate
$\overline{\Pn}$ arbitrarily well (both in the Euclidean and geodesic distances),
no harm is done by assuming that we are dealing with a rounded
compact manifold with a boundary.

\begin{figure}
\begin{center}
\begin{tabular}{ccc}
\includegraphics[scale=0.22]{diffusionFigures/rounding1}&
\includegraphics[scale=0.22]{diffusionFigures/rounding2}&
\includegraphics[scale=0.22]{diffusionFigures/rounding3}
\end{tabular}
\end{center}
\caption{Rounding the simplex.  Since the closed simplex is not a manifold
with boundary, we carry out a ``rounding'' procedure to remove edges and corners.
The $\epsilon$-rounded simplex is the closure of the union of all $\epsilon$-balls
lying within the open simplex.}
\label{fig:rounding}
\end{figure}

%-------------------------------------------------------------
\subsection{Spectral Bounds on Covering Numbers and Rademacher Averages}
\label{sec:bounds}

We now turn to establishing bounds on the generalization performance
of kernel machines that use information diffusion kernels. We begin by
adopting the approach of \emcite{Guo2002}, estimating covering numbers by
making use of bounds on the spectrum of the Laplacian on a Riemannian
manifold, rather than on VC dimension techniques; these bounds in turn
yield bounds on the expected risk of the learning algorithms.  Our
calculations give an indication of how the underlying geometry
influences the entropy numbers, which are inverse to the covering
numbers.  We then show how bounds on Rademacher averages may be
obtained by plugging in the spectral bounds from differential
geometry.  The primary conclusion that is drawn from these analyses is
that from the point of view of generalization error bounds,
information diffusion kernels behave essentially the same as the
standard Gaussian kernel.

\subsubsection{Covering Numbers}

We begin by recalling the main result of \emcite{Guo2002}, modifying
their notation slightly to conform with ours.  Let $\cX\subset\R^d$
be a compact subset of $d$-dimensional Euclidean space, and suppose
that $K:\cX\times\cX\longrightarrow\R$ is a Mercer kernel.  Denote
by $\lambda_1\geq \lambda_2\geq \cdots \geq 0$ the eigenvalues of $K$,
i.e., of the mapping $f\mapsto \int_{\cX} K(\cdot, y)\, f(y)\, dy$,
and let $\psi_j(\cdot)$ denote the corresponding eigenfunctions.
We assume that $C_K \deff \sup_j \|\psi_j\|_\infty < \infty$.

\paragraph{}
Given $m$ points $x_i\in\cX$, the kernel hypothesis class for $\bb x = \{x_i\}$
with weight vector bounded by $R$ is defined as the collection
of functions on $\bb x$ given by
\begin{equation}
\cF_R(\bb x) = \left\{f :  f(x_i)  = \dotp{w}{\Phi(x_i)}\; \mbox{for some $\|w\|\leq R$}\right\}
\end{equation}
where $\Phi(\cdot)$ is the mapping from $M$ to feature space
defined by the Mercer kernel, and $\dotp{\cdot}{\cdot}$ and $\|\cdot\|$
denote the corresponding Hilbert space inner product and norm.
It is of interest to obtain uniform bounds on
the covering numbers $\cN(\epsilon,\cF_R(\bb x))$,
defined as the size of the smallest $\epsilon$-cover
of $\cF_R(\bb x)$ in the metric induced by the norm
$\|f\|_{\infty, \bb x} = \max_{i=1,\ldots,m} |f(x_i)|$.
The following is the main result of \cite{Guo2002}.

\begin{thrm}
Given an integer $n\in\N$, let $j_n^*$ denote the
smallest integer $j$ for which
\begin{equation}
\lambda_{j+1} < \left(\frac{\lambda_1
\cdots \lambda_j}{n^2}\right)^{\textstyle \frac{1}{j}}
\end{equation}
and define
\begin{equation}
\epsilon_n^*
= 6 C_K R
\sqrt{j_{n}^{*}
\left({\frac{\lambda_1 \cdots \lambda_{j_n^*}}{n^2}}\right)^{\textstyle\frac{1}{j_n^*}}
+ \sum_{i={j_n^*}}^{\infty} \lambda_i}\,.
\end{equation}
Then $\sup_{\{x_i\}\in M^m} \cN(\epsilon_n^*, \cF_R(\bb x)) \leq n$.
\end{thrm}

\vskip5pt
To apply this result, we will obtain bounds on the
indices $j_n^*$ using spectral theory in Riemannian geometry.
The following bounds on the eigenvalues of the Laplacian
are due to \cite{LiYau:80}.

\def\dsq{D^2}
\def\dsq{V^{\frac{2}{d}}}
\begin{thrm}
\label{thm:eigenbounds}
Let $M$ be a compact Riemannian manifold of dimension $d$ with
non-negative Ricci curvature,
%$\mbox{Ric}(M)\geq 0$.
%and assume that the boundary of $M$ is convex.
and let $\,0 < \mu_1 \leq \mu_2\leq \cdots$ denote the eigenvalues of the
Laplacian with Dirichlet boundary conditions. Then
\begin{equation}
\label{libounds}
c_1(d) \left(\frac{j}{V}\right)^{\frac{2}{d}} \leq\; \mu_j \;\leq \;
c_2(d) \left(\frac{j+1}{V}\right)^{\frac{2}{d}}
\end{equation}
%\begin{equation}
%c'_1(d) \left(\frac{j+1}{V}\right)^{\frac{2}{d}} \leq \; \eta_j \leq \;
%c'_2(d) \left(\frac{j+2}{V}\right)^{\frac{2}{d}}
%\end{equation}
where $V$ is the volume of $M$ and $c_1$ and
$c_2$ are constants depending only on the dimension.
\end{thrm}

Note that the manifold of the multinomial model satisfies the
conditions of this theorem.  Using these results we can establish the
following bounds on covering numbers for information diffusion
kernels.  We assume Dirichlet boundary conditions; a similar result
can be proven for Neumann boundary conditions.  We include the
constant $V=\mbox{vol}(M)$ and diffusion coefficient $t$ in order to
indicate how the bounds depend on the geometry.

\def\ts{\textstyle}

\begin{thrm}
Let $M$ be a compact Riemannian manifold, with volume $V$,
satisfying the conditions of Theorem~\ref{thm:eigenbounds}.
Then the covering numbers for the Dirichlet heat kernel $K_{\kern-1pt t}$
on $M$ satisfy
\begin{equation}
\label{eq:nbound}
\log \cN(\epsilon, \cF_R(\bb x)) = O\left(\left(\frac{V}{\ts t^{\frac{d}{2}}}\right)
\log^{\ts\frac{d+2}{2}}\left(\frac{1}{\epsilon}\right) \right)
\end{equation}
\end{thrm}

\begin{proof}
By the lower bound in Theorem~\ref{thm:eigenbounds}, the Dirichlet eigenvalues of the
heat kernel $K_t(x,y)$, which are given by $\lambda_j = e^{-t\mu_j}$, satisfy
$ \log \lambda_j \leq - {t c_1(d)} \left(\frac{j}{V}\right)^{\frac{2}{d}}$.
Thus,
\begin{equation}
- \frac{1}{j} \log \left(\frac{\lambda_1\cdots\lambda_j}{n^2}\right)
\;\geq\;
 \frac{t c_1}{j} \sum_{i=1}^{j} \left(\frac{i}{V}\right)^{\ts \frac{2}{d}}
 + \frac{2}{j} \log n
\;\geq\;
  {t c_1} \frac{d}{d+2} \left(\frac{j}{V}\right)^{\ts \frac{2}{d}}
  + \frac{2}{j} \log n
\end{equation}
where the second inequality comes from
$\sum_{i=1}^{j} i^p \geq \int_{0}^j x^p \, dx = \frac{j^{p+1}}{p+1} $.
Now using the upper bound of Theorem~\ref{thm:eigenbounds},
the inequality $j_n^* \leq j$ will hold if
\begin{equation}
{t c_2} \left(\frac{j+2}{V}\right)^{\ts \frac{2}{d}}  \;\geq\; -\log \lambda_{j+1} \;\geq\;
{t c_1} \frac{d}{d+2} \left(\frac{j}{V}\right)^{\ts \frac{2}{d}} +
  \frac{2}{j} \log n
\end{equation}
or equivalently
\begin{equation}
\frac{t c_2}{\dsq} \left( j (j+2)^{\ts \frac{2}{d}} -
            \frac{c_1}{c_2}\frac{d}{d+2} j^{\ts \frac{d+2}{d}}\right) \;\geq\; 2 \log n
\end{equation}
The above inequality will hold in case
\begin{equation}
j \geq \left\lceil\left(
   \frac{2\dsq}{t (c_2 - c_1\frac{d}{d+2})} \log n\right)^{\ts\frac{d}{d+2}}\right\rceil
\;\geq \left\lceil\left(
   \frac{\dsq (d+2)}{t c_1} \log n\right)^{\ts\frac{d}{d+2}}\right\rceil
\end{equation}
since we may assume that $c_2\geq c_1$;
thus, $ j_n^* \leq \left\lceil \overline c_1\left( \frac{\dsq}{t} \log
n \right)^{\frac{d}{d+2}}\right\rceil$ for a new constant $\overline c_1(d)$.
Plugging this bound on $j_n^*$ into the expression
for $\epsilon_n^*$ in Theorem~2 and using
\begin{equation}
\sum_{i={j_n^*}}^{\infty} e^{-i^\frac{2}{d}} =
O\left(e^{-{j_n^*}^\frac{2}{d}}\right)
\end{equation}
we have
after some algebra that
\begin{equation}
\log\left(\frac{1}{\epsilon_n}\right) =
\Omega\left(\left(\frac{t}{V^\frac{2}{d}}\right)^{\frac{d}{d+2}}
\log^{\frac{2}{d+2}} n\right)
\end{equation}
Inverting the above expression in $\log n$ gives
equation (\ref{eq:nbound}).
\end{proof}
We note that Theorem~4 of \cite{Guo2002} can be used to show
that this bound does not, in fact, depend on $m$ and $\bb x$.
Thus, for fixed $t$ the covering numbers scale as
$\log\cN(\epsilon, \cF) =
O\left(\log^{\frac{d+2}{2}} \left(\frac{1}{\epsilon}\right)\right)$,
and for fixed $\epsilon$ they scale as
$\log\cN(\epsilon, \cF) = O\left(t^{-\frac{d}{2}}\right)$
in the diffusion time $t$.

\subsubsection{Rademacher Averages}

We now describe a different family of generalization error bounds that
can be derived using the machinery of Rademacher averages
\emcite{Bartlett:02,Bartlett:02b}.  The bounds fall out directly from
the work of \cite{Mendelson:03} on computing local averages for
kernel-based function classes, after plugging in the eigenvalue
bounds of Theorem~3.
\paragraph{}
As seen above, covering number bounds are related
to a complexity term of the form
\begin{equation}
C(n) = \sqrt{j_{n}^{*}
\left({\frac{\lambda_1 \cdots \lambda_{j_n^*}}{n^2}}\right)^{\textstyle\frac{1}{j_n^*}}
+ \sum_{i={j_n^*}}^{\infty} \lambda_i}
\end{equation}
In the case of Rademacher complexities, risk bounds are instead
controlled by a similar, yet simpler expression of the form
\begin{equation}
C(r) = \sqrt{ j^*_r \,r
+ \sum_{i={j^*_r}}^{\infty} \lambda_i}
\end{equation}
where now $j^*_r$ is the smallest integer $j$ for which $\lambda_{j} < r$
\emcite{Mendelson:03}, with $r$ acting as a parameter bounding
the error of the family of functions.  To place this
into some context, we quote the following results from \cite{Bartlett:02b}
and \cite{Mendelson:03}, which apply to a family
of loss functions that includes the quadratic loss;
we refer to \cite{Bartlett:02b} for details on the technical conditions.
\paragraph{}
Let $(X_1,Y_1), (X_2,Y_2)\ldots,(X_n,Y_n)$ be an independent sample
from an unknown distribution~$P$ on ${\mathcal X}\times {\mathcal Y}$,
where ${\mathcal Y}\subset \R$.
For a given loss function $\ell:{\mathcal Y}\times {\mathcal
Y}\rightarrow \R$, and a family $\cF$ of measurable functions
$f:{\mathcal X}\rightarrow {\mathcal Y}$, the objective is to minimize
the expected loss $E[\ell(f(X),Y)]$. Let $E\ell_{f^*} = \inf_{f\in\cF}
E\ell_f$, where $\ell_f(X,Y) = \ell(f(X),Y)$, and let $\hat f$
be any member of $\cF$ for which $E_n \ell_{\hat f} = \inf_{f\in\cF} E_n
\ell_f$ where $E_n$ denotes the empirical expectation. The {\it
Rademacher average\/} of a family of functions
$\mathfrak{G} = \left\{g: {\mathcal X}\rightarrow \R\right\}$ is
defined as the expectation $ER_n \mathfrak{G} = E\left[\sup_{g\in\mathfrak{G}} R_n
g\right]$ with $R_n g = \frac{1}{n} \sum_{i=1}^n \sigma_i \,g(X_i)$,
where $\sigma_1,\ldots,\sigma_n$ are independent {\it Rademacher\/}
random variables; that is, $p(\sigma_i=1) = p(\sigma_i=-1) =
\frac{1}{2}$.

\begin{thrm}
Let $\cF$ be a convex class of functions and define
$\psi$ by
\begin{equation}
\psi(r)  = a\, ER_n\left\{ f\in\cF : E(f-f^*)^2 \leq r\right\} + \frac{b\, x}{n}
\end{equation}
where $a$ and $b$ are constants that depend on the loss function $\ell$.
Then when $r\geq \psi(r)$,
\begin{equation}
E\left(\ell_{\hat f} - \ell_{f^*}\right) \leq c\,r + \frac{d\, x}{n}
\end{equation}
with probability at least $1-e^{-x}$,
where $c$ and $d$ are additional constants.

Moreover, suppose that $K$ is a Mercer kernel and
$\cF = \left\{ f\in {\mathcal H}_K : \|f\|_K \leq 1\right\}$
is the unit ball in the reproducing kernel
Hilbert space associated with $K$.  Then
\begin{eqnarray}
\psi(r) \leq a
\sqrt{\frac{2}{n} \sum_{j=1}^\infty \min\{r,\lambda_j\}}
+ \frac{b x}{n}
\end{eqnarray}
\end{thrm}

Thus, to bound the excess risk for kernel machines in this framework
it suffices to bound the term
\begin{equation}
\widetilde\psi(r) = \sqrt{\sum_{j=1}^\infty \min\{r,\lambda_j\}}
\end{equation}
involving the spectrum. Given bounds on the eigenvalues, this
is typically easy to do.

\begin{thrm}
Let $M$ be a compact Riemannian manifold,
satisfying the conditions of Theorem~\ref{thm:eigenbounds}.
Then the Rademacher term $\widetilde \psi$ for the Dirichlet heat kernel $K_{\kern-1pt t}$
on $M$ satisfies
\begin{equation}
\widetilde\psi(r) \leq C \sqrt{\left(\frac{r}{t^\frac{d}{2}}\right) \log^{\frac{d}{2}}
\left(\frac{1}{r}\right)}
\end{equation}
for some constant $C$ depending on the geometry of $M$.
\end{thrm}

\begin{proof}
We have that
\begin{eqnarray}
\widetilde\psi^2(r) &=& \sum_{j=1}^\infty \min\{r, \lambda_j\} \\
&= & j_r^* \, r + \sum_{j=j_r^*}^\infty e^{-t\mu_j} \\
&\leq & j_r^*\, r + \sum_{j=j_r^*}^\infty e^{-t c_1 j^{\frac{2}{d}}} \\
&\leq & j_r^*\, r + C e^{-t c_1 {j_r^*}^{\frac{2}{d}}}
\end{eqnarray}
for some constant $C$, where the first inequality follows
from the lower bound in Theorem~\ref{thm:eigenbounds}.  But
$j_r^* \leq j$ in case $\log \lambda_{j+1} > r$, or, again
from Theorem~\ref{thm:eigenbounds}, if
\begin{eqnarray}
t\,c_2(j+1)^{\frac{2}{d}} \leq -\log \lambda_j < \log\frac{1}{r}
\end{eqnarray}
or equivalently,
\begin{equation}
j_r^* \leq \frac{C'}{t^\frac{d}{2}}\log^{\frac{d}{2}}\left(\frac{1}{r}\right)
\end{equation}
It follows that
\begin{eqnarray}
\widetilde \psi^2(r) &\leq& C{''} \left(\frac{r}{t^{\frac{d}{2}}}\right)\,
\log^{\frac{d}{2}}\left(\frac{1}{r}\right)
\end{eqnarray}
for some new constant $C{''}$.
\end{proof}
From this bound, it can be shown that, with high probability,
\begin{eqnarray}
E\left(\ell_{\hat f} - \ell_{f^*}\right)
= O\left(\frac{\log^{\frac{d}{2}} n}{n}\right)
\end{eqnarray}
which is the behavior expected of the Gaussian kernel for Euclidean space.

\paragraph{}
Thus, for both covering numbers and Rademacher averages,
the resulting bounds are essentially the same as those
that would be obtained for the Gaussian kernel on the flat
$d$-dimensional torus, which is the standard way of ``compactifying''
Euclidean space to get a Laplacian having only discrete spectrum; the
results of \cite{Guo2002} are formulated for the case $d=1$,
corresponding to the circle.  While the bounds
for information diffusion kernels were derived for the case of
positive curvature, which apply to the special case of the
multinomial, similar bounds for general manifolds with curvature
bounded below by a negative constant should also be attainable.




%---------------------------------------------------------------
\subsection{Experimental Results for Text Classification} \label{sec:experimentsDiff}

In this section we present the application of multinomial diffusion
kernels to the problem of text classification.  Text processing can be
subject to some of the ``dirty laundry'' referred to in the
introduction---documents are cast as Euclidean space vectors with
special weighting schemes that have been empirically honed through
applications in information retrieval, rather than inspired from
first principles.  However for text, the use of multinomial geometry
is natural and well motivated; our experimental results offer some
insight into how useful this geometry may be for classification.

\paragraph{}
We consider several embeddings $\hat{\theta}:\cX\to\Theta$ of documents
in bag of words representation into the probability simplex.
The  term frequency (tf) representation uses normalized counts;
the corresponding embedding is the maximum likelihood estimator
for the multinomial distribution
\begin{equation}
\widehat{\theta}_{\text{tf}}(x) =
\left(\frac{x_1}{\sum_i x_i},\ldots,\frac{x_{n+1}}{\sum_i x_i} \right).
\end{equation}
Another common representation is based on  term frequency, inverse
document frequency (tf-idf). This representation uses the distribution
of terms across documents to discount common terms; the document frequency $df_v$
of term $v$ is defined as the number of documents in which term $v$ appears.
Although many variants have been proposed, one of the simplest
and most commonly used embeddings is
\begin{equation}
\widehat{\theta}_{\text{tf-idf}}(x) =
\left(\frac{x_1 \log (D/df_1)}{\sum_i x_i \log (D/df_i)}
,\ldots, \frac{x_{n+1} \log (D/df_{n+1})}{\sum_i x_i \log (D/df_i)}\right)
\end{equation}
where $D$ is the number of documents in the corpus.

\paragraph{}
In text classification applications the  tf and tf-idf representations
are typically normalized to unit length in the $L_2$ norm rather
than the $L_1$ norm, as above \emcite{Joachims2000}.
For example, the tf representation with $L_2$ normalization
is given by
\begin{equation}
x \mapsto
\left(\frac{x_1}{\sum_i x_i^2},\ldots,\frac{x_{n+1}}{\sum_i x_i^2} \right)
\end{equation}
and similarly for tf-idf.
When used in support vector machines with linear or Gaussian kernels,
$L_2$-normalized tf and tf-idf achieve higher accuracies than their
$L_1$-normalized counterparts. However, for the diffusion kernels,
$L_1$ normalization is necessary to obtain an embedding
into the simplex.  These different embeddings or feature representations
are compared in the experimental results reported below.

\paragraph{}
The three kernels that we compare are the linear
kernel
\begin{equation}
K^{\text{Lin}}(\theta,\theta') \sum_{v=1}^{n+1} \theta_v \,\theta'_v,
\end{equation}
the Gaussian kernel
\begin{equation}
K_\sigma^{\text{Gauss}}(\theta',\theta')\; =\; (2\pi \sigma)^{-\frac{n+1}{2}}
\exp\left( - \frac{\sum_{i=1}^{n+1}|\theta_i-\theta_i'|^2}{2\sigma^2}\right)
\end{equation}
and the multinomial diffusion kernel approximation
\begin{equation}
K_t^{\mbox{\tiny Mult}}(\theta,\theta')
= (4\pi t)^{-\frac{n}{2}} \exp\left( -\frac{1}{t}\, \arccos^2\left(
\sum_{i=1}^{n+1} \sqrt{\theta_i\theta_i'}\right)\right).
\end{equation}

\paragraph{}
In our experiments, the multinomial diffusion kernel using the tf embedding
was compared to the linear or Gaussian kernel with tf and tf-idf embeddings
using a support vector machine classifier on the WebKB and
Reuters-21578 collections, which are standard data sets for text classification.
\paragraph{}
The WebKB dataset contains web pages found on the sites of four
universities \emcite{Craven1998}. The pages were classified
according to whether they were student, faculty, course, project or
staff pages; these categories contain 1641, 1124, 929, 504 and 137
instances, respectively.  Since only the \texttt{student},
\texttt{faculty}, \texttt{course} and \texttt{project} classes
contain more than 500 documents each, we restricted our attention to
these classes. The Reuters-21578 dataset is a collection of newswire
articles classified according to news topic \emcite{Lewis94}.
Although there are more than 135 topics, most of the topics have
fewer than 100 documents; for this reason, we restricted our
attention to the following five most frequent classes:
\texttt{earn}, \texttt{acq}, \texttt{moneyFx}, \texttt{grain} and
\texttt{crude}, of sizes 3964, 2369, 717, 582 and 578 documents,
respectively.
\paragraph{}
For both the WebKB and Reuters collections we created two types of
binary classification tasks. In the first task we designate a
specific class, label each document in the class as a ``positive''
example, and label each document on any of the other topics as a
``negative'' example.  In the second task we designate a class as the
positive class, and choose the negative class to be the most
frequent remaining class (\texttt{student} for WebKB and \texttt{earn}
for Reuters).  In both cases, the size of the training set is varied
while keeping the proportion of positive and negative documents
constant in both the training and test set.

\begin{figure}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-course-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-course-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-faculty-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-faculty-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-project-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-project-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-student-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-all/webkb-student-TFIDF.eps}\\
\end{tabular}
\end{center}
\caption{Experimental results on the WebKB corpus, using SVMs
for linear (dotted) and Gaussian (dash-dotted) kernels, compared with the
diffusion kernel for the multinomial (solid).  Classification
error for the task of labeling \texttt{course} (top row),
\texttt{faculty} (second row), \texttt{project} (third row), and
\texttt{student}(bottom row) is shown in these plots, as a function of training set
size.  The left plot uses tf representation and the
right plot uses tf-idf representation.
The curves shown are the error
rates averaged over 20-fold cross validation.
The results for the other ``1 vs.~all'' labeling tasks are qualitatively
similar, and therefore not shown.}
\label{fig:experiments1}
\end{figure}

\begin{figure}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/course-vs-student-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/course-vs-student-CV-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/faculty-vs-student-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/faculty-vs-student-CV-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/project-vs-student-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/webkb-1-vs-student/project-vs-student-CV-TFIDF.eps}\\
\end{tabular}
\end{center}
\caption{Results on the WebKB corpus, using SVMs
for linear (dotted) and Gaussian (dash-dotted) kernels, compared with the
diffusion kernel (solid).
The tasks are
\texttt{course} vs.  \texttt{student} (top row),
\texttt{faculty} vs.  \texttt{student} (top row) and
\texttt{project} vs.  \texttt{student} (top row).
The left plot uses tf representation and the
right plot uses tf-idf representation.
Results for
other label pairs are qualitatively similar.
 The curves shown are the error
 rates averaged over 20-fold cross validation.}
\label{fig:experiments4}
\end{figure}



\begin{figure}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-earn-vs-all-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-earn-vs-all-CV-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-acq-vs-all-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-acq-vs-all-CV-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-moneyFx-vs-all-CV-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-all/reuters-moneyFx-vs-all-CV-TFIDF.eps}\\
\end{tabular}
\end{center}
\caption{Experimental results on the Reuters corpus, using SVMs
for linear (dotted) and Gaussian (dash-dotted) kernels, compared with the
diffusion kernel (solid).  The
tasks are classifying \texttt{earn} (top row), \texttt{acq} (second row),
\texttt{moneyFx} (bottom row) vs. the rest.
Plots for the other classes are qualitatively similar.
The left column uses tf representation and the right column uses tf-idf.
The curves shown are the error rates averaged over 20-fold cross validation.}
\label{fig:experiments2}
\end{figure}

\begin{figure}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/acq-vs-earn-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/acq-vs-earn-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/moneyFx-vs-earn-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/moneyFx-vs-earn-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/grain-vs-earn-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/grain-vs-earn-TFIDF.eps}\\
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/crude-vs-earn-TF.eps}&
\includegraphics[scale=0.35]{diffusionFigures/reuters-1-vs-earn/crude-vs-earn-TFIDF.eps}\\
\end{tabular}
\end{center}
\caption{Experimental results on the Reuters corpus, using SVMs
for linear (dotted) and Gaussian (dash-dotted) kernels, compared with the
diffusion (solid).
The tasks are
\texttt{acq} vs. \texttt{earn} (top row),
\texttt{moneyFx} vs. \texttt{earn} (top row),
\texttt{grain} vs. \texttt{earn} (top row),
\texttt{crude} vs. \texttt{earn} (top row).
The left column uses tf representation and the
right column uses tf-idf representation.
The left column uses tf representation and the right column uses tf-idf.
The curves shown are the error rates averaged over 20-fold cross validation.}
\label{fig:experiments5}
\end{figure}


\paragraph{}
Figure~\ref{fig:experiments1} shows the test set error rate
for the WebKB data, for a representative instance of the
one-versus-all classification task; the designated class
was \texttt{course}. The results for the other choices
of positive class were qualitatively very similar; all
of the results are summarized in Table~\ref{tab:webkb1}.
Similarly, Figure~\ref{fig:experiments2}
shows the test set error rates for two of the one-versus-all experiments
on the Reuters data, where the designated classes were
chosen to be \texttt{acq} and \texttt{moneyFx}.
All of the results for Reuters one-versus-all tasks are
shown in Table~\ref{tab:reuters1}.

\def\nb#1{\hbox to 30pt{\hfill #1 \hfill}}
\def\bb#1{\hbox to 30pt{\hfill \bf #1 \hfill}}
\def\nb#1{#1}
\def\bb#1{{\bf #1}}
\def\str{${}^*$}
\renewcommand{\arraystretch}{1.1}

\begin{table}
\renewcommand{\arraystretch}{1.1}
\begin{center}
\begin{small}
\begin{tabular}{|c|r||ccc||ccc|}
\multicolumn{2}{c}{} & \multicolumn{3}{c}{tf Representation}
                     & \multicolumn{3}{c}{tf-idf Representation} \\[1pt]
\hline
 Task & $\;L\;$ & Linear & Gaussian & Diffusion & Linear & Gaussian & Diffusion \\
% WEBKB 1 vs all
\hline
  & 40 & \nb{0.1225} & \nb{0.1196} & \bb{0.0646}  & \nb{0.0761} & \nb{0.0726} & \bb{0.0514} \\
  & 80 & \nb{0.0809} & \nb{0.0805} & \bb{0.0469}  & \nb{0.0569} & \nb{0.0564} & \bb{0.0357} \\
course vs.~all
  & 120 & \nb{0.0675} & \nb{0.0670} & \bb{0.0383}  & \nb{0.0473} & \nb{0.0469} & \bb{0.0291} \\
  & 200 & \nb{0.0539} & \nb{0.0532} & \bb{0.0315}  & \nb{0.0385} & \nb{0.0380} & \bb{0.0238} \\
  & 400 & \nb{0.0412} & \nb{0.0406} & \bb{0.0241}  & \nb{0.0304} & \nb{0.0300} & \bb{0.0182} \\
  & 600 & \nb{0.0362} & \nb{0.0355} & \bb{0.0213}  & \nb{0.0267} & \nb{0.0265} & \bb{0.0162} \\
\hline
\hline
  & 40 & \nb{0.2336} & \nb{0.2303} & \bb{0.1859}  & \nb{0.2493} & \nb{0.2469} & \bb{0.1947} \\
  & 80 & \nb{0.1947} & \nb{0.1928} & \bb{0.1558}  & \nb{0.2048} & \nb{0.2043} & \bb{0.1562} \\
faculty vs.~all
  & 120 & \nb{0.1836} & \nb{0.1823} & \bb{0.1440}  & \nb{0.1921} & \nb{0.1913} & \bb{0.1420} \\
  & 200 & \nb{0.1641} & \nb{0.1634} & \bb{0.1258}  & \nb{0.1748} & \nb{0.1742} & \bb{0.1269} \\
  & 400 & \nb{0.1438} & \nb{0.1428} & \bb{0.1061}  & \nb{0.1508} & \nb{0.1503} & \bb{0.1054} \\
  & 600 & \nb{0.1308} & \nb{0.1297} & \bb{0.0931}  & \nb{0.1372} & \nb{0.1364} & \bb{0.0933} \\
\hline
\hline
  & 40 & \nb{0.1827} & \nb{0.1793} & \bb{0.1306}  & \nb{0.1831} & \nb{0.1805} & \bb{0.1333} \\
  & 80 & \nb{0.1426} & \nb{0.1416} & \bb{0.0978}  & \nb{0.1378} & \nb{0.1367} & \bb{0.0982} \\
project vs.~all
  & 120 & \nb{0.1213} & \nb{0.1209} & \bb{0.0834}  & \nb{0.1169} & \nb{0.1163} & \bb{0.0834} \\
  & 200 & \nb{0.1053} & \nb{0.1043} & \bb{0.0709}  & \nb{0.1007} & \nb{0.0999} & \bb{0.0706} \\
  & 400 & \nb{0.0785} & \nb{0.0766} & \bb{0.0537}  & \nb{0.0802} & \nb{0.0790} & \bb{0.0574} \\
  & 600 & \nb{0.0702} & \nb{0.0680} & \bb{0.0449}  & \nb{0.0719} & \nb{0.0708} & \bb{0.0504} \\
\hline
\hline
  & 40 & \nb{0.2417} & \nb{0.2411} & \bb{0.1834}  & \nb{0.2100} & \nb{0.2086} & \bb{0.1740} \\
  & 80 & \nb{0.1900} & \nb{0.1899} & \bb{0.1454}  & \nb{0.1681} & \nb{0.1672} & \bb{0.1358} \\
student vs.~all
  & 120 & \nb{0.1696} & \nb{0.1693} & \bb{0.1291}  & \nb{0.1531} & \nb{0.1523} & \bb{0.1204} \\
  & 200 & \nb{0.1539} & \nb{0.1539} & \bb{0.1134}  & \nb{0.1349} & \nb{0.1344} & \bb{0.1043} \\
  & 400 & \nb{0.1310} & \nb{0.1308} & \bb{0.0935}  & \nb{0.1147} & \nb{0.1144} & \bb{0.0874} \\
  & 600 & \nb{0.1173} & \nb{0.1169} & \bb{0.0818}  & \nb{0.1063} & \nb{0.1059} & \bb{0.0802} \\
\hline
\end{tabular}
\end{small}
\end{center}
\caption{Experimental results on the WebKB corpus, using SVMs for linear,
Gaussian, and multinomial diffusion kernels.  The left columns use
tf representation and the right columns use tf-idf
representation.  The error rates shown are averages obtained using
20-fold cross validation.  The best performance for each training set
size $L$ is shown in boldface.  All differences are statistically
significant according to the paired $t$
test at the 0.05 level.}
\label{tab:webkb1}
\end{table}


\begin{table}
\renewcommand{\arraystretch}{1.1}
\begin{center}
\begin{small}
\begin{tabular}{|c|r||ccc||ccc|}
\multicolumn{2}{c}{} & \multicolumn{3}{c}{tf Representation}
                     & \multicolumn{3}{c}{tf-idf Representation} \\[1pt]
\hline
 Task & $\;L\;$ & Linear & Gaussian & Diffusion & Linear & Gaussian & Diffusion \\
% WEBKB 1 vs student
\hline
  & 40 & \nb{0.0808} & \nb{0.0802} & \bb{0.0391}  & \nb{0.0580} & \nb{0.0572} & \bb{0.0363} \\
  & 80 & \nb{0.0505} & \nb{0.0504} & \bb{0.0266}  & \nb{0.0409} & \nb{0.0406} & \bb{0.0251} \\
course vs.~student
  & 120 & \nb{0.0419} & \nb{0.0409} & \bb{0.0231}  & \nb{0.0361} & \nb{0.0359} & \bb{0.0225} \\
  & 200 & \nb{0.0333} & \nb{0.0328} & \bb{0.0184}  & \nb{0.0310} & \nb{0.0308} & \bb{0.0201} \\
  & 400 & \nb{0.0263} & \nb{0.0259} & \bb{0.0135}  & \nb{0.0234} & \nb{0.0232} & \bb{0.0159} \\
  & 600 & \nb{0.0228} & \nb{0.0221} & \bb{0.0117}  & \nb{0.0207} & \nb{0.0202} & \bb{0.0141} \\
\hline
\hline
  & 40 & \nb{0.2106} & \nb{0.2102} & \bb{0.1624}  & \nb{0.2053} & \nb{0.2026} & \bb{0.1663} \\
  & 80 & \nb{0.1766} & \nb{0.1764} & \bb{0.1357}  & \nb{0.1729} & \nb{0.1718} & \bb{0.1335} \\
faculty vs.~student
  & 120 & \nb{0.1624} & \nb{0.1618} & \bb{0.1198}  & \nb{0.1578} & \nb{0.1573} & \bb{0.1187} \\
  & 200 & \nb{0.1405} & \nb{0.1405} & \bb{0.0992}  & \nb{0.1420} & \nb{0.1418} & \bb{0.1026} \\
  & 400 & \nb{0.1160} & \nb{0.1158} & \bb{0.0759}  & \nb{0.1166} & \nb{0.1165} & \bb{0.0781} \\
  & 600 & \nb{0.1050} & \nb{0.1046} & \bb{0.0656}  & \nb{0.1050} & \nb{0.1048} & \bb{0.0692} \\
\hline
\hline
  & 40 & \nb{0.1434} & \nb{0.1430} & \bb{0.0908}  & \nb{0.1304} & \nb{0.1279} & \bb{0.0863} \\
  & 80 & \nb{0.1139} & \nb{0.1133} & \bb{0.0725}  & \nb{0.0982} & \nb{0.0970} & \bb{0.0634} \\
project vs.~student
  & 120 & \nb{0.0958} & \nb{0.0957} & \bb{0.0613}  & \nb{0.0870} & \nb{0.0866} & \bb{0.0559} \\
  & 200 & \nb{0.0781} & \nb{0.0775} & \bb{0.0514}  & \nb{0.0729} & \nb{0.0722} & \bb{0.0472} \\
  & 400 & \nb{0.0590} & \nb{0.0579} & \bb{0.0405}  & \nb{0.0629} & \nb{0.0622} & \bb{0.0397} \\
  & 600 & \nb{0.0515} & \nb{0.0500} & \bb{0.0325}  & \nb{0.0551} & \nb{0.0539} & \bb{0.0358} \\
\hline
\end{tabular}
\end{small}
\end{center}
\caption{Experimental results on the WebKB corpus, using SVMs for linear,
Gaussian, and multinomial diffusion kernels.  The left columns use
tf representation and the right columns use tf-idf
representation.  The error rates shown are averages obtained using
20-fold cross validation.  The best performance for each training set
size $L$ is shown in boldface.
All differences are statistically significant according to the paired $t$
test at the 0.05 level.}
\label{tab:webkb2}
\end{table}

\begin{table}
\renewcommand{\arraystretch}{1.1}
\begin{center}
\begin{small}
\begin{tabular}{|c|r||ccc||ccc|}
\multicolumn{2}{c}{} & \multicolumn{3}{c}{tf Representation}
                     & \multicolumn{3}{c}{tf-idf Representation} \\[1pt]
\hline
 Task & $\;L\;$ & Linear & Gaussian & Diffusion & Linear & Gaussian & Diffusion \\
% Reuters 1 vs all
\hline
  & 80 & \nb{0.1107} & \nb{0.1106} & \bb{0.0971}  & \nb{0.0823} & \nb{0.0827} & \bb{0.0762} \\
  & 120 & \nb{0.0988} & \nb{0.0990} & \bb{0.0853}  & \nb{0.0710} & \nb{0.0715} & \bb{0.0646} \\
earn vs.~all
  & 200 & \nb{0.0808} & \nb{0.0810} & \bb{0.0660}  & \nb{0.0535} & \nb{0.0538} & \bb{0.0480} \\
  & 400 & \nb{0.0578} & \nb{0.0578} & \bb{0.0456}  & \nb{0.0404} & \nb{0.0408} & \bb{0.0358} \\
  & 600 & \nb{0.0465} & \nb{0.0464} & \bb{0.0367}  & \nb{0.0323} & \nb{0.0325} & \bb{0.0290} \\
\hline
\hline
  & 80 & \nb{0.1126} & \nb{0.1125} & \bb{0.0846}  & \nb{0.0788} & \nb{0.0785} & \bb{0.0667} \\
  & 120 & \nb{0.0886} & \nb{0.0885} & \bb{0.0697}  & \nb{0.0632} & \nb{0.0632} & \bb{0.0534} \\
acq vs.~all
  & 200 & \nb{0.0678} & \nb{0.0676} & \bb{0.0562}  & \nb{0.0499} & \nb{0.0500} & \bb{0.0441} \\
  & 400 & \nb{0.0506} & \nb{0.0503} & \bb{0.0419}  & \nb{0.0370} & \nb{0.0369} & \bb{0.0335} \\
  & 600 & \nb{0.0439} & \nb{0.0435} & \bb{0.0363}  & \nb{0.0318} & \nb{0.0316} & \bb{0.0301} \\
\hline
\hline
  & 80 & \nb{0.1201} & \nb{0.1198} & \bb{0.0758}  & \nb{0.0676} & \nb{0.0669} & \bb{0.0647\str} \\
  & 120 & \nb{0.0986} & \nb{0.0979} & \bb{0.0639}  & \nb{0.0557} & \nb{0.0545} & \bb{0.0531\str} \\
moneyFx vs.~all
  & 200 & \nb{0.0814} & \nb{0.0811} & \bb{0.0544}  & \nb{0.0485} & \nb{0.0472} & \bb{0.0438} \\
  & 400 & \nb{0.0578} & \nb{0.0567} & \bb{0.0416}  & \nb{0.0427} & \nb{0.0418} & \bb{0.0392} \\
  & 600 & \nb{0.0478} & \nb{0.0467} & \bb{0.0375}  & \nb{0.0391} & \nb{0.0385} & \bb{0.0369\str} \\
\hline
\hline
  & 80 & \nb{0.1443} & \nb{0.1440} & \bb{0.0925}  & \nb{0.0536} & \bb{0.0518\str} & \nb{0.0595} \\
  & 120 & \nb{0.1101} & \nb{0.1097} & \bb{0.0717}  & \nb{0.0476} & \bb{0.0467\str} & \nb{0.0494} \\
grain vs.~all
  & 200 & \nb{0.0793} & \nb{0.0786} & \bb{0.0576}  & \nb{0.0430} & \bb{0.0420\str} & \nb{0.0440} \\
  & 400 & \nb{0.0590} & \nb{0.0573} & \bb{0.0450}  & \nb{0.0349} & \bb{0.0340\str} & \nb{0.0365} \\
  & 600 & \nb{0.0517} & \nb{0.0497} & \bb{0.0401}  & \nb{0.0290} & \bb{0.0284\str} & \nb{0.0306} \\
\hline
\hline
  & 80 & \nb{0.1396} & \nb{0.1396} & \bb{0.0865}  & \nb{0.0502} & \bb{0.0485\str} & \nb{0.0524} \\
  & 120 & \nb{0.0961} & \nb{0.0953} & \bb{0.0542}  & \nb{0.0446} & \bb{0.0425\str} & \nb{0.0428} \\
crude vs.~all
  & 200 & \nb{0.0624} & \nb{0.0613} & \bb{0.0414}  & \nb{0.0388} & \nb{0.0373} & \bb{0.0345\str} \\
  & 400 & \nb{0.0409} & \nb{0.0403} & \bb{0.0325}  & \nb{0.0345} & \nb{0.0337} & \bb{0.0297} \\
  & 600 & \nb{0.0379} & \nb{0.0362} & \bb{0.0299}  & \nb{0.0292} & \nb{0.0284} & \bb{0.0264\str} \\
\hline
\end{tabular}
\end{small}
\end{center}
\caption{Experimental results on the Reuters corpus, using SVMs for linear,
Gaussian, and multinomial diffusion kernels.  The left columns use
tf representation and the right columns use tf-idf
representation.  The error rates shown are averages obtained using
20-fold cross validation.  The best performance for each training set
size $L$ is shown in boldface. An asterisk (*) indicates that the
difference is not statistically significant according to the paired $t$
test at the 0.05 level.}
\label{tab:reuters1}
\end{table}

\begin{table}
\renewcommand{\arraystretch}{1.1}
\begin{center}
\begin{small}
\begin{tabular}{|c|r||ccc||ccc|}
\multicolumn{2}{c}{} & \multicolumn{3}{c}{tf Representation}
                     & \multicolumn{3}{c}{tf-idf Representation} \\[1pt]
\hline
 Task & $\;L\;$ & Linear & Gaussian & Diffusion & Linear & Gaussian & Diffusion \\
% Reuters 1 vs earn
\hline
  & 40 & \nb{0.1043} & \nb{0.1043} & \bb{0.1021\str}  & \nb{0.0829} & \nb{0.0831} & \bb{0.0814\str} \\
  & 80 & \nb{0.0902} & \nb{0.0902} & \bb{0.0856\str}  & \nb{0.0764} & \nb{0.0767} & \bb{0.0730\str} \\
acq vs.~earn
  & 120 & \nb{0.0795} & \nb{0.0796} & \bb{0.0715}  & \nb{0.0626} & \nb{0.0628} & \bb{0.0562} \\
  & 200 & \nb{0.0599} & \nb{0.0599} & \bb{0.0497}  & \nb{0.0509} & \nb{0.0511} & \bb{0.0431} \\
  & 400 & \nb{0.0417} & \nb{0.0417} & \bb{0.0340}  & \nb{0.0336} & \nb{0.0337} & \bb{0.0294} \\
\hline
\hline
  & 40 & \nb{0.0759} & \nb{0.0758} & \bb{0.0474}  & \nb{0.0451} & \nb{0.0451} & \bb{0.0372\str} \\
  & 80 & \nb{0.0442} & \nb{0.0443} & \bb{0.0238}  & \nb{0.0246} & \nb{0.0246} & \bb{0.0177} \\
moneyFx vs.~earn
  & 120 & \nb{0.0313} & \nb{0.0311} & \bb{0.0160}  & \nb{0.0179} & \nb{0.0179} & \bb{0.0120} \\
  & 200 & \nb{0.0244} & \nb{0.0237} & \bb{0.0118}  & \nb{0.0113} & \nb{0.0113} & \bb{0.0080} \\
  & 400 & \nb{0.0144} & \nb{0.0142} & \bb{0.0079}  & \nb{0.0080} & \nb{0.0079} & \bb{0.0062} \\
\hline
\hline
  & 40 & \nb{0.0969} & \nb{0.0970} & \bb{0.0543}  & \nb{0.0365} & \nb{0.0366} & \bb{0.0336\str} \\
  & 80 & \nb{0.0593} & \nb{0.0594} & \bb{0.0275}  & \nb{0.0231} & \nb{0.0231} & \bb{0.0201\str} \\
grain vs.~earn
  & 120 & \nb{0.0379} & \nb{0.0377} & \bb{0.0158}  & \nb{0.0147} & \nb{0.0147} & \bb{0.0114\str} \\
  & 200 & \nb{0.0221} & \nb{0.0219} & \bb{0.0091}  & \nb{0.0082} & \nb{0.0081} & \bb{0.0069\str} \\
  & 400 & \nb{0.0107} & \nb{0.0105} & \bb{0.0060}  & \nb{0.0037} & \nb{0.0037} & \bb{0.0037\str} \\
\hline
\hline
  & 40 & \nb{0.1108} & \nb{0.1107} & \bb{0.0950}  & \bb{0.0583\str} & \nb{0.0586} & \nb{0.0590} \\
  & 80 & \nb{0.0759} & \nb{0.0757} & \bb{0.0552}  & \nb{0.0376} & \nb{0.0377} & \bb{0.0366\str} \\
crude vs.~earn
  & 120 & \nb{0.0608} & \nb{0.0607} & \bb{0.0415}  & \nb{0.0276} & \bb{0.0276\str} & \nb{0.0284} \\
  & 200 & \nb{0.0410} & \nb{0.0411} & \bb{0.0267}  & \bb{0.0218\str} & \nb{0.0218} & \nb{0.0225} \\
  & 400 & \nb{0.0261} & \nb{0.0257} & \bb{0.0194}  & \nb{0.0176} & \bb{0.0171\str} & \nb{0.0181} \\
\hline
\end{tabular}
\end{small}
\end{center}
\caption{Experimental results on the Reuters corpus, using support
vector machines for linear, Gaussian, and multinomial diffusion
kernels. The left columns use tf representation and the right
columns use tf-idf representation.  The error rates shown are
averages obtained using 20-fold cross validation.  The best
performance for each training set size $L$ is shown in boldface. An
asterisk (*) indicates that the difference is not statistically
significant according to the paired $t$ test at the 0.05 level.}
\label{tab:reuters2}
\end{table}

\begin{table}
\renewcommand{\arraystretch}{1.1}
\begin{small}
\begin{center}
\begin{tabular}{|l|c|c|c|}\hline
Category&Linear&RBF&Diffusion \\ \hline  \hline
\texttt{earn}    &0.01159& 0.01159& {\bf 0.01026}\\
\texttt{acq}     &0.01854& 0.01854& {\bf 0.01788}\\
\texttt{money-fx}&0.02418& 0.02451& {\bf 0.02219}\\
\texttt{grain}   &0.01391& 0.01391& {\bf 0.01060}\\
\texttt{crude}   &0.01755& 0.01656& {\bf 0.01490}\\
\texttt{trade}   &0.01722& {\bf 0.01656} & 0.01689\\
\texttt{interest}&0.01854& 0.01854& {\bf 0.01689}\\
\texttt{ship}    &0.01324& 0.01324& {\bf 0.01225}\\
\texttt{wheat}   &0.00894& 0.00794& {\bf 0.00629}\\
\texttt{corn}    &0.00794& 0.00794& {\bf 0.00563}\\ \hline
\end{tabular}
\end{center}
\caption{Test set error rates for the Reuters
top 10 classes using tf features. The train and test sets were created
using the Mod-Apt split.}
\label{tab:modapte-split1}
\end{small}
\end{table}

\begin{table}[t]
\renewcommand{\arraystretch}{1.1}
\begin{small}
\begin{center}
\begin{tabular}{|l|c|c|c|c|}\hline
Category&Linear&RBF&Diffusion & $\pm$ \\ \hline  \hline
\texttt{earn}     & 0.9781  &  0.9781  &  {\bf 0.9808 }  &  $-$\\
\texttt{acq}      & 0.9626  &  0.9626  &  {\bf 0.9660 }  &  $+$ \\
\texttt{money-fx} & 0.8254  &  0.8245  &  {\bf 0.8320 }  &  $+$ \\
\texttt{grain}    & 0.8836  &  0.8844  &  {\bf 0.9048 }  &  $-$\\
\texttt{crude}    & 0.8615  &  0.8763  &  {\bf 0.8889 }  &  $+$ \\
\texttt{trade}    & 0.7706  &  0.7797  &  {\bf 0.8050 }   &  $+$\\
\texttt{interest} & {\bf 0.8263 }  &  {\bf 0.8263 }  &  0.8221   &  $+$ \\
\texttt{ship}     & 0.8306  &  0.8404  &  {\bf 0.8827 }  &  $+$ \\
\texttt{wheat}    & 0.8613  &  0.8613  &  {\bf 0.8844 }  &  $-$\\
\texttt{corn}     & 0.8727  &  0.8727  &  {\bf 0.9310 }  &  $+$ \\
\hline
\end{tabular}
\end{center}
\caption{F1 measure for the Reuters top 10 classes using tf
features. The train and test sets were created using the Mod-Apte
split.  The last column compares the presented results with the
published results of \cite{Zhang:01}, with a $+$ indicating the
diffusion kernel F1 measure is greater than the result published in
\cite{Zhang:01} for this task.} \label{tab:modapte-split2}
\end{small}
\end{table}


\paragraph{}
Figure~\ref{fig:experiments4} and Figure~\ref{fig:experiments5} show
representative results for the second type of classification task,
where the goal is to discriminate between two specific classes.  In
the case of the WebKB data the results are shown for \texttt{course}
vs.~\texttt{student}.  In the case of the Reuters data the results
are shown for \texttt{moneyFx} vs.~\texttt{earn} and
\texttt{grain} vs.~\texttt{earn}.  Again, the results for the
other classes are qualitatively similar; the numerical results
are summarized in Tables~\ref{tab:webkb2} and~\ref{tab:reuters2}.
\paragraph{}
In these figures, the leftmost plots show the performance
of tf features while the rightmost plots show the performance of tf-idf
features. As mentioned above, in the case of the diffusion kernel we use $L_1$
normalization to give a valid embedding into the probability simplex,
while for the linear and Gaussian kernels we use $L_2$
normalization, which works better empirically than $L_1$ for these kernels.
The curves show the test set error rates averaged over
20 iterations of cross validation as a function of the training set
size. The error bars represent one standard deviation.
For both the Gaussian and diffusion kernels, we test scale
parameters ($\sqrt{2}\sigma$ for the Gaussian kernel and $2t^{1/2}$
for the diffusion kernel) in the set $\{0.5,1,2,3,4,5,7,10\}$.
The results reported are for the best parameter value in that range.

\paragraph{}
We also performed experiments with the popular Mod-Apte train and
test split for the top 10 categories of the Reuters collection.  For
this split, the training set has about 7000 documents and is highly
biased towards negative documents.  We report in
Table~\ref{tab:modapte-split1} the test set accuracies for the tf
representation.  For the tf-idf representation, the difference
between the different kernels is not statistically significant for
this amount of training and test data. The provided train set is
more than enough to achieve outstanding performance with all kernels
used, and the absence of cross validation data makes the results too
noisy for interpretation.
\paragraph{}
In Table~\ref{tab:modapte-split2} we report the F1 measure rather
than accuracy, since this measure is commonly used in text
classification.  The last column of the table compares the presented
results with the published results of \emcite{Zhang:01}, with a $+$
indicating the diffusion kernel F1 measure is greater than the
result published by \emcite{Zhang:01} for this task.
\paragraph{}
Our results are consistent with previous experiments in text
classification using SVMs, which have observed that the linear and
Gaussian kernels result in very similar performance
\cite{Joachims:01}.  However the multinomial diffusion kernel
significantly outperforms the linear and Gaussian kernels for the tf
representation, achieving significantly lower error rate than the
other kernels.  For the tf-idf representation, the diffusion kernel
consistently outperforms the other kernels for the WebKb data and
usually outperforms the linear and Gaussian kernels for the Reuters
data. The Reuters data is a much larger collection than WebKB, and
the document frequency statistics, which are the basis for the
inverse document frequency weighting in the tf-idf representation,
are evidently much more effective on this collection.  It is
notable, however, that the multinomial information diffusion kernel
achieves at least as high an accuracy without the use of any
heuristic term weighting scheme.  These results offer evidence that
the use of multinomial geometry is both theoretically motivated and
practically effective for document classification.

%---------------------------------------------------------------
\subsection{Experimental Results for Gaussian Embedding}
\label{sec:experimentsGaussian}

In this section we report experiments on synthetic data
demonstrating the applicability of the heat kernel on the hyperbolic
space $\H^n$ that corresponds to the manifold of spherical normal
distributions.

\paragraph{}
Embedding data points in the hyperbolic space is more complicated
than the multinomial case. Recall that in the multinomial case, we
embedded points by computing the maximum likelihood estimator the
the data. A similar method would fail for $\H^n$ embedding since all
the data points will be mapped to normal distributions with variance
0, which will result in a degenerate geometry that is equivalent to
the Euclidean one (see Section~\ref{sec:geomSphericalNormal}).

\paragraph{}
A more realistic embedding can be achieved by sampling from the
posterior of a Dirichlet Process Mixture Model (DPMM)
\cite{Ferguson73,Blackwell73,Antoniak74}. A Dirichlet Process
Mixture model, based on the spherical Normal distribution,
associates with the data $x_1,\ldots,x_m\in\R^n$ a posterior
$p(\theta_1,\ldots,\theta_m|x_1,\ldots,x_m)$ where
$\theta_i\in\H^{n+1}$. Instead of going into the description and
properties of Dirichlet Process Mixture Model we refer the
interested reader to the references above, and to
\cite{Escobar95,Neal2000} for relevant Monte Carlo approximations.

\paragraph{}
Obtaining $T$ samples from the DPMM posterior
$\{\theta_i^{(t)}\}_{i=1,t=1}^{m,T}$ we can measure the similarity
between points $x_i,x_j$ as
\begin{align} \tilde{K}_t(x_i,x_j) =
\frac{1}{T}\,\,\sum_{t=1}^T
K_t\left(\theta_i^{(t)},\theta_j^{(t)}\right)\end{align}
 where $K_t$ is the
heat kernel on the hyperbolic space given by equations
\eqref{eq:hyper_a}--\eqref{eq:hyper_b}. The above definition of
$\tilde K$ has the interpretation of being approximately the mean of
the heat kernel -- which is now a random variable under the DPMM
posterior $p(\theta|x)$. The details of Gibbs sampling from the
posterior of a spherical Normal based DPMM is given in
Appendix~\ref{sec:appDPMM}.

\paragraph{}
Some intuition may be provided by
Figure~\ref{fig:DPMMvis}\footnote{This Figure is better displayed in
color.}. A sample from the posterior of a DPMM based on data from
two Gaussians $N((-1,-1)^{\top},I)$ (solid blue dots) and
$N((1,1)^{\top},I)$ (hollow red dots). The sample is illustrated by
the circles that represent one standard deviation centered around
the mean. Blue dotted circles represent a parameter associated with
a point from $N((-1,-1)^{\top},I)$, red dashed circles represent a
parameter associated with a point from $N((1,1)^{\top},I)$ and solid
black circles represent parameters associated with points from both
Gaussians.

\begin{figure}
\begin{center}
\includegraphics[scale=0.6]{diffusionFigures/DPMMvis.eps}
\end{center}
\caption{A sample from the posterior of a DPMM based on data from
two Gaussians $N((-1,-1)^{\top},I)$ (solid blue dots) and
$N((1,1)^{\top},I)$ (hollow red dots). The sample is illustrated by
the circles that represent one standard deviation centered around
the mean. Blue dotted circles represent a parameter associated with
a point from $N((-1,-1)^{\top},I)$, red dashed circles represent a
parameter associated with a point from $N((1,1)^{\top},I)$ and solid
black circles represent parameters associated with points from both
Gaussians.} \label{fig:DPMMvis}
\end{figure}

\paragraph{}
After generating data from two significantly overlapping Gaussians
$N((-2,-2)^{\top},5I),N((2,2)^{\top},5I)$, and sampling 20 samples
from the posterior we compared the performance of a standard RBF
kernel and $\tilde K$. The test set accuracy for SVM, after 20-fold
cross validation, as a function of the train set size is displayed
in Figure~\ref{fig:DPMMacc}. The mean hyperbolic heat kernel
outperforms the RBF kernel consistently.


\begin{figure}
\begin{center}
\includegraphics[scale=0.6]{diffusionFigures/DPMMaccPlot.eps}
\end{center}
\caption{Test set error rate for SVM based on standard RBF kernel
and the mean hyperbolic heat kernel $\tilde{K}$ as a function of the
train set size, after 20-fold cross validation. The data was
generated from two significantly overlapping Gaussians
$N((-2,-2)^{\top},5I),N((2,2)^{\top},5I)$.} \label{fig:DPMMacc}
\end{figure}



%----------------------------------------------------------------
\subsection{Discussion}
\label{sec:discuss}

In this section we introduced a family of kernels that is intimately
based on the geometry of the Riemannian manifold associated with a statistical
family through the Fisher information metric.  The metric is
canonical in the sense that it is uniquely determined
by requirements of invariance \cite{Chentsov1982}, and
moreover, the choice of the heat kernel is natural because
it effectively encodes a great deal of geometric information about the manifold.
While the geometric perspective in statistics has most often
led to reformulations of results that can be viewed more
traditionally, the kernel methods developed here clearly
depend crucially on the geometry of statistical families.

\paragraph{}
The main application of these ideas has been to develop the
multinomial diffusion kernel.   Our
experimental results indicate that the resulting diffusion kernel is
indeed effective for text classification using support vector machine
classifiers, and can lead to significant improvements in accuracy
compared with the use of linear or Gaussian kernels, which have been
the standard for this application.  The results of
Section~\ref{sec:experimentsDiff} are notable since accuracies better or
comparable to those obtained using heuristic weighting schemes such as
tf-idf are achieved directly through the geometric approach.  In part,
this can be attributed to the role of the Fisher information metric;
because of the square root in the embedding into the sphere, terms
that are infrequent in a document are effectively up-weighted, and
such terms are typically rare in the document collection overall.  The
primary degree of freedom in the use of information diffusion kernels
lies in the specification of the mapping of data to model
parameters. For the multinomial, we have used the maximum likelihood
mapping. The use of other model families and mappings remains an
interesting direction to explore.

\paragraph{}
While kernel methods generally are ``model free,'' and do not make
distributional assumptions about the data that the learning
algorithm is applied to, statistical models offer many advantages,
and thus it is attractive to explore methods that combine data
models and purely discriminative methods. Our approach combines
parametric statistical modeling with non-parametric discriminative
learning, guided by geometric considerations.  In these aspects it
is related to the methods proposed by~\emcite{Jaakkola1998}.
However, the kernels proposed in the current section differ
significantly from the Fisher kernel of \emcite{Jaakkola1998}. In
particular, the latter is based on the score $\grad \theta \log
p(X\given\hat\theta)$ at a single point $\hat\theta$ in parameter
space.  In the case of an exponential family model it is given by a
covariance $K_F(x,x') = \sum_i \left(x_i -
E_{\hat\theta}[X_i]\right) \left(x_i' - E_{\hat\theta}[X_i]\right)$;
this covariance is then heuristically exponentiated. In contrast,
information diffusion kernels are based on the full geometry of the
statistical family, and yet are also invariant under
re-parameterizations of the family. In other conceptually related
work, \emcite{Belkin:2003} suggest measuring distances on the data
graph to approximate the underlying manifold structure of the data.
In this case the underlying geometry is inherited from the embedding
Euclidean space rather than the Fisher geometry.

\paragraph{}
While information diffusion kernels are very general, they will be
difficult to compute in many cases -- explicit formulas such as
equations (\ref{eq:hyper_a}--\ref{eq:hyper_b}) for hyperbolic space
are rare. To approximate an information diffusion kernel it may be
attractive to use the parametrices and geodesic distance between
points, as we have done for the multinomial.  In cases where the
distance itself is difficult to compute exactly, a compromise may be
to approximate the geodesic distance between nearby points in terms of the
Kullback-Leibler divergence.  In effect, this
approximation is already incorporated into the kernels recently
proposed by \emcite{Moreno:03} for multimedia applications, which have
the form $K(\theta,\theta')
\propto \exp(-\alpha D(\theta,\theta')) \approx \exp(-2\alpha d^2(\theta,\theta'))$,
and so can be viewed in terms of the leading order approximation to
the heat kernel.  The results of \emcite{Moreno:03} are suggestive that
diffusion kernels may be attractive not only for multinomial
geometry, but also for much more complex statistical families.


%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Hyperplane Margin Classifiers}
\label{sec:hyperplane}

Linear classifiers are a mainstay of machine learning algorithms,
forming the basis for techniques such as the perceptron, logistic regression,
boosting, and support vector machines.  A linear classifier,
parameterized by a vector $w\in\mathbb{R}^n$, classifies examples
according to the decision rule $\hat y(x) = \text{sign} \left( \sum_i
w_i \phi_i(x) \right) = \text{sign}(\langle w,x\rangle) \in
\{-1,+1\}$, following the common practice of identifying $x$ with the
feature vector $\phi(x)$.  The differences between different linear
classifiers lie in the criteria and algorithms used for selecting
the parameter vector $w$ based on a training set.

\paragraph{}
Geometrically, the decision surface of a linear classifier
is formed by a hyperplane or linear subspace in $n$-dimensional
Euclidean space,
$\{x\in\mathbb{R}^n: \langle x,w \rangle =0\}$ where $\langle\cdot,\cdot\rangle$
denotes the Euclidean inner product. (In both the
algebraic and geometric formulations, a bias term is
sometimes added; we prefer to absorb the bias into the notation
given by the inner product, by setting $x_n=1$ for all $x$.)
The linearity assumption made by such classifiers can be justified
on purely computational grounds; linear classifiers are generally easy
to train, and the linear form is simple to analyze and compute.

\paragraph{}
Modern learning theory emphasizes the tension between fitting the
training data well and the more desirable goal of achieving good
generalization. A common practice is to choose a model that fits the
data closely, but from a restricted class of models.  The model class
needs to be sufficiently rich to allow the choice of a good
hypothesis, yet not so expressive that the selected model is likely to
overfit the data. Hyperplane classifiers are attractive for balancing
these two goals. Indeed, linear hyperplanes are a rather restricted
set of models, but they enjoy many unique properties.  For example,
given two points $x,y\in\mathbb{R}^n$, the set of points equidistant
from $x$ and $y$ is a hyperplane; this lies behind the intuition that
a hyperplane is the correct geometric shape for separating sets of
points.  Similarly, a hyperplane is the best decision boundary to
separate two Gaussian distributions of equal covariance.
Another distinguishing
property is that a hyperplane in $\mathbb{R}^n$ is isometric to
$\mathbb{R}^{n-1}$, and can therefore be thought of as a reduced
dimension version of the original feature space.  Finally, a linear
hyperplane is the union of straight lines, which are distance
minimizing curves, or geodesics, in Euclidean geometry.

\paragraph{}
However, a fundamental assumption is implicitly
associated with linear classifiers, since they are based crucially on
the use of the Euclidean geometry of $\mathbb{R}^n$.  If the data or
features at hand lack a Euclidean structure, the arguments above for
linear classifiers break down; arguably, there is lack of Euclidean
geometry for the feature vectors in most applications.  This section
studies analogues of linear hyperplanes as a means of obtaining
simple, yet effective classifiers when the data can be represented in
terms of a natural geometric structure that is only locally Euclidean.
This is the case for categorical data that is represented in terms of
multinomial models, for which the associated geometry is spherical.

\paragraph{}
Because of the complexity of the notion of linearity in general
Riemannian spaces, we focus our attention on the multinomial manifold,
which permits a relatively simple analysis. Hyperplanes
in multinomial manifold is discussed in Section \ref{sec:snp}.
The construction and training of margin
based models is discussed in Section \ref{sec:logReg}, with an emphasis
on spherical logistic regression.  A brief examination of linear
hyperplanes in general Riemannian manifolds appears in
Section~\ref{sec:general} followed by experimental results for
text classification given in Section~\ref{sec:exp}.
Concluding remarks are made in Section~\ref{sec:sum}.



%--------------------------------------------------------------------------
\subsection{Hyperplanes and Margins on $\Sn$} \label{sec:sn}
This section generalizes the notion of linear hyperplanes and margins
to the $n$-sphere $\Sn=\{x\in\mathbb{R}^{n+1}:\sum_i x_i^2=1\}$.
A similar treatment on the positive $n$-sphere $\Snp$ is more
complicated, and is postponed to the next section.
In the remainder of this section we denote points on $\Pn,\Sn$ or $\Snp$
as vectors in $\mathbb{R}^{n+1}$ using the standard basis of the embedding
space. The notation $\langle \cdot,\cdot\rangle$ and $\|\cdot\|$ will
be used for the Euclidean inner product and norm.

A hyperplane on $\Sn$ is defined as
$H_u=\Sn\cap E_{u}$ where $E_u$ is an $n$-dimensional linear
subspace of $\mathbb{R}^{n+1}$ associated with the normal vector $u$.
We occasionally need to refer to the unit normal vector (according to the
Euclidean norm) and denote it
by $\hat{u}$.
$H_u$ is an $n-1$ dimensional submanifold of $\Sn$
which is  isometric to $\mathbb{S}^{n-1}$ \cite{Bridson1999}.
Using the common notion
of the distance of a point
from a set $d(x,S) = \inf_{y\in S} d(x,y)$
we make the following definitions.
\begin{defn}
Let $X$ be a metric space.  A {\sl decision boundary}
is a subset of $X$ that separates $X$ into two connected
components.  The {\sl margin\/} of $x$ with respect
to a decision boundary $H$
is $d(x,H)=\inf_{y\in H} d(x,y)$.
\end{defn}
Note that this definition reduces to the common definition of margin
for Euclidean geometry and affine hyperplanes.

\paragraph{}
In contrast to \emcite{Gous1998}, our submanifolds are intersections
of the sphere with linear subspaces, not affine sets.  One motivation
for the above definition of hyperplane as the correct generalization
of a Euclidean hyperplane is that $H_u$ is the set of points
equidistant from $x,y\in\Sn$ in the spherical metric. Further
motivation is given in Section~\ref{sec:general}.

\paragraph{}
Before we can obtain a closed form expression for margins on $\Sn$
we need the following definitions.
\begin{defn}
Given a point $x\in\mathbb{R}^{n+1}$, we define its {\sl reflection} with
respect to $E_u$ as
\[r_u(x)=x-2\langle x,\hat u\rangle \hat u.\]
\end{defn}
Note that if $x\in\Sn$ then $r_u(x)\in\Sn$ as well, since
$\norm{r_u(x)}^2 = \norm{x}^2-4\langle x,\hat u\rangle^2+4\langle x,\hat u\rangle^2
=1.$

\paragraph{}
\begin{defn}
The {\sl projection\/} of $x\in\Sn\setminus\{\hat u\}$ on $H_u$ is defined to be
\[p_u(x)=\frac{x-\langle x,\hat u\rangle \hat u}{\sqrt{1-\langle x,\hat u\rangle^2}}.\]
\end{defn}
Note that $p_u(x)\in H_u$, since
$\norm{p_u(x)}=1$ and
$\langle x-\langle x,\hat u\rangle \hat u,\hat u\rangle
=\langle x,\hat u\rangle - \langle x,\hat u\rangle \norm{\hat u}^2=0.$
The term projection is justified by the following proposition.

\paragraph{}
\begin{prop} \label{prop:sphereProj}
Let $x\in\Sn\setminus (H_u\cup\{\hat u\})$. Then
\begin{align*}
&(a)\quad d(x,q)=d(r_u(x),q) \quad \forall q\in H_u\\
&(b)\quad d(x,p_u(x))=
\arccos\left(\sqrt{1-\langle x,\hat u\rangle^2}\right)\\
%&\qquad\qquad =\frac{1}{2}\arccos(1-2\langle x,\hat u\rangle^2) \\
&(c)\quad d(x,H_u) = d(x,p_u(x)).
\end{align*}
\end{prop}
\begin{proof}
Since $q\in H_u$,
\begin{align*}
 \cos d(r_u(x),q) &= \langle x-2\langle x,\hat u\rangle \hat u,q\rangle \\
& =\langle x,q\rangle-2\langle x,\hat u\rangle \langle\hat u,q\rangle\\
&=\langle x,q\rangle = \cos d(x,q)
\end{align*}
and $(a)$ follows. Assertion $(b)$ follows from
\begin{align*}
\cos d(x,p_u(x)) &=
\left\langle x,\frac{x-\langle x,\hat u\rangle \hat u}{\sqrt{1-\langle x,\hat u\rangle^2}}\right\rangle
=\frac{1-\langle x,\hat u\rangle^2}{\sqrt{1-\langle x,\hat u\rangle^2}}.
\end{align*}
Finally, to prove $(c)$ note that by the
identity $\cos 2\theta=2\cos^2\theta-1$,
\begin{align*}
\cos(2d(x,p_u(x)))&=2\cos^2(d(x,p_u(x)))-1\\
&=1-2\langle x,\hat u\rangle^2
=\cos(d(x,r_u(x)))
\end{align*}
and hence $d(x,p_u(x))=\frac{1}{2} d(x,r_u(x))$.  The distance
$d(x,q), q\in H_u$ cannot be any smaller than $d(x,p_u(x))$ since this
would result in a path from $x$ to $r_u(x)$ of length shorter than the
geodesic $d(x,r_u(x))$.
\end{proof}

\paragraph{}
Parts $(b)$ and $(c)$ of
Proposition~\ref{prop:sphereProj} provide a closed form expression for
the $\Sn$ margin analogous to the
Euclidean unsigned margin $|\langle x,\hat u\rangle|$.
Similarly, the $\Sn$ analogue of the Euclidean signed margin
$y\langle \hat u,x\rangle$ is
\[ y\frac{\langle x,\hat u\rangle}{|\langle x,\hat u\rangle|}
\,\,\,\arccos\left(\sqrt{1-\langle x,\hat u\rangle^2}\right).\]
A plot of the signed margin as a function of $\langle x,\hat u\rangle$
and a geometric interpretation of the spherical margin appear in
Figure~\ref{fig:snMargin}.

\begin{figure}
\begin{center}
\includegraphics[scale=0.45]{hyperplaneFigures/acosPlot.eps}
\hspace{.45in}
{
\psfrag{x}{{\scriptsize $x$}}
\psfrag{u}{{\scriptsize $\hat u$}}
\psfrag{p(x)}{{\scriptsize $p_u(x)$}}
\psfrag{x-2<x,u>u}{{\scriptsize $r_u(x)$}}
\psfrag{<x,u>}{{\scriptsize $\langle x,\hat u\rangle$}}
\psfrag{n}{{\scriptsize $d(x,p_u(x))=\frac{1}{2}d(x,r_u(x))$}}
\includegraphics[scale=0.37]{hyperplaneFigures/sphere.eps}
}
\caption{
The signed margin
$\text{sign}(\langle x,\hat y\rangle)d(x,H_u)$ as a function of
$\langle x,\hat{u}\rangle$, which lies in the interval $[-\frac{\pi}{2},\frac{\pi}{2}]$ (left) and
a geometric interpretation of the spherical margin (right).}
 \label{fig:snMargin}
\end{center}
\end{figure}

%-----------------------------------------------------------------------
\subsection{Hyperplanes and Margins on $\Snp$} \label{sec:snp}

A hyperplane on the positive $n$-sphere $\Snp$ is defined as
$H_{u+}=E_u\cap \Snp$, assuming it is non-empty.
This definition leads to a margin
concept $d(x,H_{u+})$ different from the $\Sn$ margin $d(x,H_u)$ since
\begin{align*}
d(x,H_{u+})&=\inf_{y\in E_u\cap \Snp} d(x,y) \\&\geq
\inf_{y\in E_u\cap \Sn} d(x,y)=d(x,H_u).
\end{align*}
The above infimum is attained by the continuity of $d$ and
compactness of $E_u\cap\Snp$ justifying the notation
$q=\argmin_{y\in E_u\cap\Snp}d(x,y)$ as a point realizing the margin
distance $d(x,H_{u+})$.

\begin{figure}
\begin{center}
{
\psfrag{x}{$x$}
\psfrag{r}{$r$}
\psfrag{q}{$q$}
\psfrag{p(x)}{$p_u(x)$}
\includegraphics[scale=0.33]{hyperplaneFigures/triangles.eps}
}\end{center}
\caption{The spherical law of cosines implies $d(r,x)\leq d(q,x)$.}
\label{fig:triangles}
\end{figure}


\paragraph{}

The following theorem will be useful in  computing $d(x,H_{u+})$.
For a proof see \emcite{Bridson1999} page 17.
\begin{thrm}
\emph{(\textbf{The Spherical Law of Cosines})}\\
Consider a spherical triangle with geodesic edges of lengths $a,b,c$, where
$\gamma$ is the vertex angle opposite to edge $c$.
Then
\[ \cos c = \cos a\cos b + \sin a\sin b\cos \gamma.\]
\end{thrm}

\paragraph{}


\paragraph{}
We have the following corollaries of Proposition \ref{prop:sphereProj}.
\begin{prop}
If $x\in\Snp$ and $p_u(x)\in\Snp$ then
\begin{align*}
p_u(x)&=\argmin_{y\in\Snp\cap E_u} d(x,y)\\
d(x,H_u)&=d(x,H_{u+})
\end{align*}
%$p_u(x)=\argmin_{y\in\Snp\cap E_u} d(x,y)$ and
%$d(x,H_u)=d(x,H_{u+})$.
\end{prop}
\begin{proof} \label{prop:triangles}
This follows immediately from
the fact that $p_u(x)=\argmin_{y\in\Sn\cap E_u} d(x,y)$ and
from $\Snp\cap E_u\subset \Sn\cap E_u$.
\end{proof}
\begin{prop} \label{prop:bdP1}
For $x\in\Snp$ and  $p_u(x)\not\in\Snp$  we have
\[q=\argmin_{y\in\Snp\cap E_u} d(x,y) \in \partial \Snp\]
where $\partial\Snp$ is the boundary of $\Snp$.
\end{prop}
\begin{proof}
Assume that $q\not\in\partial\Snp$
and connect $q$ and $p_u(x)$ by a minimal geodesic $\alpha$.
Since $p_u(x)\not\in\Snp$, the geodesic
$\alpha$ intersects the boundary $\partial\Snp$ at a point $r$.
Since $q,p_u(x)\in H_u$ and $H_u$ is geodesically
convex, $\alpha \subset  H_u$. Now,
since $p_u(x)=\argmin_{y\in \alpha} d(y,x)$,
the geodesic from $x$ to $p_u(x)$ and
$\alpha$ intersect orthogonally (this is an
elementary result in Riemannian geometry, e.g. \emcite{Lee1997} p. 113).
Using the spherical law of cosines, applied to the
spherical triangles $(q,x,p_u(x))$ and $(r,x,p_u(x))$ (see Figure \ref{fig:triangles}),
we deduce that
\begin{align*}
\cos d(x,q)  &= \cos d(q,p_u(x)) \cos d(x,p_u(x))\\
  &\leq \cos d(r,p_u(x)) \cos d(x,p_u(x)) \\
  &= \cos d(x,r)
\end{align*}
Hence $r$ is closer to
$x$ than $q$.  This contradicts
the definition of $q$; thus
$q$ can not lie in the interior of $\Snp$.
\end{proof}

\paragraph{}
Before we proceed to compute $d(x,H_{u+})$ for $p_u(x)\not\in\Snp$
we define the following concepts.
\begin{defn}\label{def:bd}
The {\sl boundary} of $\Sn$ and $\Snp$ with respect to
$A\subset\{1,\ldots,n+1\}$ is
\begin{align*}
\partial_A\Sn &= \Sn \cap
\{x\in\mathbb{R}^{n+1}: \forall i\in A,\, x_i=0\}\cong \mathbb{S}^{n-|A|}\\
\partial_A\Snp &= \Snp \cap
\{x\in\mathbb{R}^{n+1}: \forall i\in A,\, x_i=0\}\cong \mathbb{S}^{n-|A|}_{+}
\end{align*}
\end{defn}
Note that if $A\subset A'$ then $\partial_{A'}\Snp\subset \partial_{A}\Snp$.
We use the notation $\langle \cdot,\cdot \rangle_A$ and
$\norm{\cdot}_A$ to refer to the Euclidean inner product and norm, where the
summation is restricted to indices \emph{not} in $A$.
\begin{defn}
Given $x\in\Sn$ we define $x|_A\in\partial_A\Sn$ as
\[(x|_A)_i =
\begin{cases}
0 & i\in A \\
x_i/\norm{x}_{A} & i\not\in A\,.
\end{cases}\]
\end{defn}
We abuse the notation by identifying $x|_A$ also with
the corresponding point
on $\mathbb{S}^{n-|A|}$ under the isometry
$\partial_A \Sn\cong\mathbb{S}^{n-|A|}$
mentioned in Definition
\ref{def:bd}.
Note that if $x\in\Snp$ then
$x|_{A}\in\partial_{A}\Snp$.
The following proposition computes the $\Snp$ margin $d(x,H_{u+})$
given the boundary set of $q=\argmin_{y\in \Snp\cap E_u}d(y,x)$.
\begin{prop}
Let $\hat u\in\Rn1$ be a unit vector, $x\in\Snp$
and $q=\argmin_{y\in \Snp\cap E_u}d(y,x)\in \partial_A\Snp$
where $A$ is the (possibly empty) set
$A = \{1\leq i\leq n+1 \,:\, q_i=0\}.$
Then
\begin{align*}
d(x,H_{u+}) =
 \arccos \left(\norm{x}_A \sqrt{1-\langle x|_A,\hat u|_A\rangle^2}\right)
\end{align*}
\end{prop}

\begin{proof}
If $p_u(x)\in\Snp$ then the proposition follows from earlier propositions
and the fact that when $A=\emptyset$,
$\norm{x}_A=\norm{x}=1$ and $v|_A=v$.
We thus restrict our attention to the case of $A\neq\emptyset$.

For all $I\subset \{1,\ldots,n+1\}$ we have
\begin{eqnarray*}
\argmin_{y\in \partial_I\Snp\cap E_u} d(x,y)
&=&\argmax_{y\in \partial_I\Snp\cap E_u} \langle x,y\rangle \\
&=&\argmax_{y\in \partial_I\Snp\cap E_u} \langle x,y\rangle_I \\
&=& \argmax_{y\in \mathbb{S}^{n-|I|}_{+}\cap E_{u|I}}
\norm{x}_I \langle x|_I,y\rangle  \\
&=& \argmin_{y\in \mathbb{S}^{n-|I|}_{+}\cap E_{u|I}} d(x|_I,y)\,.
\end{eqnarray*}
It follows that
\begin{align} \label{eq:compQ}
q|_A=
\argmin_{y\in \mathbb{S}^{n-|A|}_{+}\cap E_{u|A}} d(x|_A,y).
\end{align}
By Proposition~\ref{prop:bdP1} applied to $\mathbb{S}^{n-|A|}$
we have that since $q|_A$ lies in the interior of
$\mathbb{S}^{n-|A|}$
then so does
\begin{eqnarray*}
 p_{u|A}(x|_A) = \frac{x|_A-\langle x|_A,\hat u|_A\rangle \hat u|_A}{\sqrt{1-\langle x|_A,\hat u|_A\rangle^2}}\,, \quad x|_A, \hat u|_A \in\mathbb{S}^{n-|A|}_{+}.
\end{eqnarray*}

Using Proposition \ref{prop:sphereProj} applied to $\mathbb{S}^{n-|A|}$ we can compute
$d(x,H_{u+})$ as
\begin{align*}
  d(x, p_{u|A}(x|_A))
&= \arccos \left\langle x,
\frac{x|_A-\langle x|_A,\hat u|_A\rangle \hat u|_A}{\sqrt{1-\langle x|_A,\hat u|_A\rangle^2}} \right\rangle \\
%=  \arccos \frac{\langle x,x|_A-\langle x|_A,\hat u|_A\rangle \hat u|_A\rangle}{
%   \sqrt{1-\langle x|_A,\hat u|_A\rangle^2}}   \\
&=  \arccos
    \frac{\norm{x}_A-
    \langle x|_A,\hat u|_A\rangle \langle x, \hat u|_A\rangle}{
    \sqrt{1-\langle x|_A,\hat u|_A\rangle^2}
    }\\
%&=   \arccos \frac{\norm{x}_A(1-
%   \langle x|_A,\hat u|_A\rangle^2)}
%   {\sqrt{1-\langle x|_A,\hat u|_A\rangle^2}
%   }\\
&= \arccos \left(\norm{x}_A \sqrt{1-\langle x|_A,\hat u|_A\rangle^2}\right).
\end{align*}
\end{proof}

\paragraph{}

In practice the boundary set $A$ of $q$ is not known.
In our experiments we set $A= \{i:(p_u(x))_i \leq 0\}$;
in numerical simulations in low dimensions, the true boundary
never lies outside of this set.

%-----------------------------------------------------------------------
\subsection{Logistic Regression on the Multinomial Manifold} \label{sec:logReg}
The logistic regression model
$p(y\given x)=\frac{1}{Z}\exp(y\langle x,w\rangle)$,
with $y\in\{-1,1\}$, assumes Euclidean geometry.
It can be re-expressed as
\begin{eqnarray*}
p(y\given x\,; u) &\propto& \exp\left(y\norm{u}\langle x,\hat{u}\rangle\right)\\
&=& \exp\left(y\,\text{sign}(\langle x,\hat u\rangle)\,\theta d(x,H_u)\right)
\end{eqnarray*}
where $d$ is the Euclidean distance of $x$ from the hyperplane
that corresponds to the normal vector $\hat{u}$, and where $\theta = \norm{u}$
is a parameter.

\paragraph{}
The generalization to spherical geometry involves
simply changing the margin to reflect the appropriate geometry:
\begin{eqnarray*}
\lefteqn{p(y|x\,; \hat u,\theta) \propto} && \\
&& \hskip-.3in
\exp\left(y\,\text{sign}(\langle x,\hat u\rangle)
\,\, \theta \arccos\left(\norm{x}_A\sqrt{1-\langle x|_A,\hat{u}|_A\rangle^2}\right)\right).
\end{eqnarray*}
Denoting $s_x=y\,\text{sign}(\langle x,\hat u\rangle)$,
the log-likelihood of the example $(x,y)$ is
\begin{align*}
&\ell(\hat u,\theta\,; (x,y))\\
%&=  \log
%\frac{e^{s_x\theta\arccos\left(\|x\|_A\sqrt{1-\langle x|_A,\hat{u}|_A\rangle^2}\right)}}{
%e^{s_x\theta\arccos\left(\theta \|x\|_A\sqrt{1-\langle x|_A,\hat{u}|_A\rangle^2}\right)}
%+e^{-s_x\theta\arccos\left(\norm{x}_A\sqrt{1-\langle x|_A,\hat{u}|_A\rangle^2}\right)}}\\
&=-\log \left(1+ e^{-2s_x\theta\arccos\left(\norm{x}_A\sqrt{1-\langle x|_A,\hat{u}|_A\rangle^2}\right)}\right).
\end{align*}

\paragraph{}

We compute the derivatives of the log-likelihood in several steps, using
the chain rule and the notation $z=\langle x|_A,\hat u|_A\rangle$.
We have
\begin{eqnarray*}
{\frac{\partial\arccos\left(\norm{x}_A\sqrt{1-z^2}\right)}{\partial z} =}
\frac{z\norm{x}_A}{\sqrt{1-\norm{x}_A^2(1-z^2)}\sqrt{1-z^2}}
\end{eqnarray*}
and hence
\begin{eqnarray}
\label{eq:llGrad}
\frac{\partial\ell(\hat u,\theta\,; (x,y))}{\partial z}\;=\;
\frac{2s_x\theta z\norm{x}_A/(1+e^{2s_x\theta\arccos\left(\norm{x}_A\sqrt{1-z^2}\right)})}{ \sqrt{1-\norm{x}_A^2(1-z^2)}\sqrt{1-z^2}}.
\end{eqnarray}
The log-likelihood derivative with respect to $\hat u_i$ is
equation \eqref{eq:llGrad} times
\begin{align*}
\frac{\partial \langle x|_A,\hat u|_A\rangle}{\partial \hat u_i}
&=\begin{cases}
0 & i\in A\\
\frac{ (x|_A)_i}{\norm{\hat u}_A} - \hat u_i \frac{\langle x|_A,\hat u|_A\rangle}{\norm{u}_A^2} & i\not\in A\,.
\end{cases}
\end{align*}
The log-likelihood derivative with respect to $\theta$ is
\begin{align*}
\frac{\partial \ell(\hat u,\theta\,; (x,y))}{\partial\theta}&
%\frac{e^{-2s_x\theta\arccos(\norm{x}_A\sqrt{1-z^2})}
%}{1+e^{-2s_x\theta\arccos(\norm{x}_A\sqrt{1-z^2})}}
%2s_x \arccos(\norm{x}_A\sqrt{1-z^2})
=\frac{2s_x \arccos(\norm{x}_A\sqrt{1-z^2})}{1+e^{2s_x\theta\arccos(\norm{x}_A\sqrt{1-z^2})}}.
\end{align*}

\paragraph{}
Optimizing the log-likelihood with respect to $\hat{u}$ requires care.
Following the gradient
$\hat u^{(t+1)}=\hat u^{(t)}+\alpha \grad \ell(\hat u^{(t)})$
results in a non-normalized vector.
Performing the above gradient descent
step followed by normalization has the effect of moving along
the sphere in a curve whose tangent vector at $\hat u^{(t)}$ is the
projection of the gradient onto the tangent space $T_{\hat u^{(t)}}\Sn$.
This is the technique used in the experiments described in
Section~\ref{sec:exp}.

\paragraph{}
Note that the spherical
logistic regression model has $n+1$ parameters in contrast
to the $n+2$ parameters of Euclidean logistic regression.
This is in accordance with the intuition that a hyperplane
separating an $n$-dimensional manifold should have $n$ parameters.
The extra parameter in the Euclidean logistic regression is an
artifact of the embedding of the $n$-dimensional multinomial space,
on which the data lies, into an $(n+1)$-dimensional Euclidean space.

\paragraph{}
The derivations and formulations above assume spherical data.
If the data lies on the multinomial manifold, the isometry
$\pi$ mentioned earlier has to precede these calculations.
The net effect is that $x_i$ is replaced by $\sqrt{x_i}$
in the model equation, and in the log-likelihood and its derivatives.
\paragraph{}

Synthetic data experiments contrasting
Euclidean logistic regression and spherical logistic regression on $\Snp$,
as described in this section,  are shown in Figure~\ref{fig:toy-data-hyperplane}.
The leftmost column shows an example where both models give a similar
solution. In general, however, as is the case in the other
two columns, the two models yield significantly different decision boundaries.

\begin{figure}
\def\figsz{.37}
\begin{center}
\begin{tabular}{cc}
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData11c.eps}&
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData12c.eps}\\
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData21c.eps}&
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData22c.eps}\\
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData31c.eps}&
\includegraphics[scale=\figsz]{hyperplaneFigures/toyDataPlots/toyData32c.eps}
\end{tabular}
\end{center}
\caption{Experiments
contrasting Euclidean logistic regression (left column)
with multinomial logistic regression (right column) for several
toy data sets in $\mathbb{P}^2$.}\label{fig:toy-data-hyperplane}
\end{figure}

%-----------------------------------------------------------------------
\subsection{Hyperplanes in Riemannian Manifolds}\label{sec:general}
The definition of hyperplanes in general Riemannian manifolds
has two essential components.
In addition to discriminating between
two classes, hyperplanes should be regular in some sense
with respect to the geometry.
In Euclidean geometry, the two properties of discrimination and regularity
coincide, as
every affine subspace of dimension $n-1$ separates $\mathbb{R}^n$
into two regions. In general, however, these two properties do
not necessarily coincide, and have to be considered separately.

\paragraph{}
The separation property implies that if $N$ is a
hyperplane of $M$ then
$M\setminus N$ has two connected components.
Note that this property is topological and independent
of the metric.
The linearity property is generalized through the notion
of auto-parallelism explained below.
The following definitions and propositions are taken from
\emcite{Spivak1975}, Volume 3.
All the connections described below $\nabla$
are the metric connections inherited from the metric $g$.
\begin{defn}
Let $(\cM,g)$ be a Riemannian manifold and $\nabla$ the
metric connection.
A submanifold $N\subset M$ is {\sl auto-parallel\/} if
parallel translation in $M$ along a curve $C\subset N$
takes vectors tangent to $N$ to vectors tangent to $N$.
\end{defn}
\begin{prop}
A submanifold $N\subset M$ is auto-parallel if and only if
\[X,Y\in T_pN\Rightarrow \nabla_XY\in T_pN.\]
\end{prop}

\paragraph{}
\begin{defn}
A submanifold $N$ of $M$ is {\sl totally geodesic at
$p\in N$} if every geodesic $\gamma$ in $M$ with
$\gamma(0)=p, \gamma'(0)\in T_pN$ remains in $N$
on some interval $(-\epsilon,\epsilon)$. The submanifold $N$ is said to be
{\it totally geodesic\/} if it is totally geodesic at every point.
\end{defn}
As a consequence, we have that  $N$ is totally geodesic if and only if
every geodesic in $N$ is also a geodesic in $M$.
\begin{prop}
Let $N$ be a submanifold of $(M,\nabla)$. Then
\begin{enumerate}
\item If $N$ is auto-parallel in $M$ then
$N$ is totally geodesic.
\item If $M$ is totally geodesic and $\nabla$ is symmetric
then $M$ is auto-parallel.
\end{enumerate}
\end{prop}
Since the metric connection is symmetric, the last proposition
gives a complete  equivalence between
auto-parallelism and totally geodesic submanifolds.

\paragraph{}
We can now define linear hyperplanes on Riemannian manifolds.
\begin{defn}
A {\sl linear decision boundary\/} $N$ in $M$ is
an auto-parallel submanifold of $M$
such that
$M\setminus N$ has two connected
components.
\end{defn}
\paragraph{}

Several observations are in order. First note that
if $M$ is an $n$-dimensional manifold,
the separability condition requires $N$ to be an $(n-1)$-dimensional submanifold.
It is easy to see that every affine subspace of $\mathbb{R}^n$ is
totally geodesic and hence auto-parallel. Conversely, since
the metric connection is symmetric, every auto-parallel
submanifold of Euclidean space
that separates it is an affine subspace.
As a result, we have that our generalization does indeed reduce
to affine subspaces under Euclidean geometry.
Similarly, the above definition reduces to spherical hyperplanes
$H_u\cap \Sn$ and $H_u\cap\Snp$. Another example is the hyperbolic
half plane $\mathbb{H}^2$ where the linear decision boundaries are
half-circles whose centers lie on the $x$ axis.

\paragraph{}
Hyperplanes on $\Sn$ have the following additional nice properties.
They are the set of equidistant points from $x,y\in\Sn$ (for some $x,y$),
they are isometric to $\mathbb{S}^{n-1}$ and they are
parameterized by $n$ parameters.
These properties are
particular to the sphere and do not hold in general
\cite{Bridson1999}.



%----------------------------------------------------------------------
\subsection{Experiments} \label{sec:exp}
A natural embedding of text documents in the multinomial simplex
is the $L_1$ normalized term-frequency (tf) representation
\cite{Joachims2000}
\begin{align*}
\hat{\theta}(x) =
\left(\frac{x_1}{\sum_i x_i},\ldots,\frac{x_{n+1}}{\sum_i x_i} \right).
\end{align*}
Using this embedding we compared
the performance of spherical logistic regression with
Euclidean logistic regression.
Since Euclidean logistic regression often performs better
with $L_2$ normalized tf representation, we included these
results as well.


\paragraph{}
Experiments were conducted on both the
Web-KB  \cite{Craven1998}
and the Reuters-21578 \cite{Lewis94} datasets.
In the Web-KB dataset, the classification task that was tested was
each of the classes faculty, course, project and student vs.~the rest.  In the
Reuters dataset, the task was each of the 8 most popular classes
vs.~the rest.  The test error rates as a function of randomly sampled
training sets of different sizes are shown in Figures
\ref{fig:hyperplaneExp1}-\ref{fig:hyperplaneExp3}.
In both
cases, the positive and negative example sets are equally distributed,
and the results were averaged over a 20-fold cross validation with the
error bars indicating one standard deviation.
As mentioned in Section~\ref{sec:snp}, we assume that the boundary
set of $q=\argmin_{y\in \Snp\cap E_u}d(y,x)$
is equal to $A = \{i:(p_u(x))_i \leq 0\}$.

\paragraph{}
The experiments show that the new linearity and margin concepts
lead to more powerful classifiers than their Euclidean counterparts,
which are commonly used in the literature regardless of the geometry
of the data.

\begin{figure}
\begin{center}
\begin{tabular}{cc}
{\tiny Web-KB: faculty vs.~all} &
{\tiny Web-KB: course vs.~all}\\
\includegraphics[scale=0.37]{hyperplaneFigures/webkb-results/webkbPlot1.eps}&
\includegraphics[scale=0.37]{hyperplaneFigures/webkb-results/webkbPlot2.eps}\\
{\tiny Web-KB: project vs.~all} &
{\tiny Web-KB: student vs.~all} \\
\includegraphics[scale=0.37]{hyperplaneFigures/webkb-results/webkbPlot3.eps}&%\\[8pt]
\includegraphics[scale=0.37]{hyperplaneFigures/webkb-results/webkbPlot4.eps}
\end{tabular}
\end{center}
\caption{
Test error accuracy of spherical logistic regression (solid),
and linear logistic regression using tf representation
with $L_1$ normalization (dashed) and $L_2$ normalization (dotted).
The task is  Web-KB binary ``one vs.~all'' classification, where the name
of the topic is listed above the individual plots.
Error bars represent one standard deviation over 20-fold cross validation
for spherical logistic regression. The error bars of the other classifiers
are of similar sizes and are omitted for clarity.}
\label{fig:hyperplaneExp1}
\end{figure}

\begin{figure}
\begin{center}
\begin{tabular}{cc}
{\tiny Reuters: earn vs.~all} &
{\tiny Reuters: acq vs.~all} \\
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot1.eps}&
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot2.eps}\\%[8pt]
{\tiny Reuters: money-fx vs.~all} &
{\tiny Reuters: grain vs.~all} \\
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot3.eps}&
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot4.eps}
\end{tabular}
\end{center}
\caption{
Test error accuracy of spherical logistic regression (solid),
and linear logistic regression using tf representation
with $L_1$ normalization (dashed) and $L_2$ normalization (dotted).
The task is  Reuters-21578 binary ``one vs.~all'' classification, where the name
of the topic is listed above the individual plots.
Error bars represent one standard deviation over 20-fold cross validation
for spherical logistic regression. The error bars of the other classifiers
are of similar sizes and are omitted for clarity.}
\label{fig:hyperplaneExp2}
\end{figure}

\begin{figure}
\begin{center}
\begin{tabular}{cc}
{\tiny Reuters: crude vs.~all} &
{\tiny Reuters: trade vs.~all} \\
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot5.eps}& %[8pt]
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot6.eps}\\
{\tiny Reuters: interest vs.~all} &
{\tiny Reuters: ship vs.~all} \\
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot7.eps}&
\includegraphics[scale=0.4]{hyperplaneFigures/reuters-results/reutersPlot8.eps}
\end{tabular}
\end{center}
\caption{
Test error accuracy of spherical logistic regression (solid),
and linear logistic regression using tf representation
with $L_1$ normalization (dashed) and $L_2$ normalization (dotted).
The task is  Reuters-21578 binary ``one vs.~all'' classification, where the name
of the topic is listed above the individual plots.
Error bars represent one standard deviation over 20-fold cross validation
for spherical logistic regression. The error bars of the other classifiers
are of similar sizes and are omitted for clarity.}
\label{fig:hyperplaneExp3}
\end{figure}


%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\section{Metric Learning}
\label{sec:metricLearning} Machine learning algorithms often require
an embedding of data points into some space. Algorithms such as
$k$-nearest neighbors and neural networks assume the embedding space
to be $\mathbb{R}^n$ while SVM and other kernel methods embed the
data in a Hilbert space through a kernel operation. Whatever the
embedding space is, the notion of metric structure has to be
carefully considered. The popular assumption of a Euclidean metric
structure is often used without justification by data or  modeling
arguments. We argue that in the absence of direct evidence of
Euclidean geometry, the metric structure should be inferred from
data (if available). After obtaining the metric structure, it may be
passed to a learning algorithm for use in tasks such as
classification and clustering.
\paragraph{}

Several attempts have recently been made to learn the metric structure
of the embedding space from a given data set.
\emcite{Saul1997} use geometrical arguments
to learn optimal paths connecting two points in a space.
\emcite{Xing2003} learn a global
metric structure. that is able to capture
non-Euclidean geometry, but only in a restricted manner since the
metric is constant throughout the space.
\emcite{Lanckriet2002} learn a kernel matrix that represents
similarities between all pairs of the supplied data points.
While such an approach does learn the kernel structure from data,
the resulting Gram matrix does not generalize to unseen points.

\paragraph{}

Learning a Riemannian metric is also related to finding a
lower dimensional
representation of a dataset. Work in this area includes linear
methods such as principal component analysis and nonlinear methods
such as spherical subfamily models \cite{Gous1998} or
locally linear embedding \cite{Roweis2000} and
curved multinomial subfamilies \cite{Hall2000}.
Once such a submanifold is found, distances $d(x,y)$
may be computed as the lengths of shortest paths on the
submanifold connecting $x$ and $y$.
As shown in Section~\ref{sec:problem}, this approach
is a limiting case of learning a Riemannian metric for
the embedding high-dimensional space.
\paragraph{}

Lower dimensional representations are useful for visualizing high
dimensional data. However, these methods assume strict conditions
that are often violated in real-world, high dimensional data.
The obtained submanifold is tuned to the training data and
new data points will likely lie outside the submanifold due to noise.
It is necessary to specify some way of projecting the
off-manifold points into the manifold.
There is no notion of non-Euclidean geometry outside the submanifold
and if the estimated submanifold does not fit current and future data
perfectly, Euclidean projections are usually used.
\paragraph{}

Another source of difficulty is estimating the dimension of the submanifold.
The dimension of the submanifold is notoriously hard to
estimate in high dimensional sparse datasets.
Moreover, the data may have different lower dimensions in different
locations or may lie on several disconnected submanifolds thus
violating the assumptions underlying the submanifold approach.

\paragraph{}
We propose an alternative approach to the metric learning problem.
The obtained metric is local, thus capturing local variations
within the space, and is defined on the entire embedding space.
A set of metric candidates
is represented as a parametric family of transformations,
or equivalently as a parametric family of statistical models
and the obtained  metric is chosen from it based on
some performance criterion.

\paragraph{}
In Section~\ref{sec:problem} we discuss our formulation
of the Riemannian metric problem.
Section~\ref{sec:transformations} describes
the set of metric candidates as pull-back metrics of
a group of transformations followed by
a discussion of the resulting generative model in
Section~\ref{sec:simplexModel}.
In Section~\ref{sec:experimentsMetric}  we apply
the framework  to text classification and report experimental results
on the WebKB data.


%------------------------------------------------------------------------
\subsection{The Metric Learning Problem} \label{sec:problem}

The metric learning problem may be formulated as follows.
Given a differentiable manifold $\mathcal{M}$ and a dataset
$D=\{x_1,\ldots, x_N\}\subset\mathcal{M}$,
choose a Riemannian metric
$g$ from a set of metric candidates $\mathcal{G}$.
As in statistical inference,  $\mathcal{G}$ may be a parametric family
\begin{equation} \label{eq:paramMetricFam}
\mathcal{G}=\{g^{\theta}:\theta\in\Theta\subset\mathbb{R}^k\}
\end{equation}
or as in nonparametric statistics a less constrained set of candidates.
We focus on the parametric approach, as we believe it to
generally perform better in high dimensional sparse data such as
text documents.
The reason we use a superscript $g^{\theta}$ is that the subscript of the metric
is reserved for its value at a particular point of the manifold.

\paragraph{}
We propose to choose the metric based on maximizing
the following objective function $\mathcal{O}(g,D)$
\begin{equation} \label{eq:normInvVol}
\mathcal{O}(g,D)= \prod_{i=1}^{N}\frac{(\dvol
g(x_i))^{-1}}{\int_{\mathcal{M}}(\dvol g(x))^{-1}\text{d}x}
\end{equation}
where $\dvol g(x)=\sqrt{\det G(x)}$ is the differential volume
element, and $G(x)$ is the Gram matrix of the metric $g$ at the
point $x$. Note that $\det G(x)>0$ since $G(x)$ is positive
definite.


\paragraph{}
The volume element  $\dvol g(x)$ summarizes the
size of the metric $g$ at $x$ in one scalar. Intuitively, paths crossing
areas with high volume will tend to be longer than the same paths
over an area with low volume.
Hence %For this reason,
maximizing the inverse volume in \eqref{eq:normInvVol} will result in
shorter curves across densely populated regions of $\mathcal{M}$.
As a result, the geodesics will tend to pass through
densely populated regions. This agrees with the intuition
that distances between data points should be measured on the
lower dimensional data submanifold, thus capturing the intrinsic
geometrical structure of the data.

\paragraph{}
The normalization in \eqref{eq:normInvVol} is necessary since the
problem is clearly unidentifiable without it. Metrics $cg$ with
$0<c<1$ will always a have higher inverse volume element than $g$.
The normalized inverse volume element may be seen as a probability
distribution over the manifold. As a result, we may cast the problem
of maximizing $\mathcal{O}$ as a maximum likelihood problem.


\paragraph{}
If $\mathcal{G}$ is completely unconstrained, the metric maximizing
the above criterion will have a volume element tending to 0
at the data points and  $+\infty$  everywhere else.
Such a solution is analogous to estimating a distribution by
an impulse train at the data points and 0 elsewhere (the empirical
distribution). As in statistics we  avoid this degenerate solution by
restricting the set of candidates $\mathcal{G}$ to a small set of
relatively smooth functions.

\paragraph{}
The case of extracting a low dimensional submanifold (or linear subspace)
may be recovered from the above framework if $g\in\mathcal{G}$
is equal to the metric inherited from the embedding Euclidean space
across a submanifold and tending to $+\infty$ outside.
In this case distances between two points on the submanifold will
be measured as the shortest curve on the submanifold
using the Euclidean length element.

\paragraph{}

If $\mathcal{G}$ is a parametric family of metrics
$\mathcal{G}=\{g^{\lambda}:\lambda\in\Lambda\}$,
the log of the objective function $\mathcal{O}(g)$ is equivalent
to the loglikelihood $\ell(\lambda)$ under the model
\[p(x\,;\lambda)=\frac{1}{Z} \left(\sqrt{\det G^{\lambda}(x)}\right)^{-1}.\]
If $G$ is the Gram matrix of the Fisher information,
the above model is the inverse of Jeffreys' prior
$p(x)\propto \sqrt{\det G(x)}$.
However in the case of Jeffreys' prior, the metric
is known in advance and there is no need for  parameter estimation.
For prior work on connecting volume elements
and densities on manifolds refer to \cite{Murray1993}.

\paragraph{}
Specifying the family of metrics $\mathcal{G}$ is not an intuitive task.
Metrics are specified in terms of a local inner product and it may
be difficult to understand the implications of a specific choice
on the resulting distances.
Instead of specifying a parametric family of metrics as discussed
in the previous section, we specify a parametric family of transformations
$\{F_{\lambda}:\lambda\in\Lambda\}.$
The resulting set of metric candidates will be the pull-back metrics
$\mathcal{G}=\{F_{\lambda}^*\cJ:\lambda\in\Lambda\}$ of
the Fisher information metric $\cJ$.

\paragraph{}
If $F:(\cM,g)\to (\cN,\delta)$ is an isometry (recall that $\delta$
is the metric inherited from an embedding Euclidean space) we call
it a flattening transformation. In this case distances on the
manifold $(\cM,g)=(\cM,F^*\delta)$ may be measured as the shortest
Euclidean path on the manifold $\mathcal{N}$ between the transformed
points. $F$ thus takes a locally distorted space and converts it
into a subset of $\mathbb{R}^n$ equipped with the flat Euclidean
metric.

\paragraph{}
In the next sections we work out in detail an implementation of the
above framework in which the manifold $\mathcal{M}$ is the multinomial simplex.


%----------------------------------------------------------------------
\subsection{A Parametric Class of Metrics} \label{sec:transformations}
Consider the following family of diffeomorphisms
$F_{\lambda}:\Pn \to\Pn$
\begin{align*}
F_{\lambda}(x)=\left(\frac{x_1\lambda_1}{\dotp{x}{\lambda}},
    \ldots,
    \frac{x_{n+1}\lambda_{n+1}}{\dotp{x}{\lambda}}\right),
\quad \lambda\in\Pn
\end{align*}
where $\dotp{x}{\lambda}$ is the scalar product
$\sum_{i=1}^{n+1}x_i\lambda_i$. The family $F_{\lambda}$ is a Lie
group of transformations under composition whose parametric space is
$\Lambda=\Pn$. The identity element is
$(\frac{1}{n+1},\ldots,\frac{1}{n+1})$ and the inverse of
$F_{\lambda}$ is $(F_{\lambda})^{-1}=F_{\eta}$ where
$\eta_i=\frac{1/\lambda_i}{\sum_k 1/\lambda_k}$. The above
transformation group acts on $x\in\Pn$ by increasing the components
of $x$ with high $\lambda_i$ values while remaining in the simplex.
See Figure~\ref{fig:transGroup} for an illustration of the above
action in $\P_2$.
\begin{figure}
\begin{center}
\includegraphics[scale=0.46]{metricLearningFigures/transQuiver.eps}
\includegraphics[scale=0.46]{metricLearningFigures/transQuiver2.eps}
\caption{$F_{\lambda}$ acting on $\P_2$  for
$\lambda=(\frac{2}{10},\frac{5}{10},\frac{3}{10})$
(left) and $F_{\lambda}^{-1}$ (right) acting on $\P_2$.}
\end{center}\label{fig:transGroup}
\end{figure}


\paragraph{}
We will consider the pull-back metrics of the Fisher information $\mathcal{J}$
through the above transformation group as our parametric family of metrics
\[\mathcal{G}=\{F^*_{\lambda} \mathcal{J}: \lambda\in\Pn\}.\]
Note that since the Fisher information itself is a pullback metric from the
sphere under the square root transformation
(see Section~\ref{sec:probSimplexGeom}) we have that
$F^*_{\lambda}\mathcal{J}$ is also the pull-back metric of
$(\Snp,\delta)$ through the transformation
\begin{align*}
\hat{F}_{\lambda}(x)=\left(\sqrt{\frac{x_1\lambda_1}{\dotp{x}{\lambda}}},
    \ldots,
    \sqrt{\frac{x_{n+1}\lambda_{n+1}}{\dotp{x}{\lambda}}}\right),
\quad \lambda\in\Pn.
\end{align*}
As a result of the above observation we have the following closed form
for the geodesic distance under $F^*_{\lambda}\mathcal{J}$
\begin{align} \label{eq:geodesic}
d_{F^*_{\lambda}\mathcal{J}}(x,y)=\text{acos}\left(\sum_{i=1}^{n+1}
\sqrt{\frac{x_i\lambda_i}{\dotp{x}{\lambda}}
\frac{y_i\lambda_i}{\dotp{y}{\lambda}}} \right).
\end{align}
Note the only difference between \eqref{eq:geodesic} and
tf-idf cosine similarity measure \cite{Salton1983}
is the square root and the choice of the $\lambda$ parameters.

\paragraph{}
To apply the framework described in Section~\ref{sec:problem}
to the metric  $F^*_{\lambda}\mathcal{J}$ we need to
compute the volume element given by $\sqrt{\det F_{\lambda}^*\mathcal{J}}$.
We start by computing the Gram matrix
$[G]_{ij}=F_{\lambda}^*\mathcal{J}(\partial_i,\partial_j)$
where $\{\partial_i\}_{i=1}^n$ is a basis for $T_x\Pn$
given by the rows of the matrix
\begin{align} \label{eq:U}
U=\begin{pmatrix}
    1&0&\cdots&0&-1\\
    0&1&\cdots&0&-1\\
    \vdots&0&\ddots&0 &-1\\
    0&0&\cdots&1&-1
\end{pmatrix}\in\mathbb{R}^{n\times n+1}.
\end{align}
and computing $\det G$ in Propositions~\ref{prop:matrixForm}-\ref{prop:volElem} below.

\paragraph{}

\begin{prop} \label{prop:matrixForm}
The matrix $[G]_{ij}=F_{\lambda}^*\mathcal{J}(\partial_i,\partial_j)$
is given by
\begin{align} \label{eq:Gform}
G= JJ^{\top}=U(D-\lambda\alpha^{\top})(D-\lambda\alpha^{\top})^{\top}U^{\top}
\end{align}
where
$D\in\mathbb{R}^{n+1\times n+1}$ is a diagonal matrix whose entries are
$[D]_{ii}=\sqrt{\frac{\lambda_i}{x_i}}\frac{1}{2\sqrt{\dotp{x}{\lambda}}}$
and $\alpha$ is a column vector given by
$[\alpha]_i=\sqrt{\frac{\lambda_i}{x_i}}
\frac{x_i}{2\dotp{x}{\lambda}^{3/2}}$
\end{prop}
Note that all vectors are treated as column vectors and
for $\lambda,\alpha\in\mathbb{R}^{n+1}$,  $\lambda\alpha^{\top}\in\mathbb{R}^{n+1\times n+1}$ is the outer product matrix $[\lambda\alpha^{\top}]_{ij}=\lambda_i\alpha_j$.
\begin{proof}
The $j$th component of the vector
$\hat{F}_{\lambda *}v$ is
\begin{align*}
[\hat{F}_{\lambda*}v]_j &= \frac{\mbox{d}}{\mbox{d}t}
\sqrt{\frac{(x_j+tv_j)\lambda_j}{\dotp{x+tv}{\lambda}}} \Bigg|_{t=0}
    =\frac{1}{2}\frac{v_j\lambda_j}{\sqrt{x_j\lambda_j}\sqrt{\dotp{x}{\lambda}}}
    - \frac{1}{2}\frac{\dotp{v}{\lambda}\sqrt{x_j\lambda_j}}{\dotp{x}{\lambda}^{3/2}}.
\end{align*}

\paragraph{}
Taking the rows of $U$ to be the basis $\{\partial_i\}_{i=1}^n$
 for $T_x\P_n$ we have,
for $i=1,\ldots,n$ and $j=1,\ldots,n+1$,
\begin{align*}
[\hat{F}_{\lambda*}\partial_i]_j
    &= \frac{\lambda_j[\partial_i]_j}{2\sqrt{x_j\lambda_j}\sqrt{\dotp{x}{\lambda}}}
    - \frac{\sqrt{x_j\lambda_j}}{2\dotp{x}{\lambda}^{3/2}}\partial_i\cdot\lambda
    = \frac{\delta_{j,i}-\delta_{j,n+1}}{2\sqrt{\dotp{x}{\lambda}}}\sqrt{\frac{\lambda_j}{x_j}}
    - \frac{\lambda_i-\lambda_{n+1}}{2\dotp{x}{\lambda}^{3/2}}
        \sqrt{\frac{\lambda_j}{x_j}}x_j.
\end{align*}

\paragraph{}
If we define $J\in\mathbb{R}^{n\times n+1}$ to be the matrix whose rows are
$\{\hat{F}_*\partial_i\}_{i=1}^n$ we have
\begin{align*}
J &= U(D-\lambda\alpha^{\top}).
\end{align*}
\paragraph{}

Since the metric $F_{\lambda}^*\mathcal{J}$ is the pullback
of the  metric on $\Snp$
that is inherited from the Euclidean space through $\hat{F}_{\lambda}$
we have
\[[G]_{ij} = \dotp{\hat{F}_{\lambda *}\partial_i}{\hat{F}_{\lambda *}\partial_j}\]
and hence
\[G = JJ^{\top}=U(D-\lambda\alpha^{\top})(D-\lambda\alpha^{\top})^{\top}U^{\top}.\]
\end{proof}


\paragraph{}
\begin{prop} \label{prop:volElem}
The determinant of $F_{\lambda}^*\mathcal{J}$ is
\begin{align} \label{eq:detG}
{\text{det}} F_{\lambda}^*\mathcal{J}
 \propto \frac{\prod_{i=1}^{n+1}(\lambda_i/x_i)}{\dotp{x}{\lambda}^{n+1}}.
\end{align}
\end{prop}
\begin{proof}
We will factor $G$ into a product of square matrices
and compute $\det G$ as the product of the determinants of each factor.
Note that $G=JJ^{\top}$ does not qualify as such a factorization
since $J$ is not square.

\paragraph{}
By factoring a diagonal matrix $\Lambda$,
$[\Lambda]_{ii}=\sqrt{\frac{\lambda_i}{x_i}}\frac{1}{2\sqrt{\dotp{x}{\lambda}}}$
from $D-\lambda\alpha^{\top}$  we have
\begin{align} \label{eq:det}
J &=U\left(I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}\right)\Lambda\\
G &= U\left(I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}\right)\Lambda^2 \label{eq:det2}
\left(I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}\right)^{\top}U^{\top}.
\end{align}

\paragraph{}
We proceed by studying the eigenvalues and eigenvectors of
$I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$ in order
to simplify \eqref{eq:det2} via an eigenvalue decomposition.
First note that if $(v,\mu)$ is an eigenvector-eigenvalue pair of
$\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$ then
$(v,1-\mu)$ is an eigenvector-eigenvalue pair of
$I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$.
Next, note that vectors $v$ such that $x^{\top}v=0$ are
eigenvectors of $\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$
with eigenvalue 0. Hence they are also eigenvectors
of $I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$ with
eigenvalue $1$.
There are $n$ such independent vectors $v_1,\ldots,v_n$.
Since
$\text{trace}(I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}})=n$,
the sum of the eigenvalues is also $n$ and
we may conclude that the last of the $n+1$ eigenvalues is 0.
\paragraph{}

The eigenvectors of $I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}$
may be written in several ways. One possibility is
as the columns of the following matrix
\begin{align*} V =
\begin{pmatrix}
-\frac{x_2}{x_1} & -\frac{x_3}{x_1} & \cdots & -\frac{x_{n+1}}{x_1} & \lambda_1 \\
1                & 0                & \cdots & 0  &   \lambda_2   \\
0                & 1                & \cdots& 0 &  \lambda_3 \\
\vdots           & \vdots           & \ddots & \vdots  & \vdots\\
0                & 0                 & \cdots      & 1 &  \lambda_{n+1}
\end{pmatrix} \in\mathbb{R}^{n+1\times n+1}
\end{align*}
where the first $n$ columns are the eigenvectors that correspond to
unit eigenvalues and the last eigenvector corresponds to a 0 eigenvalue.

\paragraph{}
Using the above eigenvector decomposition  we have
$I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}=V \tilde{I} V^{-1}$
and $\tilde{I}$ is a diagonal matrix containing all the eigenvalues.
Since the diagonal of $\tilde{I}$ is $(1,1,\ldots,1,0)$ we may
write $I-\frac{\lambda x^{\top}}{\dotp{x}{\lambda}}=V^{|n}V^{-1|n}$ where
$V^{|n}\in\mathbb{R}^{n+1\times n}$ is $V$ with the last column removed
and  $V^{-1|n}\in\mathbb{R}^{n\times n+1}$  is $V^{-1}$ with the last row removed.
\paragraph{}
We have then,
\begin{align*}
\det G &= \det (U (V^{|n} V^{-1|n})\Lambda^2(V^{-1|n \top} V^{|n\top}) U^{\top}) \\
&= \det ((U V^{|n}) (V^{-1|n}\Lambda^2V^{-1|n \top}) (V^{|n\top} U^{\top}))\\
 &=  (\det (UV^{|n}))^2\,\,\, \det (V^{-1|n} \Lambda^2 V^{-1|n \top}).
\end{align*}

\paragraph{}
Noting that
\begin{align*}
UV^{|n} &=
\begin{pmatrix}
-\frac{x_2}{x_1} & -\frac{x_3}{x_1} & \cdots & -\frac{x_n}{x_1} & -\frac{x_{n+1}}{x_1}-1 \\
1                & 0                & \cdots & 0 & -1  \\
0                & 1                & \cdots& 0 &-1 \\
\vdots           & \vdots           & \ddots & \vdots & \vdots \\
0                & 0                 & \cdots & 1    & -1
\end{pmatrix} \in\mathbb{R}^{n\times n}
\end{align*}
we factor $1/x_1$ from the first row and add columns $2,\ldots,n$ to column 1
thus obtaining
\begin{align*}
\begin{pmatrix}
-\sum_{i=1}^{n+1}x_i & -x_3 & \cdots & -x_n & -x_{n+1}-x_11 \\
0                & 0                & \cdots & 0 & -1  \\
0                & 1                & \cdots& 0 &-1 \\
\vdots           & \vdots           & \ddots & \vdots & \vdots \\
0                & 0                 & \cdots & 1    & -1
\end{pmatrix}.
\end{align*}
Computing the determinant by minor expansion of the first column
we obtain
\begin{align}\label{eq:det1}
\det (UV^{|n})^2 &= \left(\frac{1}{x_1}\sum_{i=1}^{n+1}x_i\right)^2=
\frac{1}{x_1^2}.
\end{align}

An argument presented in Appendix~\ref{sec:diffVolApp} shows that
\begin{align} \label{eq:det11}
\det  V^{-1|n}\Lambda^2 V^{-1|n\top}
&=  \frac{x_1^2 \dotp{x}{\lambda}^{n-1}}{4^n\dotp{x}{\lambda}^{2n}} \prod_{i=1}^{n+1}\frac{\lambda_i}{x_i}.
\end{align}
By multiplying \eqref{eq:det11} and \eqref{eq:det1} we obtain \eqref{eq:detG}.
\end{proof}
\paragraph{}
Figure~\ref{fig:contours} displays the inverse volume
element on $\P_1$
with the corresponding geodesic distance from the left corner of
$\P_1$.
\begin{figure*}
\begin{center}
\includegraphics[scale=0.45]{metricLearningFigures/1DsimplexVolume.eps}
\includegraphics[scale=0.45]{metricLearningFigures/1DsimplexGeodesic.eps}
\end{center}
\caption{The inverse volume element $1/\sqrt{\det G(x)}$ as a function of $x\in\P_1$
(left) and the geodesic distance $d(x,0)$ from the left corner as a function $x\in\P_1$ (right).
Different plots represent different metric parameters
$\lambda\in \{(1/2,1/2), (1/3,2/3),(1/6,5/6), (0.0099, 0.9901)\}$.}
 \label{fig:contours}
\end{figure*}
\paragraph{}
Propositions~\ref{prop:matrixForm} and \ref{prop:volElem}
reveal the form of the objective function $\mathcal{O}(g,D)$.
In the next section we describe a maximum likelihood estimation
problem that is equivalent to maximizing  $\mathcal{O}(g,D)$
and study its properties.

%---------------------------------------------------------------
\subsection{An Inverse-Volume Probabilistic Model on the Simplex}
\label{sec:simplexModel}
Using proposition~\ref{prop:volElem}
we have that the objective function $\mathcal{O}(g,D)$
may be regarded as a likelihood function under the model
\begin{align} \label{eq:modelNorm}
p(x\, ; \lambda) &= \frac{1}{Z} \dotp{x}{\lambda}^{\frac{n+1}{2}}
\prod_{i=1}^{n+1}x_i^{1/2}
\quad  x\in\P_n, \,\lambda\in\Pn
\end{align}
where
$Z = \int_{\P_n}
 \dotp{x}{\lambda}^{\frac{n+1}{2}} \prod_{i=1}^{n+1}x_i^{1/2}\text{d}x$.
%\begin{prop}
The loglikelihood function for model \eqref{eq:modelNorm} is given by
\begin{align*}
\ell(\lambda\,; x) = \frac{n+1}{2}\log (\dotp{x}{\lambda}) - \log \int_{\P_n}
\dotp{x}{\lambda}^{\frac{n+1}{2}} \prod_{i=1}^{n+1}\sqrt{x_i}\,\, \text{d}x.
\end{align*}
%\end{prop}
%\begin{proof}
The Hessian matrix $H(x,\lambda)$ of the loglikelihood function
may be written as % $H_1(x,\lambda)-H_2(x,\lambda)$ where
\begin{align*}
 [H(x,\lambda)]_{ij} &= -k\frac{x_i}{\dotp{x}{\lambda}}\frac{x_j}{\dotp{x}{\lambda}}
-(k^2-k)L\left( \frac{x_i}{\dotp{x}{\lambda}}\frac{x_j}{\dotp{x}{\lambda}} \right)
+ k^2L\left( \frac{x_i}{\dotp{x}{\lambda}} \right)
L\left( \frac{x_j}{\dotp{x}{\lambda}} \right)
\end{align*}
where $k=\frac{n+1}{2}$ and $L$ is the positive linear functional
\[Lf=\frac{\int_{\P_n}\dotp{x}{\lambda}^{\frac{n+1}{2}} \prod_{l=1}^{n+1}\sqrt{x_l}\,\,\, f(x,\lambda)\,\,\, \text{d}x}{\int_{\P_n}\dotp{x}{\lambda}^{\frac{n+1}{2}} \prod_{l=1}^{n+1}\sqrt{x_l}\text{d}x}.\]
Note that the matrix given by $LH(x,\lambda)=[LH_{ij}(x,\lambda)]$
is negative definite due to its covariance-like form.
In other words, for every value of $\lambda$, $H(x,\lambda)$
is negative definite on average, with respect to the model
$p(x\,;\lambda)$.

%---------------------------------------------------------
\subsubsection{Computing the Normalization Term} \label{sec:computingZ}
We describe an efficient way
to compute the normalization term $Z$
through the use of dynamic programming and FFT.

\paragraph{}
Assuming that $n=2k-1$ for some $k\in\mathbb{N}$ we have
\begin{align*}
Z &= \int_{\P_n} \dotp{x}{\lambda}^{k} \prod_{i=1}^{n+1}x_i^{1/2} \text{d}x
= \sum_{a_1+\cdots+a_{n+1}=k:a_i\geq 0} \frac{k!}{a_1!\cdots a_{n+1}!}
    \prod_{j=1}^{n+1} \lambda_j^{a_j}
        \int_{\Pn} \prod_{j=1}^{n+1}x_j^{a_j+\frac{1}{2}}\\
&\propto \sum_{a_1+\cdots+a_{n+1}=k:a_i\geq  0}
\prod_{j=1}^{n+1} \frac{\Gamma(a_j+3/2)}{\Gamma(a_j+1)} \lambda_j^{a_j}.
\end{align*}
The following proposition and its proof describe a way to
compute the summation in $Z$ in $O(n^2\log n)$ time.

\begin{prop}
The normalization term for model \eqref{eq:modelNorm} may be
computed in $O(n^2\log n)$ time complexity.
\end{prop}
\begin{proof}
Using the notation $c_m = \frac{\Gamma(m+3/2)}{\Gamma(m+1)}$
the summation in $Z$ may be expressed as
\begin{align}
Z\propto \sum_{a_1=0}^k  c_{a_1}\lambda_1^{a_1}
 \sum_{a_2=0}^{k-a_1}  c_{a_2}\lambda_2^{a_2}
 \cdots
 \sum_{a_{n}=0}^{k-\sum_{j=1}^{n-1} a_j}  c_{a_n} \lambda_{n}^{a_{n}}
 c_{k-\sum_{j=1}^n a_j}\lambda_{n+1}^{k-\sum_{j=1}^n a_j}  \label{eq:sumZ}.
\end{align}
A trivial dynamic program can compute  equation \eqref{eq:sumZ} in
$O(n^3)$ complexity.


However, each of the single subscript sums in \eqref{eq:sumZ}
is in fact a linear convolution operation.
By defining
\[B_{ij}=\sum_{a_i=0}^j c_{a_i}\lambda_i^{a_i}
\cdots \sum_{a_n=0}^{j-\sum_{l=i}^{n-1}a_l}  c_{a_n}\lambda_n^{a_n}
c_{j-\sum_{l=i}^n a_l}\lambda_{n+1}^{j-\sum_{l=i}^n a_l}
\]
we have $Z=B_{1k}$ and the recurrence relation
$B_{ij}=\sum_{m=0}^j c_m \lambda_i^mB_{i+1,j-m}$
which is the linear convolution of $\{B_{i+1,j}\}_{j=0}^k$
with the vector $\{c_j\lambda_i^j\}_{j=0}^k$.
By performing the convolution in the frequency domain
filling in each row of the table $B_{ij}$ for $i=0,\ldots,n+1,j=0,\ldots,k$
takes $O(n\log n)$ complexity leading to a total of
$O(n^2\log n)$ complexity.
\end{proof}
The computation method described in the proof may be used to compute
the partial derivative of $Z$, resulting in $O(n^3\log n)$ computation
for the gradient.
By careful dynamic programming, the gradient vector may be
computed in $O(n^2\log n)$ time complexity as well.

%------------------------------------------------------------------------------



%----------------------------------------------------------------------
\subsection{Application to Text Classification} \label{sec:experimentsMetric}
In this section we describe applying the metric learning framework
to document classification and report some results on the WebKB
dataset \cite{Craven1998}.

\paragraph{}
We map documents to the simplex by multinomial MLE or MAP
estimation.  This mapping
results in a the well-known term-frequency (tf) representation
(see Section~\ref{sec:embedding}).

\paragraph{}
It is a well known fact that less common terms across the
text corpus tend to provide more discriminative information
than the most common terms. In the extreme case, stopwords
like \texttt{the}, \texttt{or} and \texttt{of} are often
severely down-weighted or removed from the representation.
Geometrically, this means that we would like the geodesics
to pass through corners of the simplex that correspond to
sparsely occurring words, in contrast to densely populated
simplex corners such as the ones that correspond to the stopwords above.
To account for this in our framework we learn the metric
$F_{\lambda}^*\mathcal{J}=(F_{\theta}^{-1})^*\mathcal{J}$
where $\theta$ is the MLE under model \eqref{eq:modelNorm}.
In other words, we are pulling back the Fisher information metric
through the inverse to the transformation
that maximizes the normalized inverse volume of $D$.


\paragraph{}
The standard tfidf representation of a document consists of
multiplying the tf parameter by an idf component
\[ idf_k = \log\frac{N}{\#\text{documents that word } k \text{ appears in}}.\]
Given the tfidf representation of two documents, their cosine
similarity is simply the scalar product between the two normalized
tfidf representations \cite{Salton1983}. Despite its simplicity the
tfidf representation leads to some of the best results in text
classification and information retrieval and is a natural candidate
for a baseline comparison due to its similarity to the geodesic
expression.

\paragraph{}
A comparison of the top and bottom terms between the metric learning
and idf scores is shown in Figure~\ref{fig:words}. Note that both
methods rank similar words at the bottom. These are the most common
words that often carry little information for classification
purposes. The top words however are completely different for the two
schemes. Note the tendency of tfidf to give high scores to rare
proper nouns while the metric learning method gives high scores for
rare common nouns. This difference may be explained by the fact that
idf considers appearance of words in documents as a binary event
while the metric learning looks at the number of appearances of a
term in each document. Rare proper nouns such as the high scoring
tfidf terms in Figure~\ref{fig:words} appear several times in a
single web page. As a result, these words will score higher with the
tfidf scheme but lower with the metric learning scheme.

\begin{figure}
\begin{center}
\begin{tabular}{|l|l|}\hline
tfidf & Estimated $\lambda$ \\ \hline \texttt{{\small tiff romano
potra }} &
\texttt{{\small disobedience seat alr}}\\
\texttt{{\small anitescu papeli theo}} &
\texttt{{\small seizure refuse delegated}}\\
\texttt{{\small echo chimera trestle}} &
\texttt{{\small soverigns territory}}\\
\texttt{{\small schlatter xiyong}} &
\texttt{{\small mobocracy stabbed}}\\
$\vdots$ & $\vdots$\\
\texttt{{\small at department with}}&
\texttt{{\small will course system}}\\
\texttt{{\small this by office course}}&
\texttt{{\small you page research with}}\\
\texttt{{\small are an from system}} &
\texttt{{\small  that by are at this}} \\
\texttt{{\small programming be last}}&
\texttt{{\small home from office or as}}\\
\hline
\end{tabular}
\end{center}
\caption{Comparison of top and bottom valued parameters for tfidf
and model \eqref{eq:modelNorm}. The dataset is the faculty vs.
student webpage classification task from WebKB dataset. Note that
the least scored terms are similar for the two methods while the top
scored terms are completely disjoint.} \label{fig:words}
\end{figure}
\paragraph{}
In Figure~\ref{fig:loglogPlots} the rank-value plot for the
estimated $\lambda$ values and idf is shown on a log-log scale. The
$x$ axis represents different words that are sorted by increasing
parameter value and the $y$ axis represents the $\lambda$ or idf
value. Note that the idf scores show a stronger linear trend in the
log-log scale than the $\lambda$ values.

\begin{figure}
\begin{center}
\includegraphics[scale=0.4]{metricLearningFigures/tfidfPlot.eps}
\includegraphics[scale=0.4]{metricLearningFigures/lambdaPlot.eps}
\end{center}
\caption{Log-log plots for sorted values of tfidf (top) and
estimated $\lambda$ values (bottom). The task is the same as in
Figure~\ref{fig:words}.}  \label{fig:loglogPlots}
\end{figure}

\paragraph{}
To measure performance in classification we compared the testing
error of a nearest neighbor classifier under several different
metrics. We compared tfidf cosine similarity and the geodesic
distance under the obtained metric obtained. Figure~\ref{nnResult}
displays test-set error rates as a function of the training set
size. The error rates were averaged over 20 experiments with random
sampling of the training set. The $\lambda$ parameter was obtained
by approximated gradient descent procedure using the dynamic
programming method described in Section~\ref{sec:computingZ}.
According to Figure~\ref{nnResult} the learned metric outperforms
the standard tfidf measure.

\begin{figure}
\begin{center}
\includegraphics[scale=0.45]{metricLearningFigures/studentFaculty.eps}
\includegraphics[scale=0.45]{metricLearningFigures/studentCourse.eps}
\includegraphics[scale=0.45]{metricLearningFigures/studentProject.eps}
\includegraphics[scale=0.45]{metricLearningFigures/CourseProject.eps}
\end{center}
\caption{Test set error rate for nearest neighbor classifier on
WebKB binary tasks. Distances were computed by geodesic for the
learned Riemannian metric (red dashed) and tfidf with cosine
similarity (blue solid). The plots are averaged over a set of 20
random samplings of training sets of the specified sizes, evenly
divided between positive and negative examples. } \label{nnResult}
\end{figure}

%-------------------------------------------------------

\subsection{Summary} \label{sec:summaryMetric}
We have proposed a new framework for the metric learning problem that enables
robust learning of a local metric for high dimensional sparse data. This is achieved
by restricting the set of metric candidates to a parametric family and selecting
a metric based on maximizing the inverse volume element.

\paragraph{}
In the case of learning a metric on the multinomial simplex, the metric candidates
are taken to be pull-back metrics of the Fisher information under a continuous group
of transformation. When composed with a square root, the transformations
are flattening transformation for the obtained metrics.
The resulting optimization problem may be interpreted as
 maximum likelihood estimation.

\paragraph{}
Guided by the well known principle that common words should have
little effect on the metric structure we learn the metric that is
associated with the inverse to the transformation that maximizes the
inverse volume of the training set. The resulting pull-back metric
de-emphasizes common words, in a way similar to tfidf. Despite the
similarity between the resulting geodesics and tfidf similarity
measure, there are significant qualitative and quantitative
differences between the two methods. Using a nearest neighbor
classifier in a text classification experiment, the obtained metric
is shown to significantly outperform the popular tfidf cosine
similarity.

\paragraph{}
The framework proposed in this section is quite general and allows
implementations in other domains. The key component is the
specification of the set of metric candidates possibly by parametric
transformations in a way that facilitates efficient computation and
maximization of the volume element.



%----------------------------------------------------------------
\section{Discussion}
\label{sec:discussion}

The use of geometric techniques in studying statistical learning
algorithms is not new. As mentioned in
Section~\ref{sec:previousWork} the geometric view of statistics was
developed over the past fifty years. The focus of research in this
area has been finding connections between geometric quantities under
the Fisher geometry and asymptotic statistical properties. A common
criticism is that the geometric viewpoint is little more than an
elegant way to explain these asymptotic properties. There has been
little success in carrying the geometric reasoning further to define
new models and inference methods that outperform existing models in
practical situations.

\paragraph{}
The contributions of this thesis may be roughly divided into three
parts. The first part contains the embedding principle and the novel
algorithms of Sections~\ref{sec:diffusion} and \ref{sec:hyperplane}
which are a direct response to the criticism outlined above. Based
on the Fisher geometry of the embedded data, we derive
generalizations of several popular state-of-the-art algorithms.
These geometric generalizations significantly outperform their
popular counterparts in the task of text classification.


\paragraph{}
The second part contains an extension of information geometry to
spaces of conditional models. \v{C}encov's important theorem is
extended to both normalized and non-normalized conditional models. A
previously undiscovered relationship between maximum likelihood for
conditional exponential models and minimum exponential loss for
AdaBoost leads to a deeper understanding of both models. This
relationship, together with the generalized \v{C}encov
characterization reveals an axiomatic framework for conditional
exponential models and AdaBoost. An additional bi-product of this
relationship is that it enables us to derive the AdaBoost analogue
of maximum posterior inference. This new algorithm performs better
than AdaBoost in cases where overfitting is likely to occur.

\paragraph{}
Information geometry has almost exclusively been concerned with the
Fisher geometry. The third part of this thesis extends the
Riemannian geometric viewpoint to data-dependent geometries. The
adapted geometry exhibits a close similarity in its functional form
to the tf-idf metric, but leads to a more effective metric for text
classification.


\paragraph{}
It is interesting to consider the motivation leading to the
algorithmic part of the thesis. The algorithms transfer the data
point into a Riemannian manifold $\Theta$ with the Fisher
information metric. A schematic view of this process is outlined in
Figure~\ref{fig:motivation}. The data $\{x_i\}$ is assumed to be
drawn from $\{p(x\,;\theta_i^{\text{true}}\}$. It is reasonable to
assume that the models $\{\theta_i^{\text{true}}\}$ capture the
essence of the data and may be used by the algorithms in place of
the data. Since we do not know the true parameters, we replace them
with an estimate $\{\hat\theta_i\}$ (See
Section~\ref{sec:previousWork} for more details). The final step is
to consider the estimated models $\{\hat\theta_i\}$ as points in a
manifold endowed with the Fisher information metric, whose choice is
motivated by \v{C}encov's theorem.

\begin{figure}
\begin{center}
{ \psfrag{m}{$\{x_i\}$} \psfrag{h}{$\{\theta_i^{\text{true}}\}$}
\psfrag{g}{$\{\hat\theta_i\}$}
\psfrag{f}{$\{\hat\theta_i\}\subset(\Theta,\cJ)$}
\includegraphics[scale=0.5]{motivationDiagram.eps}}
\end{center}
\caption{Motivation for the embedding principle. The data $\{x_i\}$
is assumed to be drawn from $\{p(x\,;\theta_i^{\text{true}}\}$. It
is reasonable to assume that the models $\{\theta_i^{\text{true}}\}$
capture the essence of the data and may be used by the algorithms in
place of the data. Since we do not know the true parameters, we
replace them with an estimate $\{\hat\theta_i\}$ (see
Section~\ref{sec:previousWork} for more details). The final step is
to consider the estimated models $\{\hat\theta_i\}$ as points in a
manifold endowed with the Fisher information metric, whose choice is
motivated by \v{C}encov's theorem.} \label{fig:motivation}
\end{figure}


\paragraph{}
An interesting prospect for future research is to develop further
the theory behind the embedding principle, and more generally the
geometric approach to machine learning. Theoretical results such as
generalization error bounds and decision theory might be able to
provide a comparison of different geometries. For example, error
bounds for nearest neighbor and other classification algorithms
might depend on the employed geometry. A connection between such
standard theoretical results and the geometry under consideration
would be a significant addition to the algorithmic ideas in this
thesis. Such a result may provide a more rigorous motivation for the
ideas in Figure~\ref{fig:motivation} that underlie a large part of
this thesis.

\paragraph{}
It is also interesting to consider the role of geometry in
developing learning theory results. Replacing Euclidean geometric
concepts, such as the Euclidean diameter of the data, with their
non-Euclidean counterparts may lead to a new understanding of
current algorithms and to alternative error bounds.

\paragraph{}
Despite the fact that information geometry is half a century old, I
believe it is only beginning to affect the design of practical
algorithms in statistical machine learning. Many machine learning
algorithms, including some of the most popular ones, make naive
unrealistic geometric assumptions. I hope that the contribution of
this thesis will draw greater attention to this fact and encourage
others to exploit the geometric viewpoint in the quest of designing
better practical algorithms.

%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
%----------------------------------------------------------------
\appendix
\section*{\begin{center}\Large{Appendix}\end{center}}
\section{Derivations Concerning Boosting and Exponential Models}
In this appendix we provide some technical details concerning
Section~\ref{sec:boostMl}.
We derive update rules for minimizing the exponential
loss of AdaBoost and the log-loss of exponential models,
derive the dual problem of regularized $I$-divergence minimization
and express the $I$-divergence between exponential models as a difference
between loglikelihoods.

\subsection{Derivation of the Parallel Updates} \label{sec:parUpdate}
Let $x\in\cX$ be an example in the training set, which is of size $N$,
$\tilde{y}$ be its label and $\cY$ is the set of all possible labels.
At a given iteration
$\theta_j$ denotes the $j$-th parameter of the model, and $\theta_j+\Delta\theta_j$ the
parameter at the following iteration.
\subsubsection{Exponential Loss}
The objective is to minimize
$\mathcal{E}_{exp}(\theta+\Delta\theta)-\mathcal{E}_{exp}(\theta)$.  In the
following $h_j(x,y)=f_j(x,y)-f_j(x,\tilde{y})$,
$q_{\theta}(y|x)=e^{\sum_j \theta_j h_j(x,y)}$,
$s_j(x,y)=\mbox{sign}(h_j(x,y))$,
$M=\mbox{max}_{i,y} \sum_j|h_j(x_i,y)|$,
$\omega_{i,y}=1-\sum_j \frac{|h_j(x_i,y)|}{M}$.
By Jensen's inequality applied to $e^x$ we have
\begin{eqnarray} \nonumber
\mathcal{E}_{exp}(\theta+\Delta\theta)-\mathcal{E}_{exp}(\theta) &=&
\sum_i\sum_y e^{\sum_j (\theta_j+\Delta\theta_j) h_j(x_i,y)}-\sum_i\sum_y e^{\sum_j \theta_j h_j(x_i,y)} \\ \nonumber
&=&  \sum_i\sum_y q_{\theta}(y|x_i)e^{\sum_j\Delta\theta_j
\frac{|h_j(x_i,y)|}{M}s_j(x_i,y)\, M} -\sum_i\sum_y q_{\theta}(y|x_i)\\ \nonumber
&\leq& \sum_i\sum_y  q_{\theta}(y|x_i) \left(\sum_j \frac{|h_j(x_i,y)|}{M}
e^{\Delta\theta_j s_j(x_i,y)M} +\omega_{i,y}-1\right)\\ \label{eq:auxFunc1}
&\deff& \mathcal{A}(\Delta\theta,\theta).
\end{eqnarray}

We proceed by finding the stationary point of the auxiliary function
with respect to $\Delta\theta_j$:
\begin{eqnarray} \nonumber
0&=&\frac{\partial \cA}{\partial \Delta\theta_j} \seq
 -\sum_i\sum_y q_{\theta}(y|x_i)h_j(x_i,y) e^{\Delta\theta_j s_j(x_i,y)\,M} \\ \nonumber
&=& -\sum_y \sum_{i:s_j(x_i,y)=+1}
q_{\theta}(y|x_i)h_j(x_i,y) e^{\Delta\theta_j M}-\sum_y \sum_{i:s_j(x_i,y)=-1}
q_{\theta}(y|x_i)h_j(x_i,y) e^{-\Delta\theta_j M} \\ \nonumber
&\Rightarrow&
e^{2M \Delta\theta_j}\sum_y \sum_{i:s_j(x_i,y)=+1}h_j(x_i,y)q_{\theta}(y|x_i)
\seq
\sum_y \sum_{i:s_j(x_i,y)=-1} |h_j(x_i,y)|q_{\theta}(y|x_i)
 \\ \nonumber
&\Rightarrow&
\Delta\theta_j \seq \frac{1}{2M}
\log \left(\frac{\sum_y \sum_{i:s_j(x_i,y)=-1}|h_j(x_i,y)|q_{\theta}(y|x_i)}
{\sum_y \sum_{i:s_j(x_i,y)=+1}|h_j(x_i,y)|q_{\theta}(y|x_i)}\right)
\end{eqnarray}


\subsubsection{Maximum Likelihood for Exponential Models}
For the normalized case, the
objective is to maximize the likelihood or minimize the log-loss. In
this section, the previous notation remains except for
$q_{\theta}(y|x)=\frac{e^{\sum_j \theta_j h_j(x,y)}}{\sum_y e^{\sum_j \theta_j
    h_j(x,y)}}$.  The log-likelihood is
\begin{eqnarray}\nonumber
\ell(\theta)&=&
\sum_i \log \frac{e^{\sum_j \theta_j f_j(x_i,y_i)}}{\sum_y e^{\sum_j \theta_j f_j(x_i,y_i)}}
\seq - \sum_i \log {\sum_y e^{\sum_j \theta_j (f_j(x_i,y)-f_j(x_i,y_i))}}
\end{eqnarray}
and the difference in the loss between two iterations is
\begin{eqnarray} \nonumber
\ell(\theta)-\ell(\theta+\Delta\theta)&=&
\sum_i \log \frac{\sum_y e^{\sum_j (\theta_j+\Delta\theta_j) (f_j(x_i,y)-f_j(x_i,y_i))}}
{\sum_y e^{\sum_j \theta_j (f_j(x_i,y)-f_j(x_i,y_i))}} \\ \nonumber
&=&\sum_i\log \frac{\sum_y e^{\sum_j (\theta_j+\Delta\theta_j) h_j(x_i,y)}}
{\sum_y e^{\sum_j \theta_j h_j(x_i,y)}} \\ \nonumber
&=& \sum_i \log \sum_y q_{\theta}(y|x_i) e^{\sum_j \Delta\theta_jh_j(x_i,y)} \\ \label{eq:convBound1}
&\leq& \sum_i\sum_y q_{\theta}(y|x_i)  e^{\sum_j \Delta\theta_jh_j(x_i,y)} -N\\ \nonumber
&=& \sum_i\sum_y q_{\theta}(y|x_i)  e^{\sum_j \Delta\theta_j\frac{|h_j(x_i,y)|}{M}s_j(x_i,y)\,M}-N \\ \label{eq:loglossAux}
&\leq& \sum_i\sum_y q_{\theta}(y|x_i) \left( \sum_j \frac{|h_j(x_i,y)|}{M}
e^{\Delta\theta_js_j(x_i,y)\,M}+\omega_{i,y}  \right)-N \\
&\deff& \mathcal{A}(\theta,\Delta\theta)
\end{eqnarray}
where in (\ref{eq:convBound1}) we used the inequality $\log x\leq x-1$
and in (\ref{eq:loglossAux}) we used Jensen's inequality.  The
derivative of (\ref{eq:loglossAux}) with respect to $\Delta\theta$ will be
identical to the derivative of (\ref{eq:auxFunc1}) and so the log-loss
update rule will be identical to the exponential loss update rule, but
with $q_{\theta}(y|x)$ representing a normalized exponential model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Derivation of the Sequential Updates} \label{sec:secUpdate}
The setup for sequential updates is similar to that for parallel
updates, but now only one parameter gets updated in each step,
while the rest are held fixed.

\subsubsection{Exponential Loss}
We now assume that that only $\theta_k$ gets updated. We also assume
(with no loss of generality) that each feature takes values
in $[0,1]$, making $h_k(x_i,y)\in[-1,1]$.
\begin{eqnarray} \nonumber
\mathcal{E}_{exp}(\theta+\Delta\theta)-\mathcal{E}_{exp}(\theta) &=&
\sum_i\sum_y e^{\sum_j \theta_j h_j(x_i,y)+\Delta\theta_k h_k(x_i,y)}-
\sum_i\sum_y e^{\sum_j \theta_j h_j(x_i,y)} \\ \label{eq:sameAsLogLoss}
&=& \sum_i\sum_y  q_{\theta}(y|x_i)\left(e^{\left(\frac{1+h_k(x_i,y)}{2}\right)\Delta\theta_k+\left(\frac{1-h_k(x_i,y)}{2}\right)(-\Delta\theta_k)}
-1\right)\\ \nonumber
&\leq& \sum_i\sum_y  q_{\theta}(y|x_i)
\left(\frac{1+h_k(x_i,y)}{2} e^{\Delta\theta_k} +
      \frac{1-h_k(x_i,y)}{2} e^{-\Delta\theta_k}-1\right) \\
&\deff& \mathcal{A}(\theta,\Delta\theta_k)\\ \nonumber
\end{eqnarray}
The stationary point of $\mathcal{A}$ (with respect to $\Delta\theta_k$) is
\begin{eqnarray} \nonumber
0 &=& \sum_i\sum_y  q_{\theta}(y|x_i)
\left(\frac{1+h_k(x_i,y)}{2} e^{\Delta\theta_k} +
      \frac{h_k(x_i,y)-1}{2} e^{-\Delta\theta_k}\right) \\ \nonumber
& \Rightarrow & e^{2\Delta\theta_k}\sum_i\sum_y  q_{\theta}(y|x_i)(1+h_k(x_i,y)) =
\sum_i\sum_y  q_{\theta}(y|x_i)(1-h_k(x_i,y)) \\ \nonumber
& \Rightarrow & \Delta\theta_k \seq
\frac{1}{2}\log \left(\frac{\sum_i\sum_y  q_{\theta}(y|x_i)(1-h_k(x_i,y))}{\sum_i\sum_y  q_{\theta}(y|x_i)(1+h_k(x_i,y))}\right)
\end{eqnarray}
\subsubsection{Log-Loss}
Similarly, for the log-loss we have
\begin{eqnarray} \nonumber
\ell(\theta)-\ell(\theta+\Delta\theta_k)&=&
\sum_i\log \frac{\sum_y e^{\sum_j \theta_j h_j(x_i,y) +\Delta\theta_kh_k(x_i,y)}}
{\sum_y e^{\sum_j \theta_j h_j(x_i,y)}}
= \sum_i \log \sum_y q_{\theta}(y|x_i) e^{\Delta\theta_kh_k(x_i,y)} \\ \label{eq:sameAsExpLoss}
&\leq& \sum_i\sum_y q_{\theta}(y|x_i) e^{\Delta\theta_kh_k(x_i,y)}   -N.
\end{eqnarray}
Equation (\ref{eq:sameAsExpLoss}) is the same as
(\ref{eq:sameAsLogLoss}), except that $q_{\theta}$ is now the
normalized model. This leads to exactly the same form of update rule
as in the previous subsubsection.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5
\subsection{Regularized Loss Functions} \label{sec:regulLossFunc}
We derive the dual problem for the non-normalized
regularized $I$-divergence minimization
and then proceed to derive a sequential update rule.
\subsubsection{Dual Function for Regularized Problem}
The regularized problem $(P_{1,\text{\small reg}})$ is equivalent to
\begin{eqnarray} \nonumber
\mbox{\it Minimize}  && D(p,q_0)+U(c)= \sum_{x} \tilde{p}(x) \sum_y p(y| x)
\left( \log \frac{p(y| x)}{q_0(y| x)} - 1\right)+ U(c) \\ \nonumber
\mbox{\it subject to} && f_j(p) = \sum_{x,y} \tilde{p}(x) p(y| x)
h_j(x, y)=c_j,\quad j=1,\ldots,m
\end{eqnarray}
where $c\in\mathbb{R}^m$ and $U:\mathbb{R}^m\to \mathbb{R}$ is a convex
function whose minimum is at $0$.  The Lagrangian turns out to
be
\begin{eqnarray} \nonumber
\mathcal{L}(p,c,\theta) = \sum_x \tilde{p}(x)\sum_y p(y|x)
\left( \log\frac{p(y|x)}{q_0(y|x)}-1- \dotp{\theta}{h(x,y)}\right) + U(c).
\end{eqnarray}
We will derive the dual problem for $U(c)=\sum_i \frac{1}{2}\sigma_i^2
c_i^2$.  The convex conjugate $U^*$ is
\begin{eqnarray}
{U}^*(\theta) &\deff& \inf_c \sum_i \theta_ic_i+U(c)=
\inf_c \sum_i \theta_ic_i+\sum_i \frac{1}{2}\sigma_i^2  c_i^2 \\ \nonumber
0&=&\theta_i+\sigma_i^2 c_i \quad\Rightarrow\quad c_i=-\frac{\theta_i}{\sigma_i^2} \\ \nonumber
{U}^*(\theta) &=& -\sum_i \frac{\theta_i^2}{\sigma_i^2}+
\sum_i \frac{1}{2}\sigma_i^2 \frac{\theta_i^2}{\sigma_i^4}
\;=\, -\sum_i \frac{\theta_i^2}{2\sigma_i^2}
\end{eqnarray}
and the dual problem is
\begin{eqnarray} \nonumber
\theta^\star &= & \argmax_\theta h_{1,\mbox{\it\scriptsize reg}}(\theta) \\
&=& \argmax_{\theta}  -\sum_x \tilde{p}(x) \sum_{y}
q_0(y| x)e^{\sum_{j} \theta_j \, h_j(x,y)} + {U}^*(\theta) \\ \nonumber
&=& \mbox{arg}\min_{\theta}  \sum_x \tilde{p}(x) \sum_{y}
q_0(y| x)e^{\sum_{j} \theta_j \, h_j(x,y)} + \sum_j \frac{\theta_j^2}{2\sigma_j^2}.
\end{eqnarray}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Exponential Loss--Sequential update rule}
We now derive a sequential update rule for $(P_{1,\text{\small ref}})$.
As before,
$q_0=1$ and we replace $\frac{1}{2\sigma_k^2}$ by $\beta$.
\begin{eqnarray} \nonumber
\lefteqn{\mathcal{E}_{exp}(\theta+\Delta\theta)-\mathcal{E}_{exp}(\theta) } &&\\
&=& \sum_i\sum_y \left( e^{\sum_j \theta_j h_j(x_i,y)+\Delta\theta_k h_k(x_i,y)}
- e^{\sum_j \theta_j h_j(x_i,y)}\right) +\beta (\theta_k+\Delta\theta_k)^2-\beta\theta_k^2 \\ \nonumber
&=& \sum_i\sum_y q_{\theta}(y|x_i)\left( e^{\left(\frac{1+h_k(x_i,y)}{2}\right)\Delta\theta_k+\left(\frac{1-h_k(x_i,y)}{2}\right)(-\Delta\theta_k)}
-1 \right)\\ \nonumber
&& \mbox{\ } + 2\beta\,\theta_k\,\Delta\theta_k+\beta\, \Delta\theta_k^2\\ \nonumber
&\leq& \sum_i\sum_y  q_{\theta}(y|x_i)
\left(\frac{1+h_k(x_i,y)}{2} e^{\Delta\theta_k} +
      \frac{1-h_k(x_i,y)}{2} e^{-\Delta\theta_k}-1\right) \\ \nonumber
&& \mbox{\ } + 2\beta\,\theta_k\, \Delta\theta_k+ \beta\, \Delta\theta_k^2
\\ \nonumber &\deff& \mathcal{A}(\theta,\Delta\theta_k)
\end{eqnarray}
The stationary point will be at the solution of the following equation
\begin{eqnarray} \nonumber
0 &=& \frac{\partial \mathcal{A}}{\partial\Delta\theta_k}=\frac{1}{2}\sum_i\sum_y  q_{\theta}(y|x_i)
\left((1+h_k(x_i,y)) e^{\Delta\theta_k} +
      (h_k(x_i,y)-1) e^{-\Delta\theta_k}\right)+2\beta\,(\theta_k +\Delta\theta_k).
\end{eqnarray}
that can be readily found by Newton's method. Since
the second derivative $\frac{\partial^2 \mathcal{A}}{\partial\Delta\theta_k^2}$
is positive, the auxiliary function is strictly convex and Newton's method will converge.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Divergence Between Exponential Models}  \label{sec:divExpModels}
In this subsection we derive the $I$-divergence between two exponential models,
one of which is the maximum likelihood model
as a difference in their loglikelihoods.
The log-likelihood of an exponential model $q_{\theta}$ is
\begin{eqnarray}\nonumber
\ell(\theta) &=&
  \frac{1}{n} \sum_i \log \frac{e^{\sum_j\theta_j f_j(x_i,y_i)}}{Z_{\theta,i}}
= \frac{1}{n} \sum_j \theta_j
    \sum_i f_j(x_i,y_i) - \frac{1}{n} \sum_i \log Z_{ \theta,i} \\ \nonumber
&=& \sum_j \theta_j  E_{\tilde{p}}[f_j] -  \frac{1}{n} \sum_i \log Z_{ \theta,i}
\end{eqnarray}
while the $I$-divergence is
\begin{eqnarray} \nonumber
 D(q_{\theta},q_{\eta}) &=& \frac{1}{n} \sum_i\sum_y q_{\theta}(y|x_i) \log
\frac{ q_{\eta}(y|x_i) }{ q_{\theta}(y|x_i) } \\ \nonumber
&=& \frac{1}{n}
\sum_i\sum_y q_{\theta}(y|x_i) \left( \log \frac{Z_{\eta,i}}{Z_{\theta,i}}
+\log \frac{e^{\sum_j \theta_{j}f_j(x_i,y)}}{e^{\sum_j \eta_{j}f_j(x_i,y)}} \right) \\ \nonumber
&=& \frac{1}{n} \sum_i\log \frac{Z_{\eta,i}}{Z_{\theta,i}}+\frac{1}{n}
\sum_i\sum_y q_{\theta}(y|x_i)
\sum_j f_j(x_i,y)(\theta_{j}-\eta_{j}) \\ \nonumber
&=& \frac{1}{n}  \sum_i\log \frac{Z_{\eta,i}}{Z_{\theta,i}} +
 \frac{1}{n} \sum_j(\theta_{j}-\eta_{j}) E_{q_{\theta}} [f_j].
\end{eqnarray}
This corresponds to the fact that the $I$ divergence between normalized exponential
models is the Bregman divergence, with respect to the cumulant function,
between the natural parameters.
If $q_{\theta}$ is the maximum likelihood model,  the moment constraints are
satisfied and
\begin{align*}
D(q_{\theta}^{\ss{ml}},q_{\eta}) =\frac{1}{n}
\sum_i\log \frac{Z_{\eta,i}}{Z_{\theta,i}^{\ss{ml}}} +
\sum_j(\theta^{\ss{ml}}_{j}-\eta_{j}) E_{\tilde{p}} [f_j]
= \ell(\theta^{\ss{ml}}) - \ell(\eta).
\end{align*}

%------------------------------------------------------------------------
\section{Gibbs Sampling from the Posterior of Dirichlet Process
Mixture Model based on a Spherical Normal Distribution}
\label{sec:appDPMM}

In this appendix we derive the Gibbs sampling from the posterior of
a Dirichlet Process Mixture Model (DPMM) based on a Spherical Normal
Distribution. For details on DPMM see
\cite{Ferguson73,Blackwell73,Antoniak74} and for a general
discussion on MCMC sampling from the posterior see \cite{Neal2000}.
As mentioned by \emcite{Neal2000} the vanilla Gibbs sampling
discussed below is not the most efficient and other more
sophisticated sampling scheme may be derived.

\paragraph{}
We will assume below that the data dimensionality is 2. All the
derivations may be easily extended to a higher dimensional case. We
denote the data points by $y=(y_1,\ldots,y_m)$ where $y_i\in\R^2$
and the spherical Gaussian parameters associated with the data by
$\theta=(\theta_1,\ldots,\theta_m), \theta_i\in \H^3$. We use the
notation $\theta_{-i}$ to denote
$\{\theta_1,\ldots,\theta_{i-1},\theta_{i+1},\ldots,\theta_m\}$. In
Gibbs sampling we sample from the conditionals
$p(\theta_i|\theta_{-i},y)$ repeatedly to obtain a sample from the
DPMM posterior $p(\theta|y)$. The conditional is proportional to
(e.g. \cite{Neal2000})
\begin{align*}
p(\theta_i|\theta_{-i},y) &\propto p(y_i|\theta_i)
p(\theta_i|\theta_{-i}) =
\frac{1}{2\pi\sigma_i^2}e^{-\|y_i-\mu_i\|^2/2\sigma_i^2}
\left(\frac{1}{n-1+\zeta}\sum_{j\neq i} \delta_{\theta_i,\theta_j} +
\frac{\zeta}{n-1+\zeta} G_0 (\theta_i) \right)
\end{align*}
where $\zeta$ is the ``power'' parameter of the DPMM and $G_0$ is a
conjugate prior to the spherical noral distribution
\begin{align*}
G_0(\mu_i,\sigma_i^2) = \text{Inv-}\Gamma^2(\sigma_i^2|\alpha,\beta)
N(\mu_i|\mu_0,\sigma_i)
\end{align*}

\paragraph{}
To sample from $p(\theta_i|\theta_{-i},y)$ we need to write the
product $p(y_i|\theta_i)G_0(\theta_i)$
\begin{align*}
p(y_i|\theta_i)G_0(\theta_i)  &=
\frac{1}{2\pi\sigma_i^2}e^{-\|y_i-\mu_i\|^2/2\sigma_i^2}
\frac{1}{2\pi\sigma_i^2}e^{-\|\mu_i-\mu_0\|^2/2\sigma_i^2}
\frac{\beta^{\alpha}}{\Gamma(\alpha)}(\sigma_i^2)^{-(\alpha+1)}e^{-\beta/\sigma_i^2}\\
&= \frac{\beta^{\alpha}}{4\pi^2\Gamma(\alpha)}
(\sigma_i^2)^{-(\alpha+3)}
\exp\left(-\frac{\|y_i-\mu_i\|^2+\|\mu_i-\mu_0\|^2+2\beta}{2\sigma_i^2}\right)
\end{align*}
as a distribution times a constant. To do so, we expand the
exponent's negative numerator
\begin{align*}
&\|\mu_i-y_i\|^2+\|\mu_i-\mu_0\|^2+2\beta \\
&=\left(2(\mu_i^1)^2-2\mu_i^1(y_i^1+\mu_0^1)+(y_i^1)^2+(\mu_0^1)^2+\beta\right)
+\left(2(\mu_i^2)^2-2\mu_i^2(y_i^2+\mu_0^2)+(y_i^2)^2+(\mu_0^2)^2+\beta\right)\\
&=2\left((\mu_i^1-(y_i^1+\mu_0^1)/2)^2 +
\frac{1}{2}(y_i^1)^2+\frac{1}{2}(\mu_0^1)^2+\frac{1}{2}\beta-(y_i^1+\mu_0^1)^2/4
\right)\\
&+2\left((\mu_i^2-(y_i^2+\mu_0^2)/2)^2 +
\frac{1}{2}(y_i^2)^2+\frac{1}{2}(\mu_0^2)^2+\frac{1}{2}\beta-(y_i^2+\mu_0^2)^2/4
\right)\\
&= 2\|\mu_i-(y_i+\mu_0)/2\|^2 +\|y_i\|^2 + \|\mu_0\|^2 + 2\beta -
\|y_i+\mu_0\|^2/2
\end{align*}
to obtain
\begin{align*}
&p(y_i|\theta_i)G_0(\theta_i)\\
=&\frac{\beta^{\alpha}}{4\pi^2\Gamma(\alpha)}
(\sigma_i^2)^{-(\alpha+3)}\exp\left(-\frac{\|\mu_i-(y_i+\mu_0)/2\|^2}{\sigma_i^2}\right)\exp\left(-\frac{\|y_i\|^2
+ \|\mu_0\|^2 + 2\beta - \|y_i+\mu_0\|^2/2}{2\sigma_i^2}\right)\\
&= \frac{\beta^{\alpha}}{4\pi^2\Gamma(\alpha)} \pi
N\left(\mu_i\Bigg|\frac{y_i+\mu_0}{2},\frac{\sigma_i}{\sqrt{2}}\right)
\frac{\Gamma(\alpha+1)}{(\beta^*)^{\alpha+1}}
\,\text{Inv-}\Gamma\left(\sigma_i^2|\alpha+1,\beta^*\right)\\
&=\frac{\alpha \beta^{\alpha}}{4\pi(\beta^*)^{\alpha+1}}
N\left(\mu_i\Bigg|\frac{y_i+\mu_0}{2},\frac{\sigma_i}{\sqrt{2}}\right)
\,\text{Inv-}\Gamma\left(\sigma_i^2|\alpha+1,\beta^*\right)
\end{align*}
where
\[\beta^*=\frac{\|y_i\|^2 +
\|\mu_0\|^2 + 2\beta - \|y_i+\mu_0\|^2/2}{2}.\]

\paragraph{}
Sampling is now trivial as the conditional
$p(\theta_i|\theta_{-i},y)$ is identified as a mixture model, with
known mixture coefficients, of impulses (probability concentrated on
a single element) and a normal-inverse-Gamma model.

%-------------------------------------------------------------------------
\section{The Volume Element of a Family of Metrics on the Simplex}
This appendix contains some calculations that are
used in Section~\ref{sec:metricLearning}.
Refer to that section for explanations of the notation and background.


\subsection{The Determinant of a Diagonal Matrix plus a Constant Matrix}
\label{sec:detDiagMatrix}
We prove some basic results concerning the determinants
of a diagonal matrix plus a constant matrix. These results
will be useful in Appendix~\ref{sec:diffVolApp}.

\paragraph{}
The determinant of a matrix $\det A\in\mathbb{R}^{n\times n}$
may be seen as a  function of the rows of $A$, $\{A_i\}_{i=1}^n$
\[ f:\mathbb{R}^n\times\cdots\times \mathbb{R}^n\to\mathbb{R}
\qquad f(A_1,\ldots,A_n)=\det A.\]
The multi-linearity property of the determinant means
that the function $f$ above is linear in each of its components
\begin{align*}
\forall j=1,\ldots,n\quad  f(A_1,\ldots,A_{j-1},A_j+B_j,A_{j+1},\ldots,A_n)
&= f(A_1,\ldots,A_{j-1},A_j,A_{j+1},\ldots,A_n)\\
&+ f(A_1,\ldots,A_{j-1},B_j,A_{j+1},\ldots,A_n).
\end{align*}


\begin{lemma}\label{lem1}
Let $D\in\mathbb{R}^{n\times n}$ be a diagonal matrix with $D_{11}=0$
and $\bf{1}$ a matrix of ones.
Then
\[\det (D-\boldsymbol{1}) = -\prod_{i=2}^m D_{ii}.\]
\end{lemma}
\begin{proof}
Subtract the first row from all the other rows to obtain
\[
\begin{pmatrix}
-1 & -1 & \cdots & -1\\
0 & D_{22} & \cdots & 0\\
\cdots & \cdots & \ddots &\cdots\\
0 & 0 & \cdots & D_{mm}
\end{pmatrix}.
\]
Now compute the determinant by the cofactor expansion along the first
column to obtain
\[\det(D-\boldsymbol{1})=(-1)\prod_{j=2}^m D_{jj}+0+0+\cdots+0.\]
\end{proof}


\begin{lemma}\label{lem2}
Let $D\in\mathbb{R}^{n\times n}$ be a diagonal matrix
and $\bf{1}$ a matrix of ones.
Then
\[\det (D-\boldsymbol{1}) = \prod_{i=1}^m D_{ii}-\sum_{i=1}^m\prod_{j\neq i}D_{jj}.\]
\end{lemma}
\begin{proof}
Using the multi-linearity property of the determinant we separate
the first row of $D-\boldsymbol{1}$ as
$(D_{11},0,\ldots,0)+(-1,\ldots,-1)$.
The determinant $\det D-\boldsymbol{1}$ then becomes
$\det A+\det B$ where $A$ is $D-\boldsymbol{1}$ with the first row
replaced by $(D_{11},0,\ldots,0)$ and $B$ is the
$D-\boldsymbol{1}$ with the first row
replaced by a vector or $-1$.

\paragraph{}
Using Lemma~\ref{lem1} we have
$\det B=-\prod_{j=2}^n D_{jj}$.
The determinant $\det A$ may be expanded along the first row
resulting in $\det A=D_{11}M_{11}$
where $M_{11}$ is the minor resulting from deleting
the first row and the first column.
Note that $M_{11}$ is the determinant of a matrix
similar to $D-\boldsymbol{1}$ but of size $n-1\times n-1$.
\paragraph{}
Repeating recursively the above multi-linearity argument
we have
\begin{align*}
\det(D-\boldsymbol{1})&=
-\prod_{j=2}^nD_{jj}+D_{11}
\left(
    -\prod_{j=3}^nD_{jj}+D_{22}
    \left(-\prod_{j=4}^nD_{jj}+D_{33}
        \left( -\prod_{j=5}^nD_{jj}+D_{44}(\cdots)
        \right)
    \right)
\right) \\
&=\prod_{i=1}^n D_{ii}-\sum_{i=1}^n\prod_{j\neq i}D_{jj}.
\end{align*}
\end{proof}



%-------------------------------------------------------------------
\subsection{The Differential Volume Element of $F^*_{\lambda}\mathcal{J}$} \label{sec:diffVolApp}
We compute below $\det V^{-1|n} \Lambda^2 V^{-1|n \top}$. See Proposition~\ref{prop:volElem}
for an explanation of the notation.

\paragraph{}
The inverse of $V$, as may be easily verified is,
\begin{align*}
V^{-1} &= \frac{1}{\dotp{x}{\lambda}}
\begin{pmatrix}
- x_1\lambda_2 & \dotp{x}{\lambda}-x_2\lambda_2&-x_3\lambda_2&\cdots&-x_{n+1}\lambda_2\\
- x_1\lambda_3 & -x_2\lambda_3 & \dotp{x}{\lambda}-x_3\lambda_3&\cdots&-x_{n+1}\lambda_3\\
\vdots &         \vdots &         &       \ddots \\
- x_1\lambda_{n+1} & -x_2\lambda_{n+1} & \cdots & \cdots & \dotp{x}{\lambda}-x_{n+1}\lambda_{n+1}\\
x_1\lambda_1 & x_2 \lambda_1 & \cdots & \cdots & x_{n+1}\lambda_1
\end{pmatrix}. \\
\intertext{Removing the last row gives}
V^{-1|n} &=  \frac{1}{\dotp{x}{\lambda}}
\begin{pmatrix}
- x_1\lambda_2 & \dotp{x}{\lambda}-x_2\lambda_2&-x_3\lambda_2&\cdots&-x_{n+1}\lambda_2\\
- x_1\lambda_3 & -x_2\lambda_3 & \dotp{x}{\lambda}-x_3\lambda_3&\cdots&-x_{n+1}\lambda_3\\
\vdots &         \vdots &         &       \ddots \\
- x_1\lambda_{n+1} & -x_2\lambda_{n+1} & \cdots & \cdots & \dotp{x}{\lambda}-x_{n+1}\lambda_{n+1}\\
\end{pmatrix}\\
 &=  \frac{1}{\dotp{x}{\lambda}}P
\begin{pmatrix}
- x_1 & \dotp{x}{\lambda}/\lambda_2-x_2&-x_3&\cdots&-x_{n+1}\\
- x_1 & -x_2 & \dotp{x}{\lambda}/\lambda_3-x_3&\cdots&-x_{n+1}\\
\vdots &         \vdots &         &       \ddots \\
- x_1 & -x_2 & \cdots & \cdots & \dotp{x}{\lambda}/\lambda_{n+1}-x_{n+1}
\end{pmatrix}.
\end{align*}
where
\begin{align*}
P&=
\begin{pmatrix}
\lambda_2 & 0 & \cdots &0\\
0         & \lambda_3 & \cdots &0\\
\vdots & &\ddots &\vdots\\
0 & 0 &0 &\lambda_{n+1}
\end{pmatrix}.
\end{align*}

\paragraph{}
$[V_n^{-1}\Lambda^2 V_n^{-1\top}]_{ij}$ is the scalar product
of the $i$th and $j$th rows of the following matrix
\begin{align*}
&V_n^{-1}\Lambda = \frac{1}{2}
\dotp{x}{\lambda}^{-3/2}P\\
&
\begin{pmatrix}
- \sqrt{x_1\lambda_1} & \frac{\dotp{x}{\lambda}}{\sqrt{x_2\lambda_2}}-\sqrt{x_2\lambda_2}&-\sqrt{x_3\lambda_3}&\cdots&-\sqrt{x_{n+1}\lambda_{n+1}}\\
- \sqrt{x_1\lambda_1} & -\sqrt{x_2\lambda_2} & \frac{\dotp{x}{\lambda}}{\sqrt{x_3\lambda_3}}-\sqrt{x_3\lambda_3}&\cdots&-\sqrt{x_{n+1}\lambda_{n+1}}\\
\vdots &         \vdots &         &       \ddots \\
- \sqrt{x_1\lambda_1} & -\sqrt{x_2\lambda_2} & \cdots & \cdots & \frac{\dotp{x}{\lambda}}{\sqrt{x_{n+1}\lambda_{n+1}}}-\sqrt{x_{n+1}\lambda_{n+1}}\\
\end{pmatrix}.
\end{align*}

\paragraph{}
We therefore have
\begin{align*}
 V_n^{-1}\Lambda^2 V_n^{-1\top} &=
    \frac{1}{4} \dotp{x}{\lambda}^{-2}PQP
\end{align*}
where
\begin{align*}
Q=
\begin{pmatrix}
\frac{\dotp{x}{\lambda}}{x_2\lambda_2}-1  & -1 & \cdots &-1\\
-1         & \frac{\dotp{x}{\lambda}}{x_3\lambda_3}-1  & \cdots &-1\\
\vdots & &\ddots &\vdots\\
-1 & -1 &-1 &\frac{\dotp{x}{\lambda}}{x_{n+1}\lambda_{n+1}}-1
\end{pmatrix}.
\end{align*}

As a consequence of Lemma~\ref{lem2} in Section~\ref{sec:detDiagMatrix} we have
\begin{align*}
\det Q &=
x_1\lambda_1\frac{\dotp{x}{\lambda}^n}{\prod_{i=1}^{n+1}x_i\lambda_i}
-x_1\lambda_1\frac{\dotp{x}{\lambda}^{n-1}\sum_{j=2}^{n+1}x_j\lambda_j}{\prod_{i=1}^{n+1}x_i\lambda_i}
=x_1^2\lambda_1^2 \frac{ \dotp{x}{\lambda}^{n-1}}{\prod_{i=1}^{n+1}x_i\lambda_i}.
\end{align*}
\paragraph{}
and we obtain
\begin{align*}
\det  V_n^{-1}\Lambda^2 V_n^{-1\top} &=
    (1/4)^n\dotp{x}{\lambda}^{-2n}\left(\prod_{i=2}^{n+1}\lambda_i\right)
    \,\, x_1^2\lambda_1^2 \frac{ \dotp{x}{\lambda}^{n-1}}{\prod_{i=1}^{n+1}x_i\lambda_i}
    \left(\prod_{i=2}^{n+1}\lambda_i\right)
=   \frac{x_1^2 \dotp{x}{\lambda}^{n-1}}{4^n\dotp{x}{\lambda}^{2n}} \prod_{i=1}^{n+1}\frac{\lambda_i}{x_i}.
\end{align*}


%------------------------------------------------------------------------------------------
%------------------------------------------------------------------------------------------
%------------------------------------------------------------------------------------------
\section{Summary of Major Contributions}
\label{sec:contributionSummary} Listed below are the major
contributions of this thesis and the relevant sections and
publications.
\paragraph{}
\begin{tabular}{|l|c|l|}
\hline Contribution & Section & Relevant Publications\\ \hline
\hline Equivalence of maximum likelihood for  &5, A& \emcite{Lebanon2001a} \\
conditional exponential models and  minimum&&\\
exponential loss for AdaBoost &&\\ \hline
Axiomatic characterization of Fisher geometry for   &6& \emcite{Lebanon2004a}\\
spaces of conditional probability models &&\emcite{Lebanon2005a} \\
\hline
The embedding principle and the corresponding  &7& \emcite{Lafferty2003}\\
natural geometries on the data space&&\\
\hline Diffusion Kernels on Statistical Manifolds &8& \emcite{Lafferty2003}\\
&& \emcite{Lafferty2004}\\
\hline Hyperplane Margin Classifiers on the &9& \emcite{Lebanon2004b} \\
Multinomial Manifold &&\\
\hline Learning framework for Riemannian metrics &10, B& \emcite{Lebanon2003c}\\
&& \emcite{Lebanon2003b}\\ \hline
\end{tabular}
%-------------------------------------------------------------------------------------------
\bibliographystyle{packages/mlapa}
\bibliography{bibliography}
\end{document}
