\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb}
\usepackage{algorithm,algpseudocode}
\usepackage{bm}
\usepackage{hyperref}
\usepackage{float}
\usepackage{subcaption}
\usepackage{enumitem}
\usepackage{xcolor}
\definecolor{darkgreen}{rgb}{0.0, 0.5, 0.0}
\definecolor{aaltoBlue}{RGB}{0, 51, 102}
\definecolor{pinegreen}{cmyk}{0.92,0,0.59,0.25}
\definecolor{royalblue}{cmyk}{1,0.50,0,0}
\definecolor{lavander}{cmyk}{0,0.48,0,0}
\definecolor{violet}{cmyk}{0.79,0.88,0,0}
\definecolor{lightblue}{RGB}{173, 216, 230}
\usepackage{tikz}
\usepackage{pgfplots}
\usetikzlibrary{calc,arrows.meta,positioning,fit,shapes.geometric,shapes.misc,intersections,patterns,decorations.pathreplacing,backgrounds,matrix}
\usepgfplotslibrary{fillbetween,groupplots}
\setcounter{secnumdepth}{-\maxdimen}
% Long unbreakable tokens (the ISO/IEC author names in citations, term names
% in the title) otherwise overflow the margin by a few points; a modest
% emergency stretch lets TeX loosen the paragraph instead.
\emergencystretch=2em
\usepackage{mleftright}
% Project macros used by this entry (auto-extracted).
\newcommand{\covmtxgeneric}{\mC}
\newcommand{\dataset}{\mathcal{D}}
\newcommand{\eigval}[1]{\lambda_{#1}}
\newcommand{\feature}{x}
\newcommand{\featuredim}{d}
\newcommand{\featureidx}{j}
\newcommand{\featurevec}{\vx}
\newcommand{\mA}[0]{{\bf A}}
\newcommand{\mB}[0]{{\bf B}}
\newcommand{\mC}[0]{{\bf C}}
\newcommand{\mV}{\mathbf{V}}
\newcommand{\meanvecgeneric}{{\bm \mu}}
\newcommand{\mvneval}[3]{\mathcal{N}\left(#1 \,;\, #2, #3\right)}
\newcommand{\mvnormal}[2]{\mathcal{N}\left(#1,#2\right)}
\newcommand{\nrcluster}{k}
\newcommand{\reals}{\mathbb{R}}
\newcommand{\samplecovmtxgeneric}{\widehat{\mC}}
\newcommand{\sampleidx}{r}
\newcommand{\samplemeanvecgeneric}{\widehat{\bm \mu}}
\newcommand{\samplesize}{m}
\newcommand{\vc}[0]{{\bf c}}
\newcommand{\vx}[0]{{\bf x}}
\newcommand{\vz}[0]{{\bf z}}

\begin{document}

\section{multivariate normal distribution}

\subsection*{Author and editors}
Alexander Jung\textsuperscript{*}, Department of Computer Science, Aalto University, Espoo, Finland (ORCID \mbox{\href{https://orcid.org/0000-0001-7538-0990}{0000-0001-7538-0990}}) \\
Editors: Konstantina Olioumtsevits (ORCID \mbox{\href{https://orcid.org/0000-0003-2697-8659}{0000-0003-2697-8659}}), Ekkehard Schnoor (ORCID \mbox{\href{https://orcid.org/0000-0001-8252-1499}{0000-0001-8252-1499}}), Aalto University, Espoo, Finland \\
\textsuperscript{*}Corresponding author: \href{mailto:alex.jung@aalto.fi}{alex.jung@aalto.fi}

\subsection{Synonyms}
multivariate Gaussian distribution, joint normal
		distribution, jointly Gaussian distribution

\subsection{Abstract}
The multivariate normal distribution is the probability distribution of
		a Gaussian random vector in $\reals^{\featuredim}$,
		determined by a mean vector and a covariance matrix. Its
		probability density function (pdf) is proportional to the exponential of a negative
		quadratic form in the inverse of the covariance matrix, so its
		level sets are ellipsoids centered at the mean vector
		with axes along the eigenvectors of the covariance matrix and
		half-axes proportional to the square roots of its
		eigenvalues. The family is closed under affine maps and
		under conditioning, and among all random vectors with a
		given covariance matrix the Gaussian one has the largest
		differential entropy. The maximum likelihood estimates of the two
		parameters from a dataset are the sample mean
		and the sample covariance matrix, in closed form. When the covariance matrix
		is singular, no pdf exists and the random vector is
		confined to an affine subspace. In machine learning (ML), the multivariate
		normal distribution is the component of a Gaussian mixture model (GMM), the
		probabilistic model for the noise behind the squared error loss, and the
		class-conditional probability distribution behind linear discriminant analysis (LDA).

\subsection{Definition}
Consider the $366$
		days of $2024$ at Krems an der Donau, each a data point with a
		feature vector $\featurevec \in \reals^{2}$ that holds the
		morning minimum and the maximum daytime temperature of the day. A
		warm morning usually comes with a warm afternoon, so the days form
		an elongated cloud around their mean vector
		(Fig.~\ref{fig_mvndist_dict}(a)). The multivariate normal
		distribution is a probabilistic model for such feature vectors that
		describes this picture with two parameters
		(Anderson, 2003, Sect.~2.3;
		Bishop, 2006, Sect.~2.3). A mean vector
		$\meanvecgeneric \in \reals^{\featuredim}$ locates the center of
		the cloud, and a covariance matrix $\covmtxgeneric \in
		\reals^{\featuredim \times \featuredim}$ fixes its spread along
		every direction. The multivariate normal distribution is denoted by
		$\mvnormal{\meanvecgeneric}{\covmtxgeneric}$, and a
		random vector with this probability distribution is a Gaussian
		random vector.
\par
		For an invertible covariance matrix $\covmtxgeneric$, which is
		then positive definite (pd), the multivariate normal distribution has the
		probability density function (pdf) (Bishop, 2006, Sect.~2.3, Eq.~(2.43))
		\begin{equation}
			\label{equ_mvndist_pdf_dict}
			\mvneval{\featurevec}{\meanvecgeneric}{\covmtxgeneric} =
			\frac{1}{(2\pi)^{\featuredim/2} \sqrt{\det(\covmtxgeneric)}}
			\exp\Big( -\frac{1}{2} (\featurevec - \meanvecgeneric)^{\top}
			\covmtxgeneric^{-1} (\featurevec - \meanvecgeneric) \Big)
			\quad \text{for } \featurevec \in \reals^{\featuredim}
			\text{.}
		\end{equation}
		The exponent is a quadratic form in the inverse
		$\covmtxgeneric^{-1}$, the precision matrix. The pdf
		\eqref{equ_mvndist_pdf_dict} is largest at $\meanvecgeneric$ and
		falls off with the value of this quadratic form; the factor in
		front of the exponential, which contains the determinant of
		$\covmtxgeneric$, makes it integrate to one. A
		random vector with the pdf \eqref{equ_mvndist_pdf_dict}
		has mean vector $\meanvecgeneric$ and covariance matrix
		$\covmtxgeneric$, which is what the two symbols are named for
		(Bishop, 2006, Sect.~2.3, Eqs.~(2.59)--(2.64)). For
		$\featuredim = 1$, the pdf \eqref{equ_mvndist_pdf_dict} is
		the pdf of a Gaussian random variable (Gaussian RV) with mean $\mu$ and
		variance $\sigma^{2}$.
\par
		A level set of the pdf \eqref{equ_mvndist_pdf_dict} is the
		set of feature vectors at which the quadratic form takes a
		given value $c^{2}$, for some positive number $c$,
		\begin{equation}
			\label{equ_mvndist_levelset_dict}
			\big\{ \featurevec \in \reals^{\featuredim} :
			(\featurevec - \meanvecgeneric)^{\top} \covmtxgeneric^{-1}
			(\featurevec - \meanvecgeneric) = c^{2} \big\} \text{.}
		\end{equation}
		The covariance matrix is a symmetric matrix and admits a
		spectral decomposition $\covmtxgeneric = \mV \bm{\Lambda}
		\mV^{\top}$, with orthonormal eigenvectors as the
		columns of $\mV$ and the eigenvalues $\eigval{1} \geq
		\ldots \geq \eigval{\featuredim} > 0$ on the diagonal of
		$\bm{\Lambda}$. The level set \eqref{equ_mvndist_levelset_dict}
		is then an ellipsoid centered at $\meanvecgeneric$, whose axes
		point along the eigenvectors and whose half-axis along the
		$\featureidx$th eigenvector has length $c
		\sqrt{\eigval{\featureidx}}$ (Bishop, 2006, Sect.~2.3,
		Eqs.~(2.44)--(2.52), Fig.~2.7). On it, the pdf
		equals $\exp(-c^{2}/2)$ times its value at $\meanvecgeneric$.
		Fig.~\ref{fig_mvndist_dict}(a) shows the level sets at $c = 1$ and
		$c = 2$ for the days at Krems. The eigenvalues are $130.9$
		and $7.9$ squared degrees Celsius, so the level set at $c = 2$ has
		half-axes of $22.9$ and $5.6$ degrees Celsius. Its major axis
		points along the eigenvector $(0.61, 0.79)^{\top}$, the
		direction in which minimum and maximum rise together.
\par
		The probability that a Gaussian random vector lies
		inside the level set \eqref{equ_mvndist_levelset_dict} depends on
		$c$ and $\featuredim$ only: the quadratic form of a Gaussian
		random vector has a chi-squared distribution with
		$\featuredim$ degrees of freedom
		(Anderson, 2003, Thm.~3.3.3). For
		$\featuredim = 2$, this probability is $1 - \exp(-c^{2}/2)$,
		that is, $39.3$ percent at $c = 1$, $86.5$ percent at $c = 2$, and
		$98.9$ percent at $c = 3$. Fig.~\ref{fig_mvndist_dict}(b) compares
		these values with the fraction of the days at Krems inside each
		level set: $32.2$, $89.6$, and $100$ percent. The days are spread
		along the major axis more evenly than a Gaussian
		random vector would be, since the seasons move a day along
		this axis. Only $59.8$ percent of them lie within one standard
		deviation of the mean along it, against $68.3$ percent for a
		Gaussian random vector, and no day lies beyond $c = 2.6$. A
		Gaussian mixture model (GMM) with two components, one for the cold and one for the
		warm days, describes such a cloud better than a single
		multivariate normal distribution.
\par
		Every Gaussian random vector is an affine function of
		a standard normal random vector $\vz \in \reals^{\ell}$, whose entries are
		independent and identically distributed (i.i.d.) standard Gaussian RVs: $\featurevec = \mA \vz +
		\meanvecgeneric$ with a matrix $\mA \in \reals^{\featuredim
		\times \ell}$, and then $\covmtxgeneric = \mA \mA^{\top}$
		(Anderson, 2003, Sect.~2.4). This
		construction also covers a singular covariance matrix, for which the
		pdf \eqref{equ_mvndist_pdf_dict} is undefined. The
		random vector is then confined to an affine subspace,
		the column space of $\mA$ shifted by $\meanvecgeneric$, which
		has volume zero in $\reals^{\featuredim}$, so no pdf on
		$\reals^{\featuredim}$ exists
		(Anderson, 2003, Sect.~2.4;
		Lapidoth, 2017, Ch.~23). A singular covariance matrix arises
		whenever one feature is an affine function of the
		others. Appending the daily temperature range, the maximum minus
		the minimum, as a third feature of a day at Krems yields a
		covariance matrix of rank two, and the three-dimensional
		feature vectors lie on a plane.
\par
		Two properties make the multivariate normal distribution the
		default probabilistic model for numeric feature vectors. First, the
		family is closed under affine maps
		(Anderson, 2003, Thm.~2.4.5): for
		every matrix $\mB \in \reals^{q \times \featuredim}$ and
		every vector $\vc \in \reals^{q}$,
		\begin{equation}
			\label{equ_mvndist_affine_dict}
			\featurevec \sim \mvnormal{\meanvecgeneric}{\covmtxgeneric}
			\quad \text{implies} \quad
			\mB \featurevec + \vc \sim
			\mvnormal{\mB \meanvecgeneric + \vc}
			{\mB \covmtxgeneric \mB^{\top}} \text{.}
		\end{equation}
		Choosing $\mB$ to select some entries of $\featurevec$ shows that
		every marginal distribution of a multivariate normal distribution is
		again a multivariate normal distribution
		(Bishop, 2006, Sect.~2.3.2). For the days at Krems, the daily
		temperature range is the affine map \eqref{equ_mvndist_affine_dict}
		with $\mB = (-1, 1)$ and $\vc = 0$; under the probabilistic model, it
		has mean $10.7$ and variance $\mB \covmtxgeneric
		\mB^{\top} = 19.7$ squared degrees Celsius. The family is also
		closed under conditioning. Given the values of some entries of
		$\featurevec$, the conditional probability distribution of the remaining entries is
		a multivariate normal distribution. Its mean is an affine
		function of the given values, and its covariance matrix does
		not depend on them
		(Anderson, 2003, Thm.~2.5.1;
		Bishop, 2006, Sect.~2.3.1). Consider $\featuredim = 2$,
		and write $C_{1,1}$, $C_{1,2}$, and $C_{2,2}$ for the entries of
		the covariance matrix and $\mu_{1}$ and $\mu_{2}$ for the entries of
		the mean vector. The conditional probability distribution of
		$\feature_{2}$, given $\feature_{1}$, is then
		\begin{equation}
			\label{equ_mvndist_cond_dict}
			\mvnormal{\mu_{2} + \frac{C_{1,2}}{C_{1,1}}
			(\feature_{1} - \mu_{1})}
			{C_{2,2} - \frac{C_{1,2}^{2}}{C_{1,1}}} \text{.}
		\end{equation}
		The conditional mean in the conditional probability distribution
		\eqref{equ_mvndist_cond_dict} is the prediction of the
		maximum temperature from the minimum with the smallest
		risk under the squared error loss, and it is linear in
		$\feature_{1}$ (see linear regression). For the days at Krems, it is
		the line learned by least squares, with slope $1.10$, and the
		conditional standard deviation is $4.4$ degrees Celsius. Second, among all random vectors with the
		covariance matrix $\covmtxgeneric$, the Gaussian random vector
		has the largest differential entropy (Cover and Thomas, 2006, Thm.~8.6.5).
		When only the mean vector and the covariance matrix are
		known, the multivariate normal distribution is therefore the
		least restrictive probabilistic model.
\par
		The parameters $\meanvecgeneric$ and $\covmtxgeneric$ are
		learned from a dataset $\dataset = \{\featurevec^{(1)},
		\ldots, \featurevec^{(\samplesize)}\}$ by maximum likelihood.
		Under the independent and identically distributed assumption (i.i.d.\ assumption), the data points are
		realizations of i.i.d. random vectors
		$\featurevec^{(\sampleidx)} \sim
		\mvnormal{\meanvecgeneric}{\covmtxgeneric}$. The loss
		incurred by a data point is the negative logarithm of the
		pdf \eqref{equ_mvndist_pdf_dict} at its feature vector,
		and the average of this loss over $\dataset$ is the
		empirical risk of the pair $(\meanvecgeneric, \covmtxgeneric)$.
		Minimizing it is empirical risk minimization (ERM) over the hypothesis space of all
		pdfs \eqref{equ_mvndist_pdf_dict}. The minimizer is
		available in closed form, so no iterative algorithm is
		needed (Anderson, 2003, Thm.~3.2.1;
		Bishop, 2006, Sect.~2.3.4, Eqs.~(2.121)--(2.122)):
		\begin{equation}
			\label{equ_mvndist_mle_dict}
			\samplemeanvecgeneric = \frac{1}{\samplesize}
			\sum_{\sampleidx=1}^{\samplesize} \featurevec^{(\sampleidx)}
			\quad \text{and} \quad
			\samplecovmtxgeneric = \frac{1}{\samplesize}
			\sum_{\sampleidx=1}^{\samplesize}
			\big( \featurevec^{(\sampleidx)} - \samplemeanvecgeneric \big)
			\big( \featurevec^{(\sampleidx)} - \samplemeanvecgeneric
			\big)^{\top} \text{.}
		\end{equation}
		The estimates \eqref{equ_mvndist_mle_dict} are the sample mean
		and the sample covariance matrix of $\dataset$ with the divisor
		$\samplesize$; under the i.i.d.\ assumption, the sample covariance matrix
		is smaller than $\covmtxgeneric$ on average by the factor
		$(\samplesize - 1)/\samplesize$ (Bishop, 2006, Sect.~2.3.4,
		Eq.~(2.124)). For the days at Krems, the estimates
		\eqref{equ_mvndist_mle_dict} are $\samplemeanvecgeneric = (7.2,
		17.9)^{\top}$ and a sample covariance matrix with variances
		$54.2$ and $84.6$ and covariance $59.6$, all in squared
		degrees Celsius; the correlation coefficient of minimum and
		maximum temperature is $0.88$. The sample covariance matrix has
		rank at most $\samplesize - 1$, so with fewer than
		$\featuredim + 1$ data points it is singular and the
		pdf \eqref{equ_mvndist_pdf_dict} with these estimates is
		undefined. A common remedy replaces $\samplecovmtxgeneric$ by a
		weighted average of it and a positive multiple of the identity
		matrix (Hastie et al., 2009, Sect.~4.3.1,
		Eq.~(4.14)). This shrinks the
		hypothesis space to covariance matrices whose eigenvalues are
		bounded away from zero, one of the routes that
		regularization distinguishes.
\par
		The multivariate normal distribution is used throughout machine learning (ML).
		A GMM draws the feature vector of a data point from
		one of $\nrcluster$ multivariate normal distributions, so each
		cluster of a soft clustering is a mean
		vector with a covariance matrix, drawn as an ellipse
		(Bishop, 2006, Sect.~9.2). In regression, the
		label of a data point is modeled as the value of a
		hypothesis plus a Gaussian random variable (RV) with mean zero.
		Maximum likelihood estimation of the hypothesis is then
		ERM with the squared error loss, so linear regression by
		least squares assumes this probabilistic model for the
		label (Bishop, 2006, Sects.~1.2.5 and 3.1.1). In
		classification, modeling the feature vector of each class
		by a multivariate normal distribution and comparing the
		logarithms of the pdfs \eqref{equ_mvndist_pdf_dict} yields
		a classifier with the smallest risk under that
		probabilistic model. With one covariance matrix shared by all classes, the
		quadratic terms cancel and the decision boundary is a
		hyperplane, which is linear discriminant analysis (LDA). With a covariance matrix per class,
		the decision boundary is a quadratic surface
		(Bishop, 2006, Sect.~4.2.1; Hastie et al., 2009, Sect.~4.3,
		Eqs.~(4.9) and (4.12)).
		\begin{figure}[H]
		\centering
		\begin{tikzpicture}
		\begin{groupplot}[
			group style={group size=2 by 1, horizontal sep=1.6cm},
			width=0.47\linewidth, height=0.47\linewidth,
			axis lines=left,
			tick label style={font=\scriptsize},
			label style={font=\scriptsize, align=center},
			legend style={draw=none, fill=none, font=\scriptsize,
				at={(0.03,0.97)}, anchor=north west},
			legend cell align=left,
		]
			\nextgroupplot[
				axis equal image,
				xlabel={morning minimum temperature ($^{\circ}$C)\\(a) $21$ of the $366$ days and two level sets},
				ylabel={maximum daytime temperature ($^{\circ}$C)},
				xmin=-12, xmax=28, ymin=-4, ymax=36]
			\addplot[gray, densely dotted, thin, forget plot]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_axis1.csv};
			\addplot[gray, densely dotted, thin, forget plot]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_axis2.csv};
			\addplot[only marks, mark=o, mark size=1.6pt, black]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_days_plot.csv};
			\addplot[black, thick]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_ellipse1.csv};
			\addplot[black, thick, dashed]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_ellipse2.csv};
			\addplot[only marks, mark=x, mark size=3pt, thick, black]
				table[col sep=comma, x=tmin, y=tmax]{mvndist_mean.csv};
			\nextgroupplot[
				xlabel={level $c$\\(b) fraction inside the level set at $c$},
				ylabel={fraction inside the level set},
				xmin=0, xmax=3.5, ymin=0, ymax=1.05]
			\addplot[black, thick]
				table[col sep=comma, x=c, y=days]{mvndist_coverage.csv};
			\addlegendentry{days}
			\addplot[black, thick, dashed]
				table[col sep=comma, x=c, y=gauss]{mvndist_coverage.csv};
			\addlegendentry{Gaussian}
		\end{groupplot}
		\end{tikzpicture}
		\caption{(a) Every $18$th of the $366$ days of $2024$ at Krems,
			$21$ days (circles), the mean vector of all $366$
			days (cross), and the level sets \eqref{equ_mvndist_levelset_dict}
			of the multivariate normal distribution learned from them by
			maximum likelihood: the inner one at $c = 1$ (solid curve)
			and the outer one at $c = 2$ (dashed curve). The dotted
			segments are the two axes of the outer level set, along the
			eigenvectors of the
			covariance matrix, with half-lengths $22.9$ and $5.6$ degrees
			Celsius. (b) The fraction of the $366$ days inside the level
			set at $c$ (solid curve) against the probability
			$1 - \exp(-c^{2}/2)$ that a Gaussian random vector with
			the same parameters lies inside it (dashed curve). Data
			generated by \texttt{pythondemos/mvndist.py}
			\label{fig_mvndist_dict}}
		\end{figure}

\subsection{Cross References}
Gaussian RV, normal distribution, random vector, mean, covariance matrix, pdf, precision matrix, spectral decomposition, eigenvalue, eigenvector, standard normal random vector, maximum likelihood, sample mean, sample covariance matrix, i.i.d.\ assumption, ERM, GMM, linear regression, squared error loss, LDA, differential entropy, probabilistic model

\subsection{References}
\begin{enumerate}
  \item Anderson (2003). An Introduction to Multivariate Statistical Analysis. 3rd ed., Wiley, Hoboken, NJ.
  \item Bishop (2006). Pattern Recognition and Machine Learning. Springer Science+Business Media, New York, NY, USA.
  \item Lapidoth (2017). A Foundation in Digital Communication. 2nd ed., Cambridge Univ. Press, Cambridge, U.K.
  \item Cover and Thomas (2006). Elements of Information Theory. 2nd ed., Wiley, Hoboken, NJ, USA.
  \item Hastie et al. (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction. 2nd ed., Springer Science+Business Media, New York, NY, USA.
\end{enumerate}

\end{document}
