\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb}
\usepackage{algorithm,algpseudocode}
\usepackage{bm}
\usepackage{hyperref}
\usepackage{float}
\usepackage{subcaption}
\usepackage{enumitem}
\usepackage{xcolor}
\definecolor{darkgreen}{rgb}{0.0, 0.5, 0.0}
\definecolor{aaltoBlue}{RGB}{0, 51, 102}
\definecolor{pinegreen}{cmyk}{0.92,0,0.59,0.25}
\definecolor{royalblue}{cmyk}{1,0.50,0,0}
\definecolor{lavander}{cmyk}{0,0.48,0,0}
\definecolor{violet}{cmyk}{0.79,0.88,0,0}
\definecolor{lightblue}{RGB}{173, 216, 230}
\usepackage{tikz}
\usepackage{pgfplots}
\usetikzlibrary{calc,arrows.meta,positioning,fit,shapes.geometric,shapes.misc,intersections,patterns,decorations.pathreplacing,backgrounds,matrix}
\usepgfplotslibrary{fillbetween,groupplots}
\setcounter{secnumdepth}{-\maxdimen}
% Long unbreakable tokens (the ISO/IEC author names in citations, term names
% in the title) otherwise overflow the margin by a few points; a modest
% emergency stretch lets TeX loosen the paragraph instead.
\emergencystretch=2em
\usepackage{mleftright}
% Project macros used by this entry (auto-extracted).
\DeclareMathOperator*{\argmin}{arg\,min}
\newcommand{\datapoint}{\vz}
\newcommand{\dataset}{\mathcal{D}}
\newcommand{\defeq}{:=}
\newcommand{\expect}{\mathbb{E} }
\newcommand{\feature}{x}
\newcommand{\fixedpointop}{\mathcal{F}}
\newcommand{\hypospace}{\mathcal{H}}
\newcommand{\hypothesis}{h}
\newcommand{\iteridx}{t}
\newcommand{\learnthypothesis}{\widehat{\hypothesis}}
\newcommand{\lossfunc}[2]{L\left(#1,#2 \right)}
\newcommand{\mvnormal}[2]{\mathcal{N}\left(#1,#2\right)}
\newcommand{\paramspace}{\mathcal{W}}
\newcommand{\pdf}[2]{p^{(#1)}\left(#2\right)}
\newcommand{\reals}{\mathbb{R}}
\newcommand{\sampleidx}{r}
\newcommand{\samplesize}{m}
\newcommand{\vw}[0]{{\bf w}}
\newcommand{\vz}[0]{{\bf z}}
\newcommand{\weights}{\vw}

\begin{document}

\section{maximum likelihood}

\subsection*{Author and editors}
Alexander Jung\textsuperscript{*}, Department of Computer Science, Aalto University, Espoo, Finland (ORCID \mbox{\href{https://orcid.org/0000-0001-7538-0990}{0000-0001-7538-0990}}) \\
Editors: Konstantina Olioumtsevits (ORCID \mbox{\href{https://orcid.org/0000-0003-2697-8659}{0000-0003-2697-8659}}), Ekkehard Schnoor (ORCID \mbox{\href{https://orcid.org/0000-0001-8252-1499}{0000-0001-8252-1499}}), Aalto University, Espoo, Finland \\
\textsuperscript{*}Corresponding author: \href{mailto:alex.jung@aalto.fi}{alex.jung@aalto.fi}



\subsection{Abstract}
Maximum likelihood chooses the model parameters of a
		probabilistic model under which an observed dataset has the
		largest probability density (or probability mass), that is, the
		model parameters that maximize the likelihood function. The
		dataset is modeled
		as realizations of independent and identically distributed (i.i.d.) random variables (RVs) with a common
		probability density function (pdf) from a family indexed by the model parameters.
		Taking the negative logarithm turns the product of pdfs
		into an average of losses, so maximum likelihood is
		empirical risk minimization (ERM) with the negative log-likelihood as loss function;
		linear regression with Gaussian noise, logistic regression, and the
		Gaussian mixture model (GMM) are maximum likelihood under specific
		probabilistic models. For a normal distribution the estimates are the
		sample mean and the mean squared deviation; where no
		closed form exists, gradient descent (GD) or the expectation–maximization (EM) algorithm
		computes the estimate as a fixed-point iteration. Under regularity
		conditions the estimate converges to the true model parameters
		as the dataset grows and is asymptotically normal with the
		variance of the Cram\'{e}r--Rao bound. With a large
		hypothesis space the estimate overfits, and adding a
		penalty term to the negative log-likelihood, which is
		maximum a posteriori estimation, or shrinking the hypothesis space
		regularizes it.

\subsection{Definition}
Consider the $366$ daily maximum
		temperatures recorded in $2024$ at the weather station Krems an
		der Donau. Fig.~\ref{fig_maxlikelihood_dict}(a) shows their
		histogram. A probabilistic model for these numbers treats each
		of them as the realization of a random variable (RV) whose probability density function (pdf)
		belongs to a family indexed by a few model parameters, such as
		the normal distributions indexed by mean and variance.
		Maximum likelihood chooses the model parameters under which the
		recorded year is most probable: among all normal distributions, it
		picks the one whose pdf, evaluated at the $366$ observed
		temperatures and multiplied out, is largest. For the days at
		Krems, this is the normal distribution with mean
		$17.9^{\circ}$C and standard deviation $9.2^{\circ}$C, drawn in
		Fig.~\ref{fig_maxlikelihood_dict}(a).
\par
		Formally, consider a dataset $\dataset = \{\datapoint^{(1)},
		\ldots, \datapoint^{(\samplesize)}\}$ whose data points are
		realizations of independent and identically distributed (i.i.d.) RVs with a common
		pdf (or probability mass function (pmf)) $\pdf{\datapoint;\weights}{\cdot}$ that
		depends on the model parameters $\weights \in \paramspace$. The
		likelihood function of $\weights$ is the joint pdf of the
		dataset, viewed as a function of $\weights$ with the
		dataset fixed,
		\begin{equation}
			\label{equ_maxlikelihood_likelihood_dict}
			L^{(\dataset)}(\weights) \defeq
			\prod_{\sampleidx=1}^{\samplesize}
			\pdf{\datapoint;\weights}{\datapoint^{(\sampleidx)}} \text{.}
		\end{equation}
		The maximum likelihood estimate is a maximizer of the
		likelihood function \eqref{equ_maxlikelihood_likelihood_dict}
		(Casella and Berger, 2002; Fisher, 1922;
		Lehmann and Casella, 1998, Def.~3.1, p.~444). Since the logarithm is
		increasing, it is also a minimizer of the negative
		log-likelihood per data point,
		\begin{equation}
			\label{equ_maxlikelihood_estimate_dict}
			\widehat{\weights} \in \argmin_{\weights \in \paramspace}
			\frac{1}{\samplesize} \sum_{\sampleidx=1}^{\samplesize}
			\big( - \ln \pdf{\datapoint;\weights}{\datapoint^{(\sampleidx)}}
			\big) \text{,}
		\end{equation}
		in which the product has become a sum. The pdf is taken
		with respect to a base measure that does not depend on
		$\weights$; changing it multiplies the likelihood function by a
		factor independent of $\weights$, so the estimate is unchanged
		(Hastie et al., 2009, Sect.~8.2.2).
\par
		The negative log-likelihood per data point in the estimate
		\eqref{equ_maxlikelihood_estimate_dict} is an average loss.
		Each value $\weights$ of the model parameters defines a
		hypothesis $\hypothesis^{(\weights)}$, the pdf
		$\pdf{\datapoint;\weights}{\cdot}$, and the loss it incurs
		on a data point $\datapoint$ is
		$\lossfunc{\datapoint}{\hypothesis^{(\weights)}} \defeq - \ln
		\pdf{\datapoint;\weights}{\datapoint}$. A data point at which
		the pdf is small costs much, one at which it is large costs
		little. Maximum likelihood is therefore empirical risk minimization (ERM) over the
		hypothesis space $\hypospace = \{\hypothesis^{(\weights)} :
		\weights \in \paramspace\}$ with the negative log-likelihood as
		loss function, and the minimum in the estimate
		\eqref{equ_maxlikelihood_estimate_dict} is the training error of
		the learned hypothesis $\learnthypothesis =
		\hypothesis^{(\widehat{\weights})}$. Several machine learning (ML) methods are
		maximum likelihood under a specific probabilistic model. Linear regression
		with Gaussian noise of a fixed variance is maximum likelihood
		with the squared error loss (Bishop, 2006, Sect.~3.1.1).
		Logistic regression is maximum likelihood with the logistic loss, the
		negative log-likelihood of the label under a
		probabilistic model in which the sigmoid function of a linear
		function is the probability of the label
		(Bishop, 2006, Sect.~4.3.2). Learning a Gaussian mixture model (GMM) is maximum
		likelihood with the clustering error of a
		soft clustering, the negative logarithm of the mixture
		pdf (Bishop, 2006, Sect.~9.2.1).
\par
		Consider the normal distribution $\mvnormal{\mu}{\sigma^{2}}$ for
		scalar data points $\feature^{(1)}, \ldots,
		\feature^{(\samplesize)} \in \reals$, so that the
		model parameters are $\weights = (\mu, \sigma^{2})$. The
		negative log-likelihood per data point in the estimate
		\eqref{equ_maxlikelihood_estimate_dict} is then
		\begin{equation}
			\label{equ_maxlikelihood_gauss_dict}
			\frac{1}{2} \ln \big( 2 \pi \sigma^{2} \big)
			+ \frac{1}{2 \sigma^{2} \samplesize}
			\sum_{\sampleidx=1}^{\samplesize}
			\big( \feature^{(\sampleidx)} - \mu \big)^{2} \text{.}
		\end{equation}
		For a fixed $\sigma^{2}$, the Gaussian negative log-likelihood
		\eqref{equ_maxlikelihood_gauss_dict} is an increasing affine
		function of the average squared error, so the mean
		with the largest likelihood function is the mean with the
		smallest average squared error. The zero-gradient condition
		for the Gaussian negative log-likelihood
		\eqref{equ_maxlikelihood_gauss_dict} delivers the closed form
		\begin{equation}
			\label{equ_maxlikelihood_closedform_dict}
			\widehat{\mu} = \frac{1}{\samplesize}
			\sum_{\sampleidx=1}^{\samplesize} \feature^{(\sampleidx)}
			\text{,} \qquad
			\widehat{\sigma}^{2} = \frac{1}{\samplesize}
			\sum_{\sampleidx=1}^{\samplesize}
			\big( \feature^{(\sampleidx)} - \widehat{\mu} \big)^{2}
			\text{,}
		\end{equation}
		the sample mean and the mean squared deviation from it
		(Bishop, 2006, Sect.~1.2.4, Eqs.~(1.55)--(1.56)). For the
		days at Krems, $\widehat{\mu} = 17.9^{\circ}$C and
		$\widehat{\sigma}^{2} = 84.6$ squared degrees, and no pair of
		model parameters on a fine grid around them has a larger
		likelihood function. Fig.~\ref{fig_maxlikelihood_dict}(b) shows the
		average log-likelihood as a function of $\mu$ with
		$\sigma^{2}$ fixed at $\widehat{\sigma}^{2}$: a downward parabola
		with its vertex at the sample mean. The variance
		estimate in the closed form
		\eqref{equ_maxlikelihood_closedform_dict} is biased. Its
		expectation is $(\samplesize - 1) \sigma^{2} /
		\samplesize$, so it underestimates the variance by the
		factor $(\samplesize - 1) / \samplesize$, which is $0.997$ for
		the $366$ days (Bishop, 2006, Sect.~1.2.4, Eq.~(1.58)). The
		closed form is computed in one step, so there is no
		iteration to read as a fixed-point iteration.
\par
		For a GMM, for logistic regression, and for most
		probabilistic models with more than a few model parameters, the
		zero-gradient condition has no closed-form solution, and the
		estimate \eqref{equ_maxlikelihood_estimate_dict} is computed by
		an iterative method. Gradient descent (GD) on the average negative
		log-likelihood is a fixed-point iteration
		$\weights^{(\iteridx+1)} = \fixedpointop
		\big(\weights^{(\iteridx)}\big)$ for $\iteridx = 0, 1, \ldots$,
		whose fixed points are the points where the gradient
		of the negative log-likelihood vanishes. For probabilistic models
		with an unobserved attribute, such as the cluster index of
		a GMM, the expectation–maximization (EM) algorithm
		(Dempster et al., 1977) is the standard fixed-point iteration:
		one iteration never decreases the likelihood function, and
		its fixed points are again points with vanishing
		gradient (Bishop, 2006, Sect.~9.2.2). The negative
		log-likelihood of a GMM is not convex, so either
		method can stop at a local minimum
		(Bishop, 2006, Sect.~9.2.2).
\par
		How close is $\widehat{\weights}$ to the model parameters
		$\weights^{\star}$ of the probability distribution that generated the
		dataset? This is the generalization question for
		maximum likelihood, posed under the independent and identically distributed assumption (i.i.d.\ assumption). The
		expectation of the loss
		$\lossfunc{\datapoint}{\hypothesis^{(\weights)}}$ under
		$\pdf{\datapoint;\weights^{\star}}{\cdot}$ is the risk of
		$\hypothesis^{(\weights)}$. It exceeds the risk of
		$\hypothesis^{(\weights^{\star})}$ by the Kullback–Leibler divergence (KL divergence)
		from $\pdf{\datapoint;\weights^{\star}}{\cdot}$ to
		$\pdf{\datapoint;\weights}{\cdot}$, which is nonnegative
		(Bishop, 2006, Sect.~1.6.1). The true model parameters
		therefore minimize the risk, and maximum likelihood
		minimizes its empirical counterpart. Consider a single
		model parameter $w \in \paramspace \subseteq \reals$ with a
		true value $w^{\star}$ in the interior of $\paramspace$. Assume
		that the pdf is smooth in $w$, that distinct values of
		$w$ give distinct probability distributions, and that the support of the
		pdf does not depend on $w$. Then the
		zero-gradient condition of the log-likelihood has, with
		probability tending to one as $\samplesize \to \infty$, a
		root $\widehat{w}$ that converges in probability to
		$w^{\star}$ (Lehmann and Casella, 1998, Thm.~3.7, p.~447). Maximum likelihood is
		consistent. Any such consistent sequence of roots is
		asymptotically normal: with the Fisher information
		$I(w) \defeq \expect \big\{ \big( \frac{\partial}{\partial w}
		\ln \pdf{\datapoint;w}{\datapoint} \big)^{2} \big\}$ of one
		data point,
		\begin{equation}
			\label{equ_maxlikelihood_normal_dict}
			\sqrt{\samplesize} \big( \widehat{w} - w^{\star} \big)
			\to \mvnormal{0}{1 / I(w^{\star})}
			\quad \text{in distribution as } \samplesize \to \infty
			\text{,}
		\end{equation}
		(Hastie et al., 2009, Sect.~8.2.2, Eq.~(8.16);
		Lehmann and Casella, 1998, Thm.~3.10, p.~449). The information inequality,
		also known as the Cram\'{e}r--Rao bound, states that no
		unbiased estimator of $w$ that is computed from $\samplesize$
		data points and satisfies mild regularity conditions has
		a variance below $1 / (\samplesize I(w))$
		(Cramér, 1946; Lehmann and Casella, 1998, Thm.~5.10, p.~120). The
		asymptotic normality \eqref{equ_maxlikelihood_normal_dict} says
		that the variance of $\widehat{w}$ approaches this bound,
		which is why maximum likelihood is called asymptotically
		efficient. For the mean of a normal distribution with known
		variance $\sigma^{2}$, the Fisher information is $I(\mu) =
		1 / \sigma^{2}$, and the sample mean attains the bound
		exactly: its variance is $\sigma^{2} / \samplesize$. For
		the days at Krems, the standard deviation of $\widehat{\mu}$ is
		therefore $\widehat{\sigma} / \sqrt{366} = 0.48^{\circ}$C. The
		second derivative of the parabola in
		Fig.~\ref{fig_maxlikelihood_dict}(b) is $-1 /
		\widehat{\sigma}^{2}$, the Fisher information with a minus sign.
		The $365$ days of $2023$ give a second parabola in the same
		panel, with its vertex at the $2023$ mean of
		$17.4^{\circ}$C, half a degree from the $2024$ estimate.
\par
		Maximum likelihood chooses the model parameters that are best
		for the training set, and with a large hypothesis space this is
		overfitting (Murphy, 2012, Sect.~7.5.1). Enlarging the
		hypothesis space for the days at Krems shows the effect. With
		twelve means, one per calendar month, and one common
		variance, the average log-likelihood rises from $-3.64$ to
		$-2.84$ on the $366$ days of $2024$. It also rises, from $-3.63$
		to $-3.12$, on the $365$ days of $2023$, which did not enter the
		estimate: the seasons are real, so the larger hypothesis space
		helps on new data too. With one mean per calendar day,
		maximum likelihood places every mean on its observation,
		and the variance estimate is zero. The likelihood function
		of $2024$ then exceeds every bound as $\sigma \to 0$, while the
		average log-likelihood of $2023$ under the same
		model parameters falls below every bound: $-22$ at $\sigma =
		1^{\circ}$C and $-2083$ at $\sigma = 0.1^{\circ}$C. The same
		singularity is present in a
		GMM with a component whose mean coincides with one
		data point and whose variance shrinks to zero
		(Bishop, 2006, Sect.~9.2.1). Both remedies are forms of
		regularization. One shrinks the hypothesis space: fewer
		means, or a lower bound on the variance. The other
		adds a penalty term to the negative log-likelihood. This
		is maximum a posteriori (MAP) estimation,
		\begin{equation}
			\label{equ_maxlikelihood_map_dict}
			\widehat{\weights} \in \argmin_{\weights \in \paramspace}
			\frac{1}{\samplesize} \sum_{\sampleidx=1}^{\samplesize}
			\big( - \ln \pdf{\datapoint;\weights}{\datapoint^{(\sampleidx)}}
			\big) - \frac{1}{\samplesize} \ln \pdf{\weights}{\weights}
			\text{,}
		\end{equation}
		in which the penalty term is the negative logarithm of a
		prior distribution with pdf $\pdf{\weights}{\cdot}$ over the
		model parameters. A minimizer of the penalized negative
		log-likelihood \eqref{equ_maxlikelihood_map_dict} maximizes the
		posterior distribution of $\weights$ given the dataset
		(Bishop, 2006, Sect.~1.2.5, Eq.~(1.67)). A Gaussian
		prior distribution on the weight vector of linear regression
		gives ridge regression (Murphy, 2012, Sect.~7.5.1). A
		Gaussian prior distribution $\mvnormal{\mu_{0}}{\tau^{2}}$ on the
		mean of a normal distribution with known $\sigma^{2}$ moves
		the estimate from the sample mean $\widehat{\mu}$ toward
		$\mu_{0}$, to $(\samplesize \tau^{2} \widehat{\mu} + \sigma^{2}
		\mu_{0}) / (\samplesize \tau^{2} + \sigma^{2})$
		(Bishop, 2006, Sect.~2.3.6, Eq.~(2.141)). The
		penalty term in the MAP estimate
		\eqref{equ_maxlikelihood_map_dict} is divided by $\samplesize$,
		so it vanishes as the dataset grows, and the MAP estimate
		approaches the maximum likelihood estimate.
		\begin{figure}[H]
		\centering
		\begin{tikzpicture}
		\begin{groupplot}[
			group style={group size=2 by 1, horizontal sep=1.9cm},
			width=0.5\linewidth, height=0.44\linewidth,
			axis lines=left,
			tick label style={font=\scriptsize},
			label style={font=\scriptsize, align=center},
			legend style={draw=none, fill=none, font=\scriptsize,
				at={(0.03,0.97)}, anchor=north west},
			legend cell align=left,
		]
			\nextgroupplot[
				xlabel={daily maximum temperature in $^{\circ}$C\\(a) histogram and maximum likelihood pdf},
				ylabel={relative frequency},
				xmin=-8, xmax=44, ymin=0, ymax=0.056,
				scaled y ticks=false,
				yticklabel style={/pgf/number format/fixed, /pgf/number format/precision=2}]
			\addplot[ybar interval, fill=gray!25, draw=gray!60]
				table[col sep=comma, x=t, y=freq]{maxlikelihood_hist.csv};
			\addplot[black, thick]
				table[col sep=comma, x=t, y=pdf]{maxlikelihood_pdf.csv};
			\node[font=\scriptsize, anchor=east] at (axis cs:43,0.050) {$\mvnormal{\widehat{\mu}}{\widehat{\sigma}^{2}}$ (solid curve)};
			\nextgroupplot[
				xlabel={mean $\mu$ in $^{\circ}$C, $\sigma^{2} = \widehat{\sigma}^{2}$\\(b) average log-likelihood against the mean},
				ylabel={average log-likelihood},
				xmin=7.5, xmax=28.5, ymin=-4.52, ymax=-3.55,
				legend style={at={(0.5,0.02)}, anchor=south}]
			\addplot[black, thick]
				table[col sep=comma, x=mu, y=ll2024]{maxlikelihood_loglik.csv};
			\addlegendentry{days of 2024 (solid curve)}
			\addplot[black, thick, dashed]
				table[col sep=comma, x=mu, y=ll2023]{maxlikelihood_loglik.csv};
			\addlegendentry{days of 2023 (dashed curve)}
			\draw[gray, densely dotted, thick] (axis cs:17.907,-4.3) -- (axis cs:17.907,-3.55);
			\node[font=\scriptsize, anchor=south west] at (axis cs:18.2,-3.64) {$\widehat{\mu}$};
		\end{groupplot}
		\end{tikzpicture}
		\caption{(a) Histogram of the $366$ daily maximum
			temperatures of $2024$ at Krems an der Donau (bars, bins of
			$2^{\circ}$C) and the pdf of the normal distribution with
			the largest likelihood function (solid curve), with mean
			$\widehat{\mu} = 17.9^{\circ}$C and standard deviation
			$\widehat{\sigma} = 9.2^{\circ}$C. The two seasons give the
			histogram two bumps that no single normal distribution can
			follow. (b) The average log-likelihood of the $366$ days of
			$2024$ (solid curve) and of the $365$ days of $2023$ (dashed
			curve) as a function of the mean $\mu$, with the
			variance fixed at $\widehat{\sigma}^{2}$; the dotted
			line marks the maximum likelihood estimate $\widehat{\mu}$ of
			$2024$. Data generated by \texttt{pythondemos/maxlikelihood.py}
			\label{fig_maxlikelihood_dict}}
		\end{figure}

\subsection{Cross References}
likelihood function, probabilistic model, probability distribution, pdf, model parameter, estimator, ERM, loss function, training error, risk, i.i.d.\ assumption, normal distribution, sample mean, variance, linear regression, squared error loss, logistic regression, logistic loss, GMM, clustering error, EM, GD, fixed-point iteration, zero-gradient condition, KL divergence, overfitting, regularization, penalty term, prior distribution, posterior distribution, ridge regression, optimization problem

\subsection{References}
\begin{enumerate}
  \item Bishop (2006). Pattern Recognition and Machine Learning. Springer Science+Business Media, New York, NY, USA.
  \item Casella and Berger (2002). Statistical Inference. 2nd ed., Duxbury, Pacific Grove, CA.
  \item Fisher (1922). On the mathematical foundations of theoretical statistics. Philosophical Transactions of the Royal Society of London. Series A, 222, pp.~309--368.
  \item Lehmann and Casella (1998). Theory of Point Estimation. 2nd ed., Springer-Verlag, New York, NY, USA.
  \item Murphy (2012). Machine Learning: A Probabilistic Perspective. MIT Press, Cambridge, MA, USA.
  \item Cramér (1946). A contribution to the theory of statistical estimation. Skand. Akt. Tidskr., 29, pp.~85--94.
  \item Dempster et al. (1977). Maximum likelihood from incomplete data via the EM algorithm. J. Roy. Statist. Soc.: Ser. B (Methodological), 39(1), pp.~1--22.
  \item Hastie et al. (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction. 2nd ed., Springer Science+Business Media, New York, NY, USA.
\end{enumerate}

\end{document}
