Bayesian_Methods.tex

% !TeX spellcheck = en_GB
\documentclass[10pt]{beamer}
\usetheme{CambridgeUS}
%\usetheme{Boadilla}
\definecolor{myred}{RGB}{163,0,0}
%\usecolortheme[named=blue]{structure}
\usecolortheme{dove}
\usefonttheme[]{professionalfonts}
\usepackage[english]{babel}
\usepackage{amsmath,amsfonts,amssymb}
\usepackage{xcolor}
\usepackage{bm}
\usepackage{gensymb}
\usepackage{verbatim} 
\usepackage{paratype}
\usepackage{mathpazo}
\usepackage{listings}
\lstset{language=R}

\usepackage{tikz}
\usetikzlibrary{matrix}

\DeclareMathOperator*{\interior}{int}
\DeclareMathOperator*{\rank}{rank}

% Number theorem environments
\setbeamertemplate{theorem}[ams style]
\setbeamertemplate{theorems}[numbered]

% Reset theorem-like environments so that each is numbered separately
\usepackage{etoolbox}
\undef{\definition}
\theoremstyle{definition}
\newtheorem{definition}{\translate{Definition}}
\newtheorem{Fact}{\translate{Fact}}

% Change colours for theorem-like environments
\definecolor{mygreen1}{RGB}{0,96,0}
\definecolor{mygreen2}{RGB}{229,239,229}
\setbeamercolor{block title}{fg=white,bg=mygreen1}
\setbeamercolor{block body}{fg=black,bg=mygreen2}



\alt<presentation>
{\lstset{%
  basicstyle=\footnotesize\ttfamily,
  commentstyle=\slshape\color{green!50!black},
  frame = single,  
  keywordstyle=\bfseries\color{blue!50!black},
  identifierstyle=\color{blue},
  stringstyle=\color{orange},
  %escapechar=\#,
  showstringspaces = false,
  showtabs = false,
  tabsize = 2,
  emphstyle=\color{red}}
}
{
  \lstset{%
    basicstyle=\ttfamily,
    keywordstyle=\bfseries,
    commentstyle=\itshape,
    escapechar=\#,
    showtabs = false,
	tabsize = 2,
    emphstyle=\bfseries\color{red}
  }
} 

\title{R404: Advanced Estimation Techniques}
\subtitle{Topic: Bayesian Methods in Econometrics}
\author{Andrey Vassilev}

\date{} 
    
\AtBeginSection{\frame{\usebeamerfont{section title}\centering\insertsection}}

\begin{document}
\maketitle



\begin{frame}[fragile]
\frametitle{Lecture Contents}
\tableofcontents
\end{frame}

\begin{section}{The philosophy of the Bayesian approach}\label{sec:BayesPhil}

\begin{frame}
\frametitle{A classical estimation example}
\begin{itemize}\itemsep1em
\item Consider a sample of iid observations $x_1,\ldots,x_n$ coming from a random variable $\xi \sim N(\mu,\sigma^2)$.
\item We are interested in obtaining an estimate of the mean.
\item A standard approach would be to use the method of maximum likelihood (MML).
\item We construct the likelihood function (recall the convention to write it as if conditioning on the data):
\begin{equation}
\label{eq:NormLik}
\begin{split}
L(\mu,\sigma^2|x_1,\ldots,x_n)= & \prod_{i=1}^{n}\frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{1}{2}\frac{(x_i-\mu)^2}{\sigma^2}}\\
= & \left(
\frac{1}{2\pi\sigma^2}
\right)^{n/2}e^{-\frac{1}{2}\sum_{i=1}^{n}\frac{(x_i-\mu)^2}{\sigma^2}}.
\end{split}
\end{equation}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{A classical estimation example}
\begin{itemize}\itemsep1em
\item According to the MML, we maximize $ L $ w.r.t. $ \mu $.
\item It is well-known (or, if your recollections are hazy, you can derive it) that the solution is given by the estimator $\hat{\mu}=\sum_{i=1}^{n}x_i/n$, i.e. the sample mean.
\item By definition, the statistic $\hat{\mu}$ is a random variable.
\item Consequently, if we keep repeating the experiment and regenerating the $ n $ observations, we'll obtain new samples $\tilde{x}_1,\ldots,\tilde{x}_n$,  $\tilde{\tilde{x}}_1,\ldots,\tilde{\tilde{x}}_n$ etc. and therefore new values of $\hat{\mu}$.
\item For each of those samples the corresponding value $\hat{\mu}$ will be our estimate of the unknown parameter $ \mu $.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{A classical estimation example}
\begin{itemize}\itemsep1em
\item When we speak of the statistical properties of $\hat{\mu}$ like unbiasedness or consistency, we are implicitly referring to an ability to repeat the experiment many times or to extend the sample size $ n $ within an experiment.
\item In this context, any probabilistic reasoning about $\hat{\mu}$ is based on a \emph{classical} notion of probability as the theoretical limit of the ratio of occurrences of an event to the total number of trials (i.e. the relative frequency).
\item It can be argued that this notion of probability is ``objective'' -- it derives from an experiment and reflects mechanisms external to an observer.
\item At the same time it is operational only when repeatability is ensured.
	\begin{itemize}\itemsep1em
	\item A football player is allowed to shoot one penalty but misses. Does it matter that he is the best scorer in his team?
	\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Subjective probability}
\begin{itemize}\itemsep1em
\item The idea of probability is often used in contexts where the classical interpretation as the limit of the relative frequency is not applicable.
\item Consider a person making the following statement: 
\begin{quote}
The probability that there is life on Mars is 1/1000000.
\end{quote}
\item What is this person trying to say?
	\begin{itemize}\itemsep1em
	\item If we could repeatedly try to find life on Mars, this would occur on average once in a million attempts?
	\item If we could recreate the universe over and over, the planet Mars would materialize and there would be life on it once in a million runs?
	\end{itemize}
\item \alert{The term ``probability'' in this context is used as a measure of the \emph{subjective} degree of certainty in the correctness of a statement.}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Subjective probability}
\begin{itemize}\itemsep0.7em
\item ``Probability'' in the above sense can be interpreted as a way to define a fair bet.
\item Take the statement \begin{quote}
The probability that team A will win the game against team B is 2/5.
\end{quote}
\item This can be construed to mean that the person making the statement considers it fair if he has to pay 2 dollars to enter a bet paying back 5 dollars if team A wins.
\item The is related to the concept of \emph{odds}. If the probability of an event is  $\frac{m}{m+n}$, then the associated odds would be $m:n$ ($m$ to $n$).
\item Thus, I'm willing to bet $ m $ dollars to get a profit of $ n $ dollars (recover the initial $ m $ and get additional $ n $ dollars) and I think neither side of the bet is getting an unfair advantage.
\item Clearly, subjective assessments of probability can differ in this context.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Axiomatic foundations of subjective probability}
\begin{itemize}\itemsep0.7em
\item The above considerations may seem informal but they turn out to be consistent with a formal definition of probability.
\item Given a measurable space $(\Omega,\mathcal{F})$ and a relation $\preceq$ over the elements of the $\sigma$-algebra $\mathcal{F}$ having particular properties, it can be shown that the relation $\preceq$ induces a probability measure $P$ on the space $(\Omega,\mathcal{F})$.
\item The ``particular properties'' are basically a formal way of expressing the idea that one event is ``more likely'' or ``more plausible'' than another.
\item The relation is called \emph{relative likelihood}.
\item The corresponding probability measure is known as \emph{subjective probability}.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Two interpretations of probability}
\begin{itemize}\itemsep1em
\item To summarise, the notion of probability can be used in (at least) two contexts:
	\begin{itemize}\itemsep1em
	\item To measure how likely a particular outcome of an ``experiment'' is. This encompasses all sorts of situations whose outcomes can be considered objective: coin tosses, recording data with some degree of imprecision (measurement error) etc.
	\item To measure the personal degree of certainty or conviction about something. This is by definition subjective and assessments about one and the same event can (will!) differ between different people. 
	\end{itemize}
\item With some simplification, objective probability can be thought of as applicable to the modelling of data generation processes with a stochastic element, while subjective probability is applicable to situations where we need to quantify our personal certainty (or ignorance) about something.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Subjective probability in a statistical context}
\begin{itemize}\itemsep1em
\item As a consequence of the above, a probability distribution can be used to measure our degree of uncertainty about the precise value of a numerical variable.
\item This is applicable to unknown parameters that are to be estimated from data.
\item In our example of estimating the mean $ \mu $ of the normally distributed random variable, taking a subjective probability point of view, it would be perfectly acceptable to treat $ \mu $ as a random variable with certain properties.
\item This would merely be a way to formalize our degree of knowledge about $ \mu $ and is therefore applicable even in situations when we know that $ \mu $ is in fact a (unknown) constant.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{The Bayesian approach}
\begin{itemize}\itemsep1em
\item Since both the objective and the subjective notions of probability ultimately lead to the same mathematical object, we can also combine them, provided that this combination has a meaningful interpretation. \pause
\item We can treat the available data as generated by an appropriate stochastic mechanism.
\item We can also treat model parameters as random if we agree to work with a subjectivist interpretation of probability.
\item This allows us to combine the sample $\mathbf{x}=(x_1,\ldots,x_n)$ and the parameters in a joint probability distribution (or density, in appropriate contexts).
\item In our example, the joint density would involve $\mathbf{x}$ and
$\mu$: $f(\mathbf{x},\mu)$. (It would also depend on $ \sigma^2 $ but we omit that for simplicity.)
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{The Bayesian approach}
\begin{itemize}\itemsep1em
\item This approach allows us to formalize parameter uncertainty but cannot magically help with getting more data or re-running the data generation process.
\item Since in many practical situations the sample $\mathbf{x}$ is fixed and getting more data is difficult or impossible, the object of interest is actually the conditional density of the parameter(s) given available data, i.e. $f(\mu|\mathbf{x})$.
\item The density $f(\mu|\mathbf{x})$ can be obtained by means of Bayes' formula for probability density functions, hence the term Bayesian approach:
\begin{equation}
\label{eq:Bayesformula}
f(\mu|\mathbf{x})=\frac{f(\mathbf{x},\mu)}{f(\mathbf{x})}=\frac{f(\mathbf{x}|\mu)f(\mu)}{f(\mathbf{x})}.
\end{equation}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{The Bayesian approach}
\begin{itemize}\itemsep1em
\item The density $ f(\mathbf{x}|\mu) $ is the \emph{likelihood} function for the model. It reflects the data generation mechanism and thus captures the ``objective'' component of the density $f(\mu|\mathbf{x})$.
\item The density $ f(\mu) $ is called the \emph{prior} density. It is formed on the basis of information outside the model (prior to observing the data) and can incorporate a variety of factors like:
	\begin{itemize}\itemsep1em
	\item theoretical considerations
	\item results from previous studies and experiments
	\item subjective judgement on what is reasonable
	\item the modeller's uncertainty about the parameters
	\end{itemize}
\item While the prior corresponds to the ``subjective'' component of the density $f(\mu|\mathbf{x})$, the above factors show that it need not be \emph{ad hoc} or arbitrary but can rigorously incorporate additionally available information.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{The Bayesian approach}
\begin{itemize}\itemsep1em
\item The density $f(\mu|\mathbf{x})$ is called the \emph{posterior} density. It combines information coming from the data via the likelihood with additional information coming from the prior.
\item For a given sample the last component of formula \eqref{eq:Bayesformula} -- the density $f(\mathbf{x})$ -- is a constant and does not affect the computations substantively but acts only as a scaling factor. For that reason, formula \eqref{eq:Bayesformula} is often written as
\begin{equation}
\label{eq:BayesProportional}f(\mu|\mathbf{x})\propto
f(\mathbf{x}|\mu)f(\mu),
\end{equation}where $\propto$ denotes proportionality.
\item Sometimes the expression on the right-hand side of $ \propto $ in \eqref{eq:BayesProportional} is called the \emph{kernel} of the posterior density.
\item The above construction obviously does not depend on the dimension of $ \mu $ and will remain valid in a multidimensional setting.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Interpretations of Bayes' formula}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item We can interpret \eqref{eq:BayesProportional} as a mechanism to update an initial body of information in light of new empirical evidence (data).
\item Alternatively, \eqref{eq:BayesProportional} can be interpreted as a mechanism to inform empirical analysis by introducing already available information.
\item In any case this presents the issue of how information is encoded in a prior distribution.
\item At a minimum, this is done by choosing a distribution with appropriate support and calibrating its parameters to ensure, for example, a specific mean and variance.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Some advantages and limitations of the Bayesian approach}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item Bayesian methods can alleviate small-sample problems where the data do not contain enough information to use a classical approach.
\item Bayesian methods can be used to impose theoretically motivated constraints on model parameters.
\item There is the risk that a strong prior will predetermine the outcome of an analysis. This means we can essentially force any result we want.
\item Bayesian methods can be computationally demanding.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{How does it work?}
\framesubtitle{Revisiting the $ \mu $ estimation example in a Bayesian framework}
Let's go back to the example of estimating $ \mu $ from a sample of iid observations from a $N(\mu,\sigma^2)$ random variable with $ \sigma^2 $ known. \bigskip

The likelihood \eqref{eq:NormLik} contains the expression
\begin{equation}
\label{eq:NormLik1}
\begin{split} \sum_{i=1}^{n}(x_i-\mu)^2= &
\sum_{i=1}^{n}\left((x_i-\hat{\mu})-(\mu-\hat{\mu})\right)^2\\
=&\sum_{i=1}^{n}(x_i-\hat{\mu})^2-\sum_{i=1}^{n}2(x_i-\hat{\mu})(\mu-\hat{\mu})+n(\mu-\hat{\mu})^2\\=&\sum_{i=1}^{n}(x_i-\hat{\mu})^2-2(\mu-\hat{\mu})\underbrace{\sum_{i=1}^{n}(x_i-\hat{\mu})}_{0}+n(\mu-\hat{\mu})^2\\
=&\sum_{i=1}^{n}(x_i-\hat{\mu})^2+n(\mu-\hat{\mu})^2 = \nu s^2+n(\mu-\hat{\mu})^2,
\end{split}
\end{equation}
where $\nu=n-1$ and $s^2=\nu^{-1}\sum_{i=1}^{n}(x_i-\hat{\mu})^2$.
\end{frame}

\begin{frame}[fragile]
\frametitle{How does it work?}
\framesubtitle{Revisiting the $ \mu $ estimation example in a Bayesian framework}
Therefore, the likelihood \eqref{eq:NormLik} can be written as
\begin{equation}
\label{eq:NormLik2}
L(\mu,\sigma^2|x_1,\ldots,x_n)=\left( \frac{1}{2\pi\sigma^2}
\right)^{n/2}\exp\left(-\frac{1}{2\sigma^2}\left(\nu
s^2+n(\mu-\hat{\mu})^2\right)\right).
\end{equation} \bigskip

Suppose our prior information on $\mu$ can be summarised by the density
\begin{equation}
\label{eq:NormPrior}
f(\mu)=\frac{1}{\sqrt{2\pi}\sigma^2_a}\exp\left(
-\frac{1}{2\sigma^2_a}(\mu-\mu_a)^2 \right),
\end{equation}
where $\mu_a$ is the prior mean and $\sigma_a^2$ is the prior variance of the person conducting the analysis.
\end{frame}

\begin{frame}[fragile]
\frametitle{How does it work?}
\framesubtitle{Revisiting the $ \mu $ estimation example in a Bayesian framework}
Combining \eqref{eq:NormLik2} and
\eqref{eq:NormPrior} by means of \eqref{eq:BayesProportional},
we obtain \begin{equation}
\label{NormPosterior} \begin{split}
f(\mu|\mathbf{x})\propto & \exp\left( -\frac{1}{2}\left[
\frac{(\mu-\mu_a)^2}{\sigma_a^2}+\frac{n}{\sigma^2}(\mu-\hat{\mu})^2
\right] \right) \\ \propto & \exp \left( -\left(
\frac{\sigma_a^2+\sigma^2/n}{2\sigma^2_a\sigma^2/n} \right)\left(
\mu-\frac{\hat{\mu}\sigma^2_a+\mu_a\frac{\sigma^2}{n}}{\sigma^2_a+\sigma^2/n}
\right)^2 \right). \end{split}
\end{equation}\bigskip

Thus, the posterior distribution of $\mu$ is normal with mean
$$\mathbb{E}[\mu]=\frac{\hat{\mu}\sigma^2_a+\mu_a\sigma^2/n}{\sigma^2_a+\sigma^2/n}=\frac{\hat{\mu}(\sigma^2/n)^{-1}+\mu_a(\sigma_a^2)^{-1}}{(\sigma^2/n)^{-1}+(\sigma^2_a)^{-1}}$$
and variance
$$\mathbb{D}[\mu]=\frac{\sigma^2_a\sigma^2/n}{\sigma^2_a+\sigma^2/n}=\frac{1}{(\sigma^2/n)^{-1}+(\sigma^2_a)^{-1}}.$$
\end{frame}

\begin{frame}[fragile]
\frametitle{How does it work?}
\framesubtitle{Revisiting the $ \mu $ estimation example in a Bayesian framework}
The preceding result can be clarified if we introduce the notation $h_0=(\sigma^2/n)^{-1}$ and $h_a=(\sigma^2_a)^{-1}$. These are called \emph{precision parameters}. \bigskip

Then, we can equivalently write 
\[ \mathbb{E}[\mu]=\dfrac{h_0\hat{\mu}+h_a\mu_a}{h_0+h_a}, \]  \[ \mathbb{D}[\mu]=\frac{1}{h_0+h_a}. \]\bigskip

In other words, the posterior mean turns out to be a weighted average of the sample mean and the prior mean.
\end{frame}
\end{section}


\begin{section}{Extending the Bayesian approach}\label{sec:BayesExt}

\begin{frame}[fragile]
\frametitle{Bayesian updating with new data}
\begin{itemize}\itemsep1em
\item We can use Bayes' theorem to update our information sequentially as new data arrive.
\item Let the initial sample be $\mathbf{x_1}$ and the prior density be $f(\mu)$, leading to a posterior $f(\mu|\mathbf{x_1}) \propto
f(\mu)f(\mathbf{x_1}|\mu)$.
\item Suppose we obtain an additional sample $\mathbf{x_2}$.
\item Then the posterior $f(\mu|\mathbf{x_1})$ can be treated as a prior with respect to the new sample and, using Bayes' theorem, we get
\begin{equation}
\label{eq:Bayesupdate1} f(\mu|\mathbf{x_1},\mathbf{x_2}) \propto
f(\mu|\mathbf{x_1})f(\mathbf{x_2}|\mu),
\end{equation} 
where $f(\mu|\mathbf{x_1},\mathbf{x_2})$ is the posterior obtained with the two samples merged.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian updating with new data}
\begin{itemize}\itemsep1em
\item Formula \eqref{eq:Bayesupdate1} can also be written as 
\begin{equation}
\label{eq:Bayesupdate2} f(\mu|\mathbf{x_1},\mathbf{x_2}) \propto
f(\mu)f(\mathbf{x_1}|\mu)f(\mathbf{x_2}|\mu).
\end{equation}
\item Since the likelihood function for the merged samples $\mathbf{x_1}$ and $\mathbf{x_2}$ is $f(\mathbf{x_1}|\mu)f(\mathbf{x_2}|\mu)$, the new posterior will be the same regardless of whether we obtain the samples sequentially or we have the full sample $(\mathbf{x_1},\mathbf{x_2})$ from the start.
\item Clearly, this approach generalizes to the case of more than one sample.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Marginal and conditional posterior densities}
\begin{itemize}\itemsep1em
\item As noted, the Bayesian approach works the same way when we are interested in a vector of parameters $\boldsymbol{\theta}$. In this context we denote the joint posterior density by $f(\boldsymbol{\theta}|\mathbf{x})$.
\item In some cases we are interested only in a subset of the parameters $\boldsymbol{\theta}$, i.e. given $\boldsymbol{\theta}=\left(\boldsymbol{\theta_1},\boldsymbol{\theta_2}\right)'$, we would like to separate out the posterior information for $\boldsymbol{\theta_1}$ only.
\item In other words, we are interested in the marginal posterior density of the vector $\boldsymbol{\theta_1}$.
\item It can be obtained as \begin{equation}
\label{eq:MargPost}f(\boldsymbol{\theta_1}|\mathbf{x}) = \int_{R_{\boldsymbol{\theta_2}}}f(\boldsymbol{\theta_1},\boldsymbol{\theta_2}|\mathbf{x})\,d\boldsymbol{\theta_2} = \int_{R_{\boldsymbol{\theta_2}}}f(\boldsymbol{\theta_1}|\boldsymbol{\theta_2},\mathbf{x})f(\boldsymbol{\theta_2|\mathbf{x}})\,d\boldsymbol{\theta_2},
\end{equation}
where $R_{\boldsymbol{\theta_2}}$ is the domain of $\boldsymbol{\theta_2}$ and $f(\boldsymbol{\theta_1}|\boldsymbol{\theta_2},\mathbf{x})$
is the conditional posterior density of $\boldsymbol{\theta_1}$ for given
$\boldsymbol{\theta_2}$ and $\mathbf{x}$.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Marginal and conditional posterior densities}
\begin{itemize}\itemsep1em
\item The expression following the second equality sign in \eqref{eq:MargPost}, \[ \int_{R_{\boldsymbol{\theta_2}}}f(\boldsymbol{\theta_1}|\boldsymbol{\theta_2},\mathbf{x})f(\boldsymbol{\theta_2|\mathbf{x}})\,d\boldsymbol{\theta_2}, \] shows that the marginal posterior density $f(\boldsymbol{\theta_1}|\mathbf{x})$ can be interpreted as the result of averaging the conditional posterior density $f(\boldsymbol{\theta_1}|\boldsymbol{\theta_2},\mathbf{x})$ by using the marginal posterior density $f(\boldsymbol{\theta_2}|\mathbf{x})$ as the weight function.
\item The integration operation in \eqref{eq:MargPost} serves to eliminate the information on the parameters we are \textbf{not} interested in, leaving only the posterior information on the relevant parameters.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point estimates in a Bayesian framework}
\begin{itemize}\itemsep1em
\item The posterior summarises all the available information (sample and non-sample) about the parameters of interest.
\item The downside is that the posterior is ultimately a probability distribution, while we may need simpler characterizations of the parameters.
\item One example is a situation where we want to produce specific numerical values for the unknown parameters, i.e. we want \emph{point estimates}.
\item Once we have the posterior for the parameter vector $\boldsymbol{\theta}$, the Bayesian way of using it to produce a point estimate $\boldsymbol{\hat{\theta}}=\boldsymbol{\hat{\theta}}(\mathbf{x})$ is in a decision-theoretic framework.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point estimates in a Bayesian framework}
\begin{itemize}\itemsep1em
\item Decision theory requires us to have \emph{loss function} $L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})$ that measures how ``harmful'' deviations of the estimates $\boldsymbol{\hat{\theta}}$ from the true value $\boldsymbol{\theta}$ are. 
\item Since in our case $\boldsymbol{\theta}$ is a random variable, the loss function $L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})$ will also be random, even though the sample $\mathbf{x}$ is fixed.
\item Thus, we need to work with the expected loss function.
\item Since the posterior summarizes the information on $\boldsymbol{\theta}$, it is natural to construct the expectation with respect to $ f(\boldsymbol{\theta}|\mathbf{x}) $.
\item We are then in a position to compute the expected loss associated with an estimate $\boldsymbol{\hat{\theta}}$.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point estimates in a Bayesian framework}
\begin{itemize}\itemsep1em
\item Then, the optimal point estimate would be the one that minimizes the expectation of the loss function: \begin{equation}
\label{ExpLoss}
\boldsymbol{\hat{\theta^*}}=\min_{\boldsymbol{\hat{\theta}}}\mathbb{E}[L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})]=\min_{\boldsymbol{\hat{\theta}}}\int_{R_{\boldsymbol{\theta}}}
L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})f(\boldsymbol{\theta}|\mathbf{x})\,
d\boldsymbol{\theta}.
\end{equation} 
\item The expected loss function is called \emph{risk} or \emph{risk function}.
\item We are implicitly assuming that both the expectation and the minimum exist.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point estimates in a Bayesian framework}
\begin{itemize}\itemsep1em
\item As an example, take the quadratic loss function $L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})=(\boldsymbol{\theta}-\boldsymbol{\hat{\theta}})'C(\boldsymbol{\theta}-\boldsymbol{\hat{\theta}})$,
where $C$ is a fixed symmetric positive definite matrix.
\item Then the posterior expectation of $L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})$ is 
\begin{equation}
\label{eq:QuadExpLoss}\begin{split}
\mathbb{E}[L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})]=&\mathbb{E}[(\boldsymbol{\theta}-\boldsymbol{\hat{\theta}})'C(\boldsymbol{\theta}-\boldsymbol{\hat{\theta}})]\\=&\mathbb{E}[((\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}])-(\boldsymbol{\hat{\theta}}-\mathbb{E}[\boldsymbol{\theta}]))'C((\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}])-(\boldsymbol{\hat{\theta}}-\mathbb{E}[\boldsymbol{\theta}]))]\\=&\mathbb{E}[(\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}])'C(\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}])]+(\boldsymbol{\hat{\theta}}-\mathbb{E}[\boldsymbol{\theta}])'C(\boldsymbol{\hat{\theta}}-\mathbb{E}[\boldsymbol{\theta}]),
\end{split}
\end{equation}
where the second term in the last equality is not stochastic and can be taken out of the expectation. (Note that in going from the second to the third line terms of the form $ \mathbb{E}[(\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}])'C (\boldsymbol{\hat{\theta}}-\mathbb{E}[\boldsymbol{\theta}])] $ will disappear since $ \mathbb{E}[\boldsymbol{\theta}-\mathbb{E}[\boldsymbol{\theta}]] = \mathbf{0}  $.)
\item At the same time, it is precisely the last term of  \eqref{eq:QuadExpLoss} that depends on $\boldsymbol{\hat{\theta}}$ and determines the minimum.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point estimates in a Bayesian framework}
\begin{itemize}\itemsep1em
\item Clearly for a positive definite $ C $ the minimum is attained at $\boldsymbol{\hat{\theta^*}}=\mathbb{E}[\boldsymbol{\theta}]$.
\item Thus, for the quadratic loss function the optimal point estimate is given by the mean\footnote{Assuming it exists in the first place.} of the respective posterior distribution.\bigskip \pause
\item As further examples, for a univariate parameter $ \theta $ and an absolute deviation loss function $L(\theta,\hat{\theta})=|\theta-\hat{\theta}|$ (plus technical assumptions), the optimal point estimate can be shown to be the median.
\item A zero-one loss function of the form \begin{equation*} L(\boldsymbol{\theta},\boldsymbol{\hat{\theta}})=\left\{ \begin{aligned} &0,~\boldsymbol{\hat{\theta}}=\boldsymbol{\theta}\\ &1,~\boldsymbol{\hat{\theta}}\neq\boldsymbol{\theta} \end{aligned}\right.
,\end{equation*} will yield a mode of the posterior as the optimal point estimate (again under technical assumptions).
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian credible regions}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item The Bayesian counterpart of a confidence interval is called a \emph{credible region}.
\item If we have the posterior density $f(\boldsymbol{\theta}|\mathbf{x})$, we can compute the probability that the vector of parameters $\boldsymbol{\theta}$ belongs to a given subset $\bar{R}$ of the space of parameters:
\begin{equation}
\label{eq:CredReg} P(\boldsymbol{\theta}\in
\bar{R}|\mathbf{x}) = \int_{\bar{R}}f(\boldsymbol{\theta} | \mathbf{x}) \, d\boldsymbol{\theta}.
\end{equation}\pause
\item We can also approach the problem from the opposite direction: for a fixed probability $P(\boldsymbol{\theta}\in \bar{R}|\mathbf{x})$, find a region $\bar{R}$ such that \eqref{eq:CredReg} holds.
\item In general such a region will not be unique.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian credible regions}
\begin{itemize}\itemsep1em
\item If the posterior is unimodal, in some cases a unique credible region can be obtained by imposing an additional requirement known as \emph{highest posterior density}.
\item This requirement is intuitively described along the lines ``the posterior density values over that region should not be smaller than those for any other region with the same probability''...
\item ...but a more precise characterization would be related to the fact that they minimize the volume enclosed in the parameter space among all regions with the same probability.
\item For example, a unimodal symmetric density in the case of one parameter would produce a credible region that is an interval centred on the mode and containing the largest values of the posterior density (or, equivalently, being the smallest in length).
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian credible regions}
\begin{itemize}\itemsep1em
\item It should be remembered that, despite the superficial similarity, Bayesian credible regions are conceptually different from classical confidence intervals.
\item A classical confidence interval is considered random and probabilistic statements relate to the probability that this random interval will cover the fixed (but unknown) value of the parameter of interest.
\item In contrast, a Bayesian credible region is considered deterministic and probabilistic statements relate to the probability with which the random parameter of interest will fall in that pre-specified region.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Marginal distribution of the observations}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item Sometimes we are interested in the marginal density of the observations $f(\mathbf{x})$.
\item These can be derived as follows:
\begin{equation}
\label{eq:MargObs}f(\mathbf{x})=\int_{R_{\boldsymbol{\theta}}}f(\boldsymbol{\theta},\mathbf{x})\,d\boldsymbol{\theta}=\int_{R_{\boldsymbol{\theta}}}f(\mathbf{x}|\boldsymbol{\theta})f(\boldsymbol{\theta})\,d\boldsymbol{\theta}.
\end{equation}
\item The expression after the second equality sign in \eqref{eq:MargObs} means that the marginal density can be viewed as averaging the likelihood function by using the prior density as a weighting function.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Predictive probability density functions}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item One important issue arising in a forecasting context is that of the distribution of still unobserved events $\mathbf{\tilde{x}}$.
\item To answer that, we consider the joint density of the unobserved data $\mathbf{\tilde{x}}$ and the parameters $\boldsymbol{\theta}$, given already observed data $\mathbf{x}$:
\begin{equation}
\label{eq:JntData}
f(\mathbf{\tilde{x}},\boldsymbol{\theta}|\mathbf{x})=f(\mathbf{\tilde{x}}|\boldsymbol{\theta},\mathbf{x})f(\boldsymbol{\theta}|\mathbf{x}).
\end{equation}
\item The predictive density of $\mathbf{\tilde{x}}$ conditional on $\mathbf{x}$ can be obtained after integrating \eqref{eq:JntData} w.r.t. $\boldsymbol{\theta}$:
\begin{equation}
\label{eq:MargForec}
f(\mathbf{\tilde{x}}|\mathbf{x})=\int_{R_{\boldsymbol{\theta}}}f(\mathbf{\tilde{x}},\boldsymbol{\theta}|\mathbf{x})\,d\boldsymbol{\theta}=\int_{R_{\boldsymbol{\theta}}}f(\mathbf{\tilde{x}}|\boldsymbol{\theta},\mathbf{x})f(\boldsymbol{\theta}|\mathbf{x})\,d\boldsymbol{\theta}.
\end{equation}
\item The last expression in \eqref{eq:MargForec} means that the predictive density can be interpreted as averaging the conditional predictive density with the posterior used as a weighting function.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Point prediction}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item The predictive density $f(\mathbf{\tilde{x}}|\mathbf{x})$ can be used to produce point forecasts just like the posterior is used to produce point estimates.
\item The principle is the same as for point estimates of parameters.
\item To produce a point prediction $\mathbf{\hat{x}}$ for the yet unobserved data $\mathbf{\tilde{x}}$, we need a loss function $L(\mathbf{\tilde{x}},\mathbf{\hat{x}})$.
\item We then construct the risk function and minimize it w.r.t. $\mathbf{\hat{x}}$:
\begin{equation}
\label{eq:PointForec}
\min_{\mathbf{\hat{x}}}\int_{R_{\mathbf{\tilde{x}}}}L(\mathbf{\tilde{x}},\mathbf{\hat{x}})f(\mathbf{\tilde{x}}|\mathbf{x})d\mathbf{\tilde{x}},
\end{equation}
where $R_{\mathbf{\tilde{x}}}$ is the set of possible values of $\mathbf{\tilde{x}}$.
\item The solution to \eqref{eq:PointForec}, if it exists, is the optimal point prediction for $\mathbf{\tilde{x}}$.
\item We can establish counterparts to point estimate results on the mean, median etc. corresponding to specific loss functions.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Prediction regions}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item Given a predictive density $f(\mathbf{\tilde{x}}|\mathbf{x})$, we can compute the probability that future observations $\mathbf{\tilde{x}}$ belong to a given subset $\bar{R}$ of their domain:
\begin{equation}
\label{eq:CredRegForec}
P(\mathbf{\tilde{x}}\in \bar{R}|\mathbf{x}) = \int_{\bar{R}}f(\mathbf{\tilde{x}}|\mathbf{x})\,d\mathbf{\tilde{x}}.
\end{equation}
\item Similarly to credible regions, we can also study the inverse problem, where we seek a region $\bar{R}$ satisfying \eqref{eq:CredRegForec} -- a \emph{prediction region} -- for a given probability $P(\mathbf{\tilde{x}}\in \bar{R}|\mathbf{x})$.
\item Uniqueness generally cannot be guaranteed in this case but we can develop the concept of a ``highest predictive density'' region along the lines of highest posterior density regions.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian hypothesis testing}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item In a Bayesian context, hypothesis testing is a comparison of equally standing alternatives. This stands in contrast to the classical treatment, where the null hypothesis is the object of special interest.
\item In general, we study statements $H_0$ and $H_1$ relating to different models $M_0$ and $M_1$, given data $\mathbf{x}$. 
\item For a model $M_i,~i=0,1,$ the respective likelihood function is $f_i(\mathbf{x}|\boldsymbol{\theta_i})$, while the prior is $f_i(\boldsymbol{\theta_i})$.
\item The subscripts $i$ in the likelihoods and the priors indicate that it is possible to compare hypotheses for different data generating mechanisms and different priors.
\item It is still possible to analyse simpler cases, e.g. when the parameters $\boldsymbol{\theta_0}$ and $\boldsymbol{\theta_1}$, corresponding to hypotheses $H_0$ and $H_1$, take values in different domains of the same parametric space. 
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian hypothesis testing}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item If the true model is model $M_i$, the marginal density of the data (\emph{marginal likelihood function}) is 
\begin{equation}
\label{eq:MargLik}
f(\mathbf{x}|M_i)=\int_{R_{\boldsymbol{\theta_i}}}f_i(\mathbf{x}|\boldsymbol{\theta_i})f_i(\boldsymbol{\theta_i})\,d\boldsymbol{\theta_i}.
\end{equation}
\item Then, a \emph{Bayes factor} is defined as the ratio of the marginal likelihood functions for the two models:
\begin{equation}
\label{eq:BayesFactor}B_{01}(\mathbf{x}):=\frac{f(\mathbf{x}|M_0)}{f(\mathbf{x}|M_1)}.
\end{equation}
\item A Bayes factor greater than 1 can be interpreted as evidence that $M_0$ (or, respectively, hypothesis $H_0$) is more plausible compared to model $M_1$ (hypothesis $H_1$).
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian hypothesis testing}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item It is also possible to have prior probabilities on the models themselves and take these probabilities into account when testing hypotheses. 
\item If $P(M_i)$ is the prior probability of model $M_i$, then we can calculate the posterior probability of this model given the data $\mathbf{x}$ using Bayes' formula:
\begin{equation}
\label{eq:ModelPost}P(M_i|\mathbf{x})=\frac{P(M_i)f(\mathbf{x}|M_i)}{\sum_{j=0}^{1}P(M_j)f(\mathbf{x}|M_j)}.
\end{equation}
\item The \emph{posterior odds} of model $M_0$ compared to model $M_1$ is defined as the ratio of the posterior probabilities of the two models:
\begin{equation}
\label{eq:PostOdds}K_{01}(\mathbf{x}):=\frac{P(M_0|\mathbf{x})}{P(M_1|\mathbf{x})}=\frac{P(M_0)}{P(M_1)}\times
\frac{f(\mathbf{x}|M_0)}{f(\mathbf{x}|M_1)}=\frac{P(M_0)}{P(M_1)}B_{01}(\mathbf{x}).
\end{equation}
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian hypothesis testing}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item Equation \eqref{eq:PostOdds} illustrates why the quantity $B_{01}(\mathbf{x})$ is called a Bayes factor: it is the quantity that multiplies the \emph{prior odds} $P(M_0)/P(M_1)$ to adjust it with information coming from the data and thus obtain the posterior odds. In other words, the Bayes factor tells us whether the data increase or decrease the plausibility of one model compared to another.
\item The above considerations were framed in terms of model comparison but they can equally be applied to hypothesis testing.
\item Thus, if we have the posterior odds $K_{01}(\mathbf{x})$ needed to compare hypothesis $H_0$ to $H_1$, we can use it to decide which hypothesis is more plausible.
\item In the absence of prior information in favour of one hypothesis, the hypotheses can be viewed as equally likely and the prior odds can be taken to be 1. Then the posterior odds is equal to the Bayes factor and $B_{01}(\mathbf{x})$ suffices to choose between the hypotheses tested.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Bayesian hypothesis testing}
\framesubtitle{}
\begin{itemize}\itemsep1em
\item If we focus on accepting or rejecting hypothesis $H_0$ in favour of $H_1$, then using only the posterior odds -- for instance, by applying rules such as ``Accept $H_0$ if $K_{01}(\mathbf{x})\geq 1$, else reject $H_0$.'' -- is a relatively blunt decision-making tool.
\item In such cases, it is recommended to work in a decision-theoretic framework by introducing an explicit loss function $L(H_i,\hat{H}_j)$, where $H_i$ is the true situation and $\hat{H}_j$ is the hypothesis accepted by the researcher.
\item  We can then compute the risk associated with accepting  $\hat{H}_j$ and choose the risk-minimizing hypothesis $\hat{H}_j$.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Additional information on Bayesian methods}
\framesubtitle{Informative and non-informative priors}
\begin{itemize}\itemsep1em
\item Up to this point we implicitly assumed that the priors were chosen to reflect additional available information. That is, they were taken to be \emph{informative}.
\item We can, however, imagine situations in which we lack non-experimental information, requiring a \emph{non-informative} prior.
\item It is intuitive to treat the parameter values as equally probable in such cases.
\item This approach seems natural but gives rise to technical difficulties.
\item For example, attempting to use the uniform distribution as a prior for a continuous parameter either implicitly introduces information (if we define it over a finite interval, thus limiting the support of the distribution), or else leads to a divergent integral.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Additional information on Bayesian methods}
\framesubtitle{Informative and non-informative priors}
\begin{itemize}\itemsep1em
\item If a non-informative ``density'' (under the above definition) leads to a divergent integral, it obviously cannot be a proper probability density function.
\item Nevertheless, we can agree to consider such priors if the resulting posteriors are proper, i.e. if they integrate to a finite quantity.
\item Such situations can be construed as taking only sample information into account.
\item In many cases this leads to results that coincide with the classical results for large enough samples. 
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Additional information on Bayesian methods}
\framesubtitle{Simulation techniques: Markov chain Monte Carlo}
\begin{itemize}\itemsep1em
\item In a general formulation like ours, the posterior densities can take a huge variety of forms.
\item In most cases closed-form solutions cannot be obtained and we need to resort to numerical methods.
\item If the parameter space is high-dimensional, generating a random sample from the posterior or integrating it leads to serious computational difficulties.
\item In the past, this problem was circumvented by using the so-called \emph{conjugate priors}, i.e. priors which lead to the posterior belonging to the same class of distributions. In addition, conjugate priors that were easy to study were favoured.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Additional information on Bayesian methods}
\framesubtitle{Simulation techniques: Markov chain Monte Carlo}
\begin{itemize}\itemsep1em
\item With the growth of computing power over the past few decades, Bayesian methods have progressively gravitated towards a powerful class of numerical simulation methods known as Markov chain Monte Carlo (MCMC).
\item The main idea of MCMC is to draw a large enough sample from a  Markov chain that asymptotically has the same distribution as the posterior. 
\item The properties of that sample are then studied to learn about the posterior.
\item Most contemporary empirical studies in a Bayesian framework employ some form of MCMC or similar methods.
\end{itemize}
\end{frame}
\end{section}


\begin{section}{The linear regression model in a Bayesian framework}\label{sec:BayesLM}

\begin{frame}[fragile]
\frametitle{Formulation}
\framesubtitle{}
We study the familiar regression model \begin{equation}
\label{eq:LinRegMod}
\mathbf{y}=X\boldsymbol{\beta}+\mathbf{e},
\end{equation}
where there are $T$ observations and $K$ parameters. We take the matrix $X$ to be non-stochastic (the same approach works for a stochastic $ X $ if it is independent of the error $\mathbf{e}$), and assume that $\rank(X)=K$ and
$\mathbf{e}\in N(\mathbf{0},\sigma^2 I_T)$.\bigskip \bigskip

The likelihood function for \eqref{eq:LinRegMod} under our normality assumption is \begin{equation}
\label{eq:LRMLik} \ell
(\mathbf{y}|\boldsymbol{\beta}, \sigma^2)=(2\pi \sigma^2)^{-T/2}\exp
\left[
-\frac{(\mathbf{y}-X\boldsymbol{\beta})'(\mathbf{y}-X\boldsymbol{\beta})}{2\sigma^2}
\right].
\end{equation} 
\end{frame}

\begin{frame}[fragile]
\frametitle{Formulation}
\framesubtitle{}
In accordance with Bayesian principles we take $\boldsymbol{\beta}$ and $\sigma^2$ as random and having a prior density $ f(\boldsymbol{\beta},\sigma) $ with the kernel given below: \begin{equation}
\label{eq:LRMPrior}f(\boldsymbol{\beta},\sigma)\propto\sigma^{-m}\exp\left[
-\frac{1}{2\sigma^2}[\eta+(\boldsymbol{\beta}-\boldsymbol{\mu})'\boldsymbol{\Psi}^{-1}(\boldsymbol{\beta}-\boldsymbol{\mu})]
\right],
\end{equation}
where $m>K+1$, $\eta>0$ and $\boldsymbol{\Psi}$ is symmetric and positive definite.\bigskip \bigskip

This is an example of a conjugate prior.
\end{frame}

\begin{frame}[fragile]
\frametitle{Interpretation of the prior}
\framesubtitle{}
The choice of \eqref{eq:LRMPrior} as our prior can be understood if we consider the decomposition of $f(\boldsymbol{\beta},\sigma)$ as \begin{equation}
\label{eq:DecompLMRPrior}
f(\boldsymbol{\beta},\sigma)=f(\boldsymbol{\beta}|\sigma)f(\sigma),
\end{equation} 
where
\begin{equation}
\label{eq:CondBeta}f(\boldsymbol{\beta}|\sigma)=\frac{1}{(2\pi)^{K/2}\sigma^K
[\det (\boldsymbol{\Psi})]^{1/2}}\exp \left[
-\frac{1}{2\sigma^2}(\boldsymbol{\beta}-\boldsymbol{\mu})'\boldsymbol{\Psi}^{-1}(\boldsymbol{\beta}-\boldsymbol{\mu})\right]
\end{equation} and 
\begin{equation}
\label{eq:SigmaPrior}f(\sigma)\propto \sigma^{-(m-K)}\exp
\left[ -\frac{\eta}{2\sigma^2} \right],\quad \sigma>0.
\end{equation}\bigskip

In other words, conditional on $\sigma$, the vector $\boldsymbol{\beta}$ is multivariate normal with mean  $\boldsymbol{\mu}$ and covariance matrix $\sigma^2\boldsymbol{\Psi}$. The distribution of $\sigma$ that corresponds to the kernel \eqref{eq:SigmaPrior} can be obtained as a nonlinear transformation of a gamma-distributed random variable.
\end{frame}

\begin{frame}[fragile]
\frametitle{The posterior}
\framesubtitle{}
Given the prior \eqref{eq:LRMPrior} and the likelihood \eqref{eq:LRMLik} the posterior of the model can be shown to be \begin{equation}
\label{eq:LRMPost}\begin{split}f(\boldsymbol{\beta},\sigma|\mathbf{y})\propto
& \frac{1}{\sigma^{T+m}}\exp\left[
-\frac{1}{2\sigma^2}\left[(T-K)\hat{\sigma}^2+(\boldsymbol{\beta}-\mathbf{b})'X'X(\boldsymbol{\beta}-\mathbf{b})\right]
\right]\\& \times \exp \left[
-\frac{1}{2\sigma^2}\left[\eta+(\boldsymbol{\beta}-\boldsymbol{\mu})'\boldsymbol{\Psi}^{-1}(\boldsymbol{\beta}-\boldsymbol{\mu})\right]
\right], \end{split}
\end{equation}
where $\hat{\sigma}^2=RSS/(T-K)$ and
$\mathbf{b}$ are the OLS estimates for the variance and the coefficient vector.

The kernel \eqref{eq:LRMPost} can be written equivalently as
\begin{equation}
\label{eq:LRMPost1}f(\boldsymbol{\beta},\sigma|\mathbf{y})\propto\frac{1}{\sigma^{T+m}}\exp \left[ -\frac{1}{2\sigma^2}[(\boldsymbol{\beta}-\boldsymbol{\beta_*})' (\boldsymbol{\Psi}^{-1}+X'X)(\boldsymbol{\beta}-\boldsymbol{\beta_*})+\xi]\right],
\end{equation}
where \begin{align*}
&\boldsymbol{\beta_*} = (\boldsymbol{\Psi}^{-1}+X'X)^{-1}(\boldsymbol{\Psi}^{-1} \boldsymbol{\mu}+X'X\mathbf{b}),\\
&\xi = \eta+(T-K)\hat{\sigma}^2+\boldsymbol{\mu}'\boldsymbol{\Psi}^{-1} \boldsymbol{\mu}+\mathbf{b}'X'X\mathbf{b}-\boldsymbol{\beta_*}'(\boldsymbol{\Psi}^{-1}+X'X)\boldsymbol{\beta_*}.
\end{align*}
\end{frame}

\begin{frame}[fragile]
\frametitle{The posterior}
\framesubtitle{}
The marginal posterior density for $\boldsymbol{\beta}$ can be obtained in a standard manner by integrating \eqref{eq:LRMPost1} w.r.t.
$\sigma$. It has the kernel \begin{equation}
\label{eq:MargPostBeta}\begin{split} f(\boldsymbol{\beta}|\mathbf{y})\propto
& \left[ \xi +
(\boldsymbol{\beta}-\boldsymbol{\beta_*})'(\boldsymbol{\Psi}^{-1}+X'X)(\boldsymbol{\beta}-\boldsymbol{\beta_*})
\right]^{-(T+m-1)/2}\\ \propto & \left[
v+(\boldsymbol{\beta}-\boldsymbol{\beta_*})'\frac{v}{\xi}(\boldsymbol{\Psi}^{-1}+X'X)(\boldsymbol{\beta}-\boldsymbol{\beta_*})\right]^{-(v+K)/2},
\end{split}
\end{equation} 
where $v=T+m-K-1$. \bigskip \bigskip

This is the kernel of a multivariate $t$-distribution. 
\end{frame}

\begin{frame}[fragile]
\frametitle{The posterior}
\framesubtitle{}
A random vector distributed with kernel \eqref{eq:MargPostBeta} will have mean $\boldsymbol{\beta_*}$ for $T+m-K>2$ and covariance matrix $$\left( \frac{\xi}{T+m-K-3}
\right)(\boldsymbol{\Psi}^{-1}+X'X)^{-1} \textrm{ за } T+m-K>3.$$ \bigskip \bigskip

If we denote $\mathbf{h}=\left[ \frac{v}{\xi}(\boldsymbol{\Psi}^{-1}+X'X)
\right]$, then the distribution of the $j$-th element of $\boldsymbol{\beta}$,
$\beta_j$, can be obtained by taking into account that 
$(\beta_j-\beta_{*j})/[h^{-1}(j,j)]^{1/2}$ has univariate $t$-distribution with $v$ degrees of freedom. The notation in the denominator stands for the square root of the $j$-th diagonal element of the matrix $\mathbf{h}^{-1}$.
\end{frame}

\begin{frame}[fragile]
\frametitle{The posterior}
\framesubtitle{}
It can be shown that the multivariate $t$-distribution forms a class that is closed with respect to linear transformations (up to a change in the parameters of the distribution). 

We can also use the fact that \begin{equation*}
%\label{eq:QuadBeta}
\frac{T+m-K-1}{K\xi}(\mathbf{B}-\boldsymbol{\beta_*})'(\boldsymbol{\Psi}^{-1}+X'X)(\mathbf{B}-\boldsymbol{\beta_*})~\sim~F(K,v),
\end{equation*} 
where $\mathbf{B}$ denotes a random vector with density kernel \eqref{eq:MargPostBeta}. 

Similarly, if $C$ is a $J\times K$ matrix, then \begin{equation*}
%\label{eq:TransfQuadBeta}
\frac{T+m-K-1}{J\xi}[C(\mathbf{B}-\boldsymbol{\beta_*})]'[C(\boldsymbol{\Psi}^{-1}+X'X)^{-1}C']^{-1}[C(\mathbf{B}-\boldsymbol{\beta_*})]~\sim~ F(J,v).
\end{equation*}

This can be used to study the properties of $\boldsymbol{\beta}$, including to construct Bayesian credible regions and test hypotheses.
\end{frame}

\end{section}


\begin{frame}[fragile]
\frametitle{Readings}
Readings:\bigskip

William H. Greene. \emph{Econometric Analysis}, $ 7^{\text{th}} $ edition. Chapter 16.\bigskip \bigskip

Additional readings:\bigskip

Gary Koop. \emph{Bayesian Econometrics}. 2003. Wiley.
\bigskip

Allen Downey. \emph{Think Bayes: Bayesian Statistics in Python}.
\end{frame}
\end{document}