Appendix3.tex

\chapter{Derivation of Standard Errors for Multilayer Mixture Model}
In a multilayer mixture model, $K$ is the number of clusters and $J_{k}$ is the number of components in the $k$th cluster ($k=1,...,K$) such that $J=\sum_{k=1}^{K}J_{k}$ is the total number of components in the entire model. Let $j$ uniquely index all of the components such that $j=1,...,J$. For ease of explanation, let $c(j):\{1,...,J\}\rightarrow \{1,2,...,K\}$ be a cluster assigning function that specifies the cluster to which a component belongs. Let $f(\B y|\BS\mu,\BS\Sigma)$ be the probability density function of a multivariate normal distribution with mean $\BS\mu$ and covariance matrix $\BS\Sigma$. Then, the probability density function for cluster $k$ is
$$f_{k}(\B y) = \sum_{j: c(j) = k} \pi_{j|c(j)}f(\B y|\mu_{j},\Sigma_{j})$$
where $\pi_{j|c(j)}$ is the probability of being in component $j$ given a subject is in cluster $c(j)$ and $\sum_{j: c(j) = k}\pi_{j|c(j)}=1$ for all $k=1,2,...,K$. Let the probability of cluster $k$ given baseline factors, $\B w$, be $\pi_{k}(\B w,\BS\gamma)=\exp(\B w^{T}\BS\gamma_{k})/\sum^{K}_{l=1}\exp(\B z^{T}\BS\gamma_{l})$ such that $\BS\gamma_{K}=0$ and $\BS\gamma=(\BS\gamma_{1},...,\BS\gamma_{K})$. The probability density function for the multilayer mixture is written as
$$g(\B y|\B w) = \sum_{k=1}^{K}\pi_{k}(\B w,\BS \gamma)f_{k}(\B y) = \sum_{k=1}^{K}\pi_{k}(\B w,\BS\gamma)\sum_{j: c(j) = k} \pi_{j|c(j)}f( \B y| \BS\mu_{j},\BS\Sigma_{j}).$$
Since every component belongs to one and only one cluster, the above equation reduces to a regular mixture model if $\bar{\pi}_{j}(\B w,\BS \gamma)=\pi_{c(j)}(\B w,\BS\gamma)\pi_{j|c(j)}$ with the density written as
$$g(\B y|\B w) = \sum_{j=1}^{J}\bar{\pi}_{j}(\B w,\gamma)f(\B y|\BS \mu_{j},\BS\Sigma_{j}).$$
\subsection{Standard Error Calculations}
Let $h(\B w,\BS\gamma)=\sum^{K}_{j=1}\exp(\B w^{T}\BS\gamma_{j})$. Then $\pi_{k}(\B w,\BS\gamma)=\frac{\exp(\B w^{T}\BS\gamma_{k})}{h(\B w,\BS\gamma)}$. I differentiate $\log\pi_{k}(\B w,\BS\gamma)$ with respect to $\BS\gamma_{1},\BS\gamma_{2},...,\BS\gamma_{K-1}$. Taking the logarithm of both sides I get,
$$\log\pi_{k}(\B w,\BS\gamma)=\B w^{T}\BS\gamma_{k}-\log h(\B w,\gamma)$$
Note that $\partial \log h(\B w,\BS\gamma)/\partial \BS\gamma_{j}=\pi_{j}(\B w,\BS\gamma)\B w$. Then
\begin{align*}
\frac{\partial \log\pi_{k}(\B w,\BS\gamma) }{\partial \BS\gamma_{j}} &= I(j=k)\B w-\pi_{k}(\B w,\BS\gamma)\B w\\
\frac{\partial^{2} \log\pi_{k}(\B w,\BS\gamma) }{\partial \BS\gamma_{j}\partial \BS\gamma_{j'}} &= [-I(j=j')\pi_{j}(\B w,\BS\gamma)+\pi_{j}(\B w,\BS\gamma)\pi_{j'}(\B w,\BS\gamma)]\B w\B w^{T}\\
\end{align*}
Define the following:
\begin{align*}
\B b_{ij} = \frac{1}{\sigma_{j}^{2}}(\B y_{i}-\alpha_{j}\B 1_{m_i}-\B x_{i}\BS\beta_{c(j)}),&\quad \B B_{ij}=\frac{m_{i}}{\sigma^{2}_{j}}-\B b_{ij}^{T}\B b_{ij}\\
\B c_{ij} = \left(\begin{array}{c}
\B 1_{m_i}^{T} b_{ij}\\
-\frac{1}{2}B_{ij}
\end{array}\right),&\quad C_{ij} =\left(\begin{array}{cc}
\frac{m_{i}}{\sigma^{2}_{j}}&\frac{\B 1_{m_i}^{T}b_{ij}}{\sigma^{2}_{j}}\\
\frac{b^{T}_{ij}\B 1_{m_i}}{\sigma^{2}_{j}}&\frac{1}{2\sigma_{j}^{2}}(-2B_{ij}+\frac{m_{i}}{\sigma^{2}_{j}})
\end{array}\right)\\
d_{ij}=\left(\begin{array}{c}
\frac{\B 1_{m_i}^{T}x_{i}}{\sigma^{2}_{j}}\\
\frac{b_{ij}^{T}x_{i}}{\sigma^{2}_{j}}\end{array}\right),&
\quad \theta_{j}=\left(\begin{array}{c}
\alpha_{j}\\
\sigma_{j}^{2}\end{array}\right)
\end{align*}
Let $\phi_{ij} = \bar{\pi}_{j}(z_{i},\gamma)f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})$ and $p_{ij} = \frac{\phi_{ij}}{\sum_{j} \phi_{ij}}$. Then since $g(y_{i}|x_{i},z_{i})=\sum_{j=1}^{J}\phi_{ij}$, we obtain
\begin{align}\label{dg}
\partial \log g(y_{i}|x_{i},z_{i})=\frac{\partial g(y_{i}|x_{i},z_{i})}{g(y_{i}|x_{i},z_{i})}=\sum^{J}_{j=1}\frac{\partial \phi_{ij} }{\sum_{k=1}^{J} \phi_{ik}}=\sum^{J}_{j=1}p_{ij}\partial \log \phi_{ij}
\end{align}
and
\begin{align}\label{d2g}
\partial^{2} \log g(y_{i}|x_{i},z_{i})&=\left(\frac{\partial^{2} g(y_{i}|x_{i},z_{i})}{g(y_{i}|x_{i},z_{i})}-\left(\frac{\partial g(y_{i}|x_{i},z_{i})}{g(y_{i}|x_{i},z_{i})}\right)^{2}\right)\nonumber\\
&=\left(\frac{\sum^{J}_{j=1}\partial^{2}\phi_{ij}}{\sum^{J}_{j=1}\phi_{ij}}-\left(\frac{\sum^{J}_{j=1}\partial\phi_{ij}}{\sum^{J}_{j=1}\phi_{ij}}\right)^{2}\right)\nonumber\\
&=\left( \sum^{J}_{j=1}p_{ij}(\partial^{2}\log \phi_{ij} + (\partial \log \phi_{ij})^{2})-(\sum^{J}_{j=1}p_{ij}\partial \log \phi_{ij})^{2}\right)
\end{align}
To evaluate these, we first need the first and second-order derivatives of $\log\phi_{ij}$. Since,
$$\log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})=-\frac{m_{i}}{2}\log(2\pi)-\frac{m_{i}}{2}\log\sigma_{j}^{2}-\frac{1}{2\sigma_{j}^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})$$
we find 
\begin{align*}
\partial \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j}) &= -\frac{m_{i}}{2}\partial \log\sigma_{j}^{2}+\frac{1}{\sigma_{j}^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}\partial (\alpha_{j}\B 1_{m_i}+x_{i}\beta_{c(j)})\\
&\quad- \frac{1}{2}\partial((\sigma_{j}^{2})^{-1})(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})\\
&= -\frac{m_{i}}{2}\frac{\partial\sigma_{j}^{2}}{\sigma_{j}^{2}}+\frac{1}{\sigma_{j}^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]\\
&\quad+\frac{1}{2}\frac{\partial\sigma_{j}^{2}}{(\sigma_{j}^{2})^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})\\
&=c_{ij}^{T}\partial\theta_{j}+b_{ij}^{T}x_{i}\partial \beta_{c(j)}
\end{align*}
and
\begin{align*}
\partial^{2} \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j}) &= \frac{m_{i}}{2}\frac{\partial^{2}\sigma_{j}^{2}}{(\sigma_{j}^{2})^{2}}-\frac{1}{\sigma_{j}^{2}}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]^{T}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]\\
&\quad-\frac{\partial \sigma_{j}^{2}}{(\sigma^{2}_{j})^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]\\
&\quad-\frac{\partial^{2}\sigma_{j}^{2}}{(\sigma_{j}^{2})^{3}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})\\
&\quad-\frac{\partial\sigma_{j}^{2}}{(\sigma_{j}^{2})^{2}}(y_{i}-\alpha_{j}\B 1_{m_i}-x_{i}\beta_{c(j)})^{T}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]\\
&= \frac{m_{i}}{2}\frac{\partial^{2}\sigma_{j}^{2}}{(\sigma_{j}^{2})^{2}}-\frac{1}{\sigma_{j}^{2}}[\partial \alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]^{T}[\partial\alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]\\
&\quad-2\frac{\partial \sigma_{j}^{2}}{(\sigma^{2}_{j})}b_{ij}^{T}[\partial\alpha_{j}\B 1_{m_i}+\partial(x_{i}\beta_{c(j)})]-\frac{\partial^{2}\sigma_{j}^{2}}{(\sigma_{j}^{2})}b_{ij}^{T}b_{ij}\\
&= -2(\partial \theta_{j}^{T}d_{ij})\partial\beta_{c(j)}+\partial\beta_{c(j)}^{T}x_{i}^{T}x_{i}\partial\beta_{c(j)}-\partial \theta_{j}^{T}C_{ij}\partial \theta_{j}\\
\end{align*}
and hence,
\begin{align}\label{dphi}
\partial \log \phi_{ij} &= \partial \log \bar{\pi}_{j}(z_{i},\gamma)+\partial \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})\nonumber\\
&= \partial \log \pi_{c(j)}(z_{i},\gamma)+\partial \log \pi_{j|c(j)}+\partial \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})\nonumber\\
&= z_{i}^{T}\partial \gamma_{c(j)}-\pi_{.}(z_{i},\gamma)z_{i}^{T}\partial \gamma_{.}+ \partial \eta_{j} - I(c(.)=c(j))\pi_{.|c(j)}(\eta)\partial \eta_{.}+c_{ij}^{T}\partial\theta_{j}+b_{ij}^{T}x_{i}\partial \beta_{c(j)}
\end{align}
and
\begin{align}\label{d2phi}
\partial^{2} \log \phi_{ij} &= \partial^{2} \log \bar{\pi}_{j}(z_{i},\gamma)+\partial^{2} \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})\nonumber\\
&= \partial^{2} \log \pi_{c(j)}(z_{i},\gamma)+\partial^{2} \log \pi_{j|c(j)}+\partial^{2} \log f(y_{i}|x_{i},\alpha_{j},\beta_{c(j)},\sigma^{2}_{j})\nonumber\\
&= -\partial \gamma_{.}^{T}\pi_{.}(z_{i},\gamma)z_{i}z_{i}^{T}\partial \gamma_{.}+\partial \gamma_{.}^{T}\pi_{.}(z_{i},\gamma)\pi_{..}(z_{i},\gamma)z_{i}z_{i}^{T}\partial \gamma_{..}\nonumber\\
&\quad+I(c(.)=c(..)=c(j))(-\partial \eta_{.}^{T}\pi_{.|c(j)}(\eta)\partial \eta_{.}+\partial \eta_{.}^{T}\pi_{.|c(j)}(\eta)\pi_{..|c(j)}(\eta)\partial \eta_{..})\nonumber\\
&\quad-2(\partial \theta_{j}^{T}d_{ij})\partial\beta_{c(j)}+\partial\beta_{c(j)}^{T}x_{i}^{T}x_{i}\partial\beta_{c(j)}-\partial \theta_{j}^{T}C_{ij}\partial \theta_{j}
\end{align}
Then by substituting (\ref{dphi}) into (\ref{dg}) as well as (\ref{dphi}) and (\ref{d2phi}) into (\ref{d2g}), we find
\begin{align*}
q_{i}^{\gamma_{k}} &= \left[\sum_{l: c(l) = k} p_{il} -\pi_{k}(z_{i},\gamma)\right]z_{i}\\
q_{i}^{\beta_{k}} &= \sum_{l: c(l) = k} p_{il} x^{T}_{i}b_{il}\\
q_{i}^{\eta_{j}} &= p_{ij}-\pi_{j|c(j)}\sum_{l:c(l)=c(j)}p_{il}\\
q_{i}^{\theta{j}} &= p_{ij} c_{ij}
\end{align*}
and
\begin{align*}
Q_{i}^{\gamma_{k}\gamma_{k'}} &= \left(I(k=k')\left[\sum_{l: c(l) = k} p_{il}-\pi_{k}(z_{i},\gamma)\right]-[\sum_{l: c(l) =k} p_{il}][\sum_{l: c(l)=k'} p_{il}] +\pi_{k}(z_{i},\gamma)\pi_{k'}(z_{i},\gamma)\right)z_{i}z_{i}^{T}\\
Q_{i}^{\beta_{k}\beta_{k'}} &= I(k=k')\left[\sum_{l: c(l) = k} p_{il}(\frac{-1}{\sigma_{j}^{2}}x_{i}^{T}x_{i}+x_{i}^{T}b_{il}b_{il}^{T}x_{i})\right]-[\sum_{l: c(l) =k}p_{il}x^{T}_{i}b_{il}][\sum_{l: c(l)=k'}p_{il}x^{T}_{i}b_{il}]^{T}\\
Q_{i}^{\eta_{j}\eta_{j'}} &= I(c(j)=c(j'))\left(I(j=j')\left[p_{ij}- \pi_{j|c(j)}\sum_{l:c(l)=c(j)}p_{il}\right]+2\pi_{j|c(j)}\pi_{j'|c(j')}\sum_{l:c(l)=c(j)}p_{il}-p_{ij'}\pi_{j|c(j)}-p_{ij}\pi_{j'|c(j')}\right)\\
&\quad - \left[p_{ij}-\pi_{j|c(j)}\sum_{l:c(l)=c(j)}p_{il}\right]\left[p_{ij'}-\pi_{j'|c(j')}\sum_{l:c(l)=c(j')}p_{il}\right]\\
Q_{i}^{\theta_{j}\theta_{j'}} &= I(j=j')\left[p_{ij}(c_{ij}c_{ij}^{T}-C_{ij})\right]-p_{ij}p_{ij'}c_{ij}c_{il}^{T}\\
\end{align*}
and
\begin{align*}
Q_{i}^{\gamma_{k}\beta_{k'}} &= \sum_{l: c(l) = k'} p_{il}\left[I(k=k')-\pi_{k}(z_{i},\gamma)\right]z_{i}b_{il}^{T}x_{i}+\left[\pi_{k}(z_{i},\gamma)-\sum_{l: c(l) =k}p_{il} \right]z_{i}\left[\sum_{l: c(l) = k'}p_{il}x_{i}^{T}b_{il}\right]^{T}\\
Q_{i}^{\gamma_{k}\eta_{j}} &=\left[p_{ij} - \pi_{j|c(j)}\sum_{l:c(l)=c(j)}p_{il}\right]\left[I(c(j)=k) - \pi_{k}(z_{i},\gamma) \right]z_{i} +   \left[\pi_{k}(z_{i},\gamma)-\sum_{l: c(l) =k}p_{il}\right]z_{i}\left[p_{ij}-\pi_{j|c(j)} \sum_{l:c(l)=c(j)}p_{il}\right]\\
Q_{i}^{\gamma_{k}\theta_{j}} &=  p_{ij}I(c(j)=k)(1-\pi_{k}(z_{i},\gamma))z_{i}c_{ij}^{T}+\left[\pi_{k}(z_{i},\gamma)-\sum_{l: c(l) =k}p_{il}\right]z_{i}p_{ij}c_{ij}^{T} \\
Q_{i}^{\beta_{k}\eta_{j}} &=I(c(j)=k)\left[p_{ij}x^{T}_{i}b_{ij}-\pi_{j|c(j)}(\eta)\sum_{l:c(l)=c(j)}p_{il}x^{T}_{i}b_{il}\right] - \left[\sum_{l: c(l) = k}p_{il}x_{i}^{T}b_{il}\right]\left[p_{ij}-\pi_{j|c(j)} \sum_{l:c(l)=c(j)}p_{il}\right]\\
Q_{i}^{\beta_{k}\theta_{}{j}} &=p_{ij}I(c(j)=k)x_{i}^{T}b_{ij}c_{ij}^{T}-\left[\sum_{l: c(l) =k}p_{il}x_{i}^{T}b_{il}\right]\left[p_{ij}c_{ij}\right]^{T} \\
Q_{i}^{\eta_{j}\theta_{j'}} &=I(j=j')p_{ij}(1-\pi_{j|c(j)}(\eta))c_{ij}^{T} - \left[p_{ij}-\pi_{j|c(j)} \sum_{l:c(l)=c(j)}p_{il}\right]\left[p_{ij'}c_{ij'}\right]^{T}\\
\end{align*}