#LyX 2.2 created this file. For more info see http://www.lyx.org/
\lyxformat 508
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass article
\use_default_options true
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize default
\use_hyperref false
\papersize default
\use_geometry false
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\justification true
\use_refstyle 1
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Standard
Suppose we are solving an optimization problem in 
\begin_inset Formula $\theta$
\end_inset

 with meta-parameters 
\begin_inset Formula $x$
\end_inset

:
\begin_inset Formula 
\begin{align*}
\hat{\theta} & =\textrm{argmin}_{\theta}L\left(\theta,x\right)
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
Note that 
\begin_inset Formula $\hat{\theta}$
\end_inset

 is a function of 
\begin_inset Formula $x$
\end_inset

 through the optimization – for each 
\begin_inset Formula $x$
\end_inset

, we get a different 
\begin_inset Formula $\hat{\theta}$
\end_inset

 in general.
 Assuming all necessary smoothness, this means that 
\begin_inset Formula $\hat{\theta}$
\end_inset

 satisfies a first order condition for any 
\begin_inset Formula $x$
\end_inset

.
\begin_inset Formula 
\begin{align*}
\left.\frac{\partial L}{\partial\theta}\right|_{\hat{\theta},x} & =0
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
Differentiating with respect to 
\begin_inset Formula $x$
\end_inset

 using the chain rule gives
\begin_inset Formula 
\begin{align*}
\frac{d}{dx^{T}}\left.\frac{\partial L}{\partial\theta}\right|_{\hat{\theta},x} & =0\\
\left.\frac{\partial^{2}L}{\partial\theta\partial\theta}\right|_{\hat{\theta},x}\left.\frac{d\hat{\theta}}{dx^{T}}\right|_{\hat{\theta},x}+\left.\frac{\partial^{2}L}{\partial\theta\partial x}\right|_{\hat{\theta},x} & =0\Rightarrow\\
\left.\frac{d\hat{\theta}}{dx^{T}}\right|_{\hat{\theta},x} & =-\left(\left.\frac{\partial^{2}L}{\partial\theta\partial\theta}\right|_{\hat{\theta},x}\right)^{-1}\left.\frac{\partial^{2}L}{\partial\theta\partial x}\right|_{\hat{\theta},x}
\end{align*}

\end_inset

By using the chain rule one more time, we can differentiate any function
 of the optimum with respect to 
\begin_inset Formula $x$
\end_inset

:
\begin_inset Formula 
\begin{align*}
\left.\frac{df\left(\hat{\theta}\right)}{dx}\right|_{\hat{\theta},x} & =\frac{\partial f\left(\hat{\theta}\right)}{\partial\hat{\theta}^{T}}\frac{d\hat{\theta}}{dx}\\
 & =-\frac{\partial f\left(\hat{\theta}\right)}{\partial\hat{\theta}^{T}}\left(\left.\frac{\partial^{2}L}{\partial\theta\partial\theta}\right|_{\hat{\theta},x}\right)^{-1}\left.\frac{\partial^{2}L}{\partial\theta\partial x}\right|_{\hat{\theta},x}
\end{align*}

\end_inset


\begin_inset Newline newline
\end_inset


\end_layout

\end_body
\end_document