2018_ipwbootstrap_zhao.tex

\documentclass{beamer}
\usetheme{metropolis}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}

\usepackage{amstext}
%\usepackage{coloremoji}
\usepackage{layout}
\usepackage{multirow}

\usepackage{graphicx}
\graphicspath{ {figs/} }

\setbeameroption{hide notes}
\setbeamertemplate{note page}[plain]
\usepackage{listings}
\usepackage{datetime}
\usepackage{url}

% specifications for presenter mode
%\beamerdefaultoverlayspecification{<+->}
%\setbeamercovered{transparent}

% math shorthand
\usepackage{bm}
\usepackage{amsmath}
\usepackage{mathtools}
\newcommand{\R}{\mathbb{R}}
\newcommand{\D}{\mathcal{D}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\pr}{\mathbb{P}}
\newcommand{\F}{\mathcal{F}}
\newcommand{\X}{\mathcal{X}}
\newcommand{\lik}{\mathcal{L}}
\DeclarePairedDelimiterX{\infdivx}[2]{(}{)}{%
  #1\;\delimsize\|\;#2%
}
\newcommand{\infdiv}{D\infdivx}
\DeclarePairedDelimiter{\norm}{\lVert}{\rVert}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}

% indepndence notation macro
\newcommand\independent{\protect\mathpalette{\protect\independenT}{\perp}}
\def\independenT#1#2{\mathrel{\rlap{$#1#2$}\mkern2mu{#1#2}}}

% Bibliography
\usepackage{natbib}
\bibpunct{(}{)}{,}{a}{}{;}
\usepackage{bibentry}
%\nobibliography*

\title[ipw-bootstrap]{Sensitivity Analysis for Inverse Probability Weighting
  Estimators via the Percentile Bootstrap \small (Q.~Zhao, D.S.~Small, \&
  B.B.~Bhattacharya, 2017)}
\subtitle{\vspace*{0.5em} \scriptsize for ``Observational Study Design and
  Causal Inference'' (Statistics 260),\\ organized by S.~Pimentel, Spring 2018,
  University of California, Berkeley}
\author{Nima Hejazi}
\institute{Group in Biostatistics,\\ University of California, Berkeley\\
  \url{https://statistics.berkeley.edu/~nhejazi}
}
\date{12 April 2018}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\AtBeginSubsection[]{
\begin{frame}{Outline}
\tableofcontents[currentsection,currentsubsection]
\end{frame}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\subsection{Preliminaries}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Preliminaries: Notation}

\begin{itemize}
  \itemsep12pt
  \item Data: $(A_1, \bm{X_1}, Y_1), \ldots, (A_n, \bm{X_n}, Y_n)
    \stackrel{\text{iid}}{\sim} F_0$, for a (binary) treatment $A_i$ and
    measured confounders $\bm{X_i}$.
  \item Outcome: $Y_i = A_iY_i(1) + (1 - A_i)Y_i(0)$, using potential outcome
    notation.
  \item Estimand (Parameter): $\Delta \coloneqq \E_0[Y(1)] - \E_0[Y(0)]$\\
    (average treatment effect)
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Preliminaries: Identifiability and Assumptions}

\begin{itemize}
  \itemsep12pt
  \item Stable Unit Treatment Value Assumption (SUTVA)
  \item No Unmeasured Confounders (NUC): $(Y(0), Y(1)) \independent A \mid
    \bm{X}$\\(strong ignorability)
  \item Overlap: $e_0(x) \coloneqq \pr_0(A = 1 \mid X = x) \in (0, 1)$\\
    (bounded propensity score)
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Preliminaries: IPW Estimators}

\begin{itemize}
  \itemsep12pt
  \item Inverse Probability of Weighting (IPW) Estimators:
    \[
      \hat{\Delta}_{\text{IPW}} = \frac{1}{n} \sum_{i = 1}^n
      \frac{A_iY_i}{\hat{e}(X_i)} - \frac{(1 - A_i)Y_i}{1 - \hat{e}(X_i)}
    \]
  \item $\hat{\Delta}$ consistently estimates $\Delta$ as long as
    $\hat{e}(\bm{X})$ converges to $e_0$.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Objectives}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{The Objective: Sensitivity Analysis}

\begin{itemize}
  \itemsep12pt
  \item Under violations of the NUC assumption, $\hat{\Delta}_{\text{IPW}}$ is
    biased and has a confidence interval that doesn't cover $\Delta$ properly.
  \item The goal of a sensitivity analysis is to gauge the degree to which a
    statistical inference is incorrect under violations of the NUC assumption.
    \begin{itemize}
      \itemsep6pt
      \item To what extent could the existence of potentially unmeasured
        confounders invalidate our findings?
      \item ...
    \end{itemize}
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{The Objective: Sensitivity Analysis}

\begin{itemize}
  \itemsep12pt
  \item Under the NUC assumption, we have the following
    \[
      e_a(x,y) \coloneqq \pr_0(A = 1 \mid \bm{X} = x, Y(a) = y) = e_a(x),
    \]
    for $a \in \{0, 1\}$.
    \begin{itemize}
      \itemsep6pt
      \item $e_a(x,y)$ --- ``complete data'' selection probability.
      \item $e_a(x)$ --- ``observed data'' selection probability.
    \end{itemize}
  \item Thus, a sensitivity model might consider gauging whether $e_a(x,y) =
    e_a(x)$ holds in order to assess violations of NUC.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Methodology}
\subsection{Sensitivity: Parameter, Analysis, Inference}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Sensitivity Parameter}

\begin{itemize}
  \itemsep12pt
  \item Marginal Sensitivity Model: let
    \[
      \mathcal{E}(\Lambda) = \left\{e(x,y): \frac{1}{\Lambda} \leq
      \text{OR}(e(x,y), e_0(x)) \leq \Lambda, x \in \mathcal{X}, y \in
      \R \right\}
    \]
  \item Then, for observational causal inference, let us assume that
    $e_a(x,y) \in \mathcal{E}(\Lambda),$ for $a \in \{0, 1\}$.
  \item ...
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Sensitivity Parameter}

\begin{itemize}
  \itemsep12pt
  \item For convenience, we'll use a logistic representation of the sensitivity
    model:
    \[ h_0(x,y) = g_0(x) - g_0(x,y),\]
    where
    \begin{itemize}
      \itemsep6pt
      \item $g_0(x)=\text{logit}(e_0(x))=\text{log}\frac{e_0(x)}{1-e_0(x)}$
      \item Similarly, let $g_0(x,y) = \text{logit}(e_0(x,y))$.
    \end{itemize}
  \item Then, we may express $\mathcal{E}(\Lambda)$ as
    \[\mathcal{E}(\Lambda) = \{e^{(h)}(x,y): h \in \mathcal{H}(\lambda)\},\]
    where $\mathcal{H}(\lambda) = \{h: \mathcal{X} \times \R \to \R$ and
    $\lVert h \rVert_{\infty} \leq \lambda\}$.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Parametric Model for Sensitivity Analysis}

\begin{itemize}
  \itemsep12pt
  \item Ideally, $e_0(x)$ would be estimated nonparametrically, but, in many
    cases, we restrict ourselves to parametric models:
    \[
    \begin{aligned}
      e_{\beta_0}(x) =& \argmin_{\beta \in \Theta} \text{KL}(\pr_0(A = 1 \mid
      \bm{X} = x) \Vert \pr_{\beta}(A = 1 \mid \bm{X} = x))\\
      =& \argmax_{\beta \in \Theta} \E_0[A \cdot \text{log}e_{\beta}(X) +
      (1 - A) \cdot \text{log}(1 - e_{\beta}(x)) \mid \bm{X} = x]
    \end{aligned}
    \]
  \item As before, now have $e_0(x,y) \in \mathcal{E}_{\beta_0}(\Lambda)$, where
    \[\mathcal{E}_{\beta_0}(\Lambda) \coloneqq \left\{e(x,y): \frac{1}{\Lambda}
      \leq \text{OR}(e(x,y), e_{\beta_0}(x,y)) \leq \Lambda, x \in \mathcal{X},
      y \in \R \right\} \]
\end{itemize}

\note{
  This parametric setup broadens the definition of sensitivity analysis:
  \begin{itemize}
    \item Model misspecification: $e_{\beta_0}(x) \neq e_0(x)$
    \item No unmeasured confouding: $e_0(x) \neq e_0(x,y)$
  \end{itemize}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Confidence Intervals in Sensitivity Analysis}

\begin{itemize}
  \itemsep12pt
  \item ($1 - \alpha$)--CI: $\pr_0(\Delta \in [\text{L}, \text{U}]) \geq 1 -
    \alpha$ is true for any $F_0$ s.t.~$h_0 \in \mathcal{H}(\lambda)$, a
    collection of sensitivity models.
  \item Asymptotic ($1 - \alpha$)--CI if $\liminf\limits_{n \rightarrow \infty}
    \pr_0(\Delta \in [\text{L}, \text{U}]) \geq 1 - \alpha$
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Inverse Probability Weighting Estimators}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{IPW Point Estimates}

\begin{itemize}
  \itemsep12pt
  \item Note that the ATE may be expressed $\hat{\Delta}^{(h_0, h_1)} \coloneqq
    \hat{\mu}^{(h_1)}(1) - \hat{\mu}^{h_0}(0)$ for $h_0, h_1 \in
    \mathcal{H}(\lambda)$.
  \item Recall that the IPW estimator for a the mean $\mu$ in a missing data
    problem may be expressed:
    \[
      \hat{\mu}^{(h)}_{\text{IPW}} = \frac{1}{n} \sum_{i = 1}^n \frac{A_i
      Y_i}{\hat{e}^(h)(\bm{X_i}, Y_i)}
    \]
  \item The \textit{stabilized} IPW (SIPW) estimator is often used instead
    \[
      \hat{\mu}^{(h)}_{\text{SIPW}} = \left[\frac{1}{n} \sum_{i = 1}^n
      \frac{A_i}{\hat{e}^(h)(\bm{X_i}, Y_i)}\right]^{-1} \left[\frac{1}{n}
      \sum_{i = 1}^n \frac{A_iY_i}{\hat{e}^(h)(\bm{X_i}, Y_i)}\right]
    \]
\end{itemize}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Interval Construction for SIPW Estimators}

\begin{itemize}
  \itemsep12pt
  \item \textbf{Problem:} need to estimate variance computationally (e.g., via
    bootstrap) while considering all violations of NUC in
    $\mathcal{H}(\lambda)$.
  \item Given ($1-\alpha$)--CIs for all $h \in \mathcal{H}(\lambda)$:
    \[
      \liminf\limits_{n \rightarrow \infty} \pr_0(\mu^{(h)} \in [L^{(h)},
      U^{(h)}]) \geq 1 - \alpha
    \]
  \item Then, we have that the following is an asymptotic ($1-\alpha$)--CI under
    the collection of sensitivity models $\mathcal{H}(\lambda)$:
    \[
      L = \inf_{h \in \mathcal{H}(\lambda)}L^{(h)}, U = \sup_{h \in
        \mathcal{H}(\lambda)}U^{(h)}
    \]
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Percentile Bootstrap and Interval Construction}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Percentile Bootstrap}

\begin{itemize}
  \itemsep12pt
  \item Finally, the centerpiece! But first, we need still more notation.
  \item Let $\pr_n$ be the empirical measure on the sample $\bm{T_1}, \ldots,
    \bm{T_n}$, where $\bm{T_i} = (A_i, \bm{X'_{i}}, A_i Y_i)$.
  \item Further, let $\bm{\hat{T_1}}, \ldots, \bm{\hat{T_n}}$ be
    i.i.d.~re-samples from $\pr_n$.
  \item Then, the SIPW estimate $\hat{\hat{\mu}}^{(h)}$ may be computed over the
    bootstrap re-samples $\{\bm{\hat{T_i}}\}_{i \in [n]}$.
  \item Now, for $h \in \mathcal{H}(\lambda)$, percentile bootstrap CI:
    \[
      [L^{(h)}, U^{(h)}] = \left[Q_{\frac{\alpha}{2}}(\hat{\hat{\mu}}^{(h)}),
      Q_{1 - \frac{\alpha}{2}}(\hat{\hat{\mu}}^{(h)})\right],
    \]
    where $Q_{\alpha}(\hat{\hat{\mu}}) \coloneqq \inf\{t:
    \hat{\pr}_n(\hat{\hat{\mu}} \leq t) \geq \alpha\}$
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Percentile Bootstrap}

\begin{itemize}
  \itemsep8pt
  \item The percentile bootstrap interval $[L^{(h)}, U^{(h)}]$ is an
    asymptotically valid CI of the target estimate for the parametric
    sensitivity model $e^{(h)} \in \mathcal{E}_{\beta_0}(\Lambda)$.
  \item The bootstrap is not valid if the missingness probability is modeled
    nonparametrically (Abadie \& Imbens, 2008).
  \item Percentile bootstrap CI under collection of sensitivity models:
    \[
      [L, U] = \left[\left(Q_{\frac{\alpha}{2}}(\inf_{h \in
        \mathcal{H}(\lambda)} \hat{\hat{\mu}}^{(h)})\right), \left(Q_{1 -
        \frac{\alpha}{2}}(\sup_{h \in \mathcal{H}(\lambda)}
        \hat{\hat{\mu}}^{(h)})\right)\right]
    \]
    \begin{itemize}
      \item \small Since infimum/supremum is inside the quantile function, the
        problem is efficiently solved by linear programming.
      \item \small Exchange of quantile and infimum/supremum is justified by a
        generalized (von Neumann's) minimax/maximin inequality.
    \end{itemize}
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Linear Fractional Programming of SIPW Point Estimates}

\begin{itemize}
  \itemsep12pt
  \item Bootstrap Intervals of the range of SIPW point estimates:
    \[
      [L_B, U_B] = \left[\left(Q_{\frac{\alpha}{2}}(\inf_{h \in
        \mathcal{H}(\lambda)} \hat{\hat{\mu}}^{(h)})_{b \in [B]}\right),
        \left(Q_{1 - \frac{\alpha}{2}}(\sup_{h \in \mathcal{H}(\lambda)}
        \hat{\hat{\mu}}^{(h)})_{b \in [B]}\right)\right]
    \]
  \item In the linear programming problem, the optimization variable is merely
    $z_i = e^{h(\bm{X}_i, Y_i)}$, as all other relevant quantities may be
    readily estimated from the observed data.
  \item Computation is extremely efficient, with complexity $O(nB + n
    \text{log}n)$.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{CI for the ATE in the Sensitivity Model}

\begin{itemize}
  \itemsep12pt
  \item Using the approach we've been discussing, we readily obtain an
    asymptotically valid CI for the ATE via the percentile bootstrap:
    \[
      \left[Q_{\frac{\alpha}{2}}(\hat{\hat{\Delta}}^{(h_0, h_1)}),
      Q_{1 - \frac{\alpha}{2}}(\hat{\hat{\Delta}}^{(h_0, h_1)})\right],
    \]
    where $Q_{\frac{\alpha}{2}}(\hat{\hat{\Delta}}^{(h_0, h_1)})$ is the
    $\alpha$-th bootstrap quantile of the SIPW estimates.
  \item From this, we obtain CIs 
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Extensions: Augmented IPW Estimators}

\begin{itemize}
  \itemsep12pt
  \item AIPW estimators are \textit{``double robust''}, incorporating an extra
    nuisance parameter: $f_0(\bm{x}) = \E_0[Y \mid A = 1, \bm{X} = \bm{x}]$.
  \item The AIPW estimator $\hat{\Delta}_{\text{AIPW}}$ is consistent for
    $\Delta$ as long as one of $\hat{e}(\bm{x})$ and $\hat{f}(\bm{x})$ is
    consistent.
  \item \textbf{Limitation:} In order to compute asymptotically valid confidence
    intervals, the outcome regression model $\hat{f}(\bm{X}_i)$ must be
    parametric.\\ \pause
    \small{\textit{Perhaps this could be loosened.}}
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}
\subsection{Comparisons}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Comparison with Rosenbaum Sensitivity}

\begin{itemize}
  \itemsep12pt
  \item Rosenbaum's method obtains point estimates and CIs under the collection
    of models $\mathcal{R}(\Gamma)$.
    \begin{itemize}
      \itemsep8pt
      \item Here, classically, we assume that the causal effect is additive and
        constant.
      \item This means the Fisher null can be used to determine whether an
        effect $\Delta$ ought to be included in the ($1-\alpha$)--CI.
    \end{itemize}
  \item The present approach is a hybrid of existing approaches in the sense
    that it considers a range of differencs between $A\mid\bm{X}$ and
    $A\mid\bm{X}, Y(a)$.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Comparison with Rosenbaum Sensitivity}

\begin{itemize}
  \itemsep8pt
  \item Generally, Rosenbaum's method treats the sample as the population, the
    present approach treats the observations as i.i.d.~samples from a
    super-population.
  \item The present method uses IPW-type estimators --- importantly, this makes
    exact matching completely unnecessary.
  \item The present approach is natural for IPW-type estimators while
    Rosenbaum's is natural for matched designs.
  \item Most methods use randomization inference based on Fisher's sharp null,
    the present approach takes a point estimation perspective.
  \item Heterogeneous treatment effects; applicability to missing data problems.
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Remarks}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Thoughts and Impressions}

  \Huge What do I think?

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{I've Talked Enough$\ldots$}

  \Huge What do you think?

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% don't want dimming with references
\setbeamercovered{}
\beamerdefaultoverlayspecification{}

\begin{frame}[c,allowframebreaks]{References}

\small
\bibliographystyle{plainnat}
\nocite{*}
\bibliography{refs}
\itemize

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}