\RequirePackage[switch, columnwise, running, mathlines, displaymath, mathlines]{lineno}
\RequirePackage{docswitch}
% \flag is set by the user, through the makefile:
%    make note
%    make apj
% etc.
\setjournal{\flag}

\documentclass[\docopts]{\docclass}

% You could also define the document class directly
%\documentclass[]{emulateapj}

% Custom commands from LSST DESC, see texmf/styles/lsstdesc_macros.sty
\usepackage{lsstdesc_macros}

\usepackage{graphicx}
\usepackage{url}
\graphicspath{{./}{./figures/}}
\bibliographystyle{apj}

% Add your own macros here:
\newcommand{\textul}[1]{\underline{#1}}

\newcommand{\aim}[1]{\textcolor{red}{#1}}
\newcommand{\changes}[1]{\textcolor{blue}{#1}}

% in case I can think of some fancy formatting later\dots
\newcommand{\lsst}{\textsc{LSST}}
\newcommand{\plasticc}{\textsc{PLAsTiCC}}
\newcommand{\proclam}{\texttt{proclam}}
\newcommand{\snmachine}{\texttt{snmachine}}
\newcommand{\snphotcc}{\textsc{SNPhotCC}}

% ======================================================================

\begin{document}
% \linenumbers

\title{The Photometric LSST Astronomical Time-series Classification Challenge (PLA\MakeLowercase{s}T\MakeLowercase{i}CC): Selection of a performance metric for classification probabilities \\ balancing diverse science goals}

\maketitlepre


\begin{abstract}

  \vspace{0.5cm}

  Classification of transient and variable light curves is an essential step in using astronomical observations to develop an understanding of the underlying physical processes from which they arise.
  However, upcoming deep photometric surveys, including the Large Synoptic Survey Telescope (\textsc{LSST}), will produce a deluge of low signal-to-noise data for which traditional type estimation procedures are inappropriate.
  Probabilistic classification is more appropriate for the data but is incompatible with the traditional metrics used on deterministic classifications.
  Furthermore, large survey collaborations like \textsc{LSST} intend to use the resulting classification probabilities for diverse science objectives, indicating a need for a metric that balances a variety of goals.
  We describe the process used to develop an optimal performance metric for an open classification challenge that seeks to identify probabilistic classifiers that can serve many scientific interests.
  The Photometric \textsc{LSST} Astronomical Time-series Classification Challenge (\textsc{PLAsTiCC}) aims to identify promising techniques for obtaining classification probabilities of transient and variable objects by engaging a broader community beyond astronomy.
  Using mock classification probability submissions emulating realistically complex archetypes of those anticipated of \textsc{PLAsTiCC}, we compare the sensitivity of two metrics of classification probabilities under various weighting schemes, finding that both yield results that are qualitatively consistent with intuitive notions of classification performance.
  We thus choose as a metric for \textsc{PLAsTiCC} a weighted modification of the cross-entropy because it can be meaningfully interpreted in terms of information content.
  Finally, we propose extensions of our methodology to ever more complex challenge goals and suggest some guiding principles for approaching the choice of a metric of probabilistic data products.

  \vspace{1cm}

\end{abstract}

% Keywords are ignored in the LSST DESC Note style:
\dockeys{}

\maketitlepost

% ----------------------------------------------------------------------
% {% raw %}

\clearpage

\input{./tex/introduction}

\input{./tex/data}

\input{./tex/methods}

\input{./tex/results}

\input{./tex/discussion}

\input{./tex/conclusions}

% ----------------------------------------------------------------------

\subsection*{Acknowledgments}

%%% Here is where you should add your specific acknowledgments, remembering that some standard thanks will be added via the \code{desc-tex/ack/*.tex} and \code{contributions.tex} files.

\input{contributions} % Standard papers only: author contribution statements. For examples, see http://blogs.nature.com/nautilus/2007/11/post_12.html

% This work used TBD kindly provided by Not-A-DESC Member and benefitted from comments by Another Non-DESC person.

% Standard papers only: A.B.C. acknowledges support from grant 1234 from ...
This paper has undergone internal review in the LSST Dark Energy Science Collaboration. % REQUIRED if true
The authors would like to thank Melissa Graham, Weikang Lin, and Chad Schafer for serving as the LSST-DESC publication review committee.
The authors further wish to thank Tom Loredo for helpful feedback provided in the preparation of this paper.

AIM is advised by David W. Hogg and was supported by National Science Foundation grant AST-1517237.
TA is supported in part by STFC.
RB and CS are supported by the Swedish Research Council (VR) through the Oskar Klein Centre.
Their work was further supported by the research environment grant ``Gravitational Radiation and Electromagnetic Astrophysical Transients (GREAT)'' funded by the Swedish Research council (VR) under Dnr 2016-06012.
AAM was supported in part by the NSF grants AST-0909182, AST-1313422, AST-1413600, and AST-1518308, and by the Ajax Foundation

The financial assistance of the National Research Foundation (NRF) towards this research is hereby acknowledged.
Opinions expressed and conclusions arrived at, are those of the authors and are not necessarily to be attributed to the NRF.
This work is partially supported by the European Research Council under the European Community’s Seventh Framework Programme (FP7/2007-2013)/ERC grant agreement no 306478-CosmicDawn.

Canadian co-authors acknowledge support from the Natural Sciences and Engineering Research Council of Canada.
The Dunlap Institute is funded through an endowment established by the David Dunlap family and the University of Toronto.
The authors at the University of Toronto acknowledge that the land on which the University of Toronto is built is the traditional territory of the Haudenosaunee, and most recently, the territory of the Mississaugas of the New Credit First Nation.
They are grateful to have the opportunity to work in the community, on this territory.

We acknowledge the University of Chicago Research Computing Center for support of this work.
This research used resources of the National Energy Research Scientific Computing Center (NERSC), a U.S. Department of Energy Office of Science User Facility operated under Contract No. DE-AC02-05CH11231.
This research at Rutgers University is supported by US Department of Energy award DE-SC0011636.


\input{desc-tex/ack/standard} % also available: key standard_short

% The DESC acknowledges ongoing support from the Institut National de Physique Nucleaire et de Physique des Particules in France; the Science \& Technology Facilities Council in the United Kingdom; and the Department of Energy, the National Science Foundation, and the LSST Corporation in the United States.
%
% DESC uses resources of the IN2P3 Computing Center (CC-IN2P3--Lyon/Villeurbanne - France) funded by the Centre National de la Recherche Scientifique; the National Energy Research Scientific Computing Center, a DOE Office of Science User Facility supported by the Office of Science of the U.S. Department of Energy under Contract No. DE-AC02-05CH11231; STFC DiRAC HPC Facilities, funded by UK BIS National E-infrastructure capital grants; and the UK particle physics grid, supported by the GridPP Collaboration.
%
% This work was performed in part under DOE Contract DE-AC02-76SF00515.

% We acknowledge the use of An-External-Tool-like-NED-or-ADS.

%{\it Facilities:} \facility{LSST}

% Include both collaboration papers and external citations:
\bibliography{main,lsstdesc}

\end{document}

% ======================================================================