https://github.com/greenelab/meta-review
Raw File
Tip revision: 58ea23c7335e63ca89353cb44cb79194b62b3c83 authored by Daniel Himmelstein on 25 May 2020, 15:06:53 UTC
PR #265 to update Rootstock on 2020-05-25
Tip revision: 58ea23c
index.html
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en-US" xml:lang="en-US">
<head>
  <meta charset="utf-8" />
  <meta name="generator" content="pandoc" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
  <meta name="author" content="Daniel S. Himmelstein" />
  <meta name="author" content="Vincent Rubinetti" />
  <meta name="author" content="David R. Slochower" />
  <meta name="author" content="Dongbo Hu" />
  <meta name="author" content="Venkat S. Malladi" />
  <meta name="author" content="Casey S. Greene" />
  <meta name="author" content="Anthony Gitter" />
  <meta name="dcterms.date" content="2020-05-25" />
  <meta name="keywords" content="manubot, deep review, publishing, open science, open access, collaboration, writing, github, git, continuous integration, markdown, pandoc" />
  <title>Open collaborative writing with Manubot</title>
  <style>
    code{white-space: pre-wrap;}
    span.smallcaps{font-variant: small-caps;}
    span.underline{text-decoration: underline;}
    div.column{display: inline-block; vertical-align: top; width: 50%;}
    div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
    ul.task-list{list-style: none;}
    pre > code.sourceCode { white-space: pre; position: relative; }
    pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
    pre > code.sourceCode > span:empty { height: 1.2em; }
    code.sourceCode > span { color: inherit; text-decoration: inherit; }
    div.sourceCode { margin: 1em 0; }
    pre.sourceCode { margin: 0; }
    @media screen {
    div.sourceCode { overflow: auto; }
    }
    @media print {
    pre > code.sourceCode { white-space: pre-wrap; }
    pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
    }
    pre.numberSource code
      { counter-reset: source-line 0; }
    pre.numberSource code > span
      { position: relative; left: -4em; counter-increment: source-line; }
    pre.numberSource code > span > a:first-child::before
      { content: counter(source-line);
        position: relative; left: -1em; text-align: right; vertical-align: baseline;
        border: none; display: inline-block;
        -webkit-touch-callout: none; -webkit-user-select: none;
        -khtml-user-select: none; -moz-user-select: none;
        -ms-user-select: none; user-select: none;
        padding: 0 4px; width: 4em;
        color: #aaaaaa;
      }
    pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa;  padding-left: 4px; }
    div.sourceCode
      {   }
    @media screen {
    pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
    }
    code span.al { color: #ff0000; font-weight: bold; } /* Alert */
    code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
    code span.at { color: #7d9029; } /* Attribute */
    code span.bn { color: #40a070; } /* BaseN */
    code span.bu { } /* BuiltIn */
    code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
    code span.ch { color: #4070a0; } /* Char */
    code span.cn { color: #880000; } /* Constant */
    code span.co { color: #60a0b0; font-style: italic; } /* Comment */
    code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
    code span.do { color: #ba2121; font-style: italic; } /* Documentation */
    code span.dt { color: #902000; } /* DataType */
    code span.dv { color: #40a070; } /* DecVal */
    code span.er { color: #ff0000; font-weight: bold; } /* Error */
    code span.ex { } /* Extension */
    code span.fl { color: #40a070; } /* Float */
    code span.fu { color: #06287e; } /* Function */
    code span.im { } /* Import */
    code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
    code span.kw { color: #007020; font-weight: bold; } /* Keyword */
    code span.op { color: #666666; } /* Operator */
    code span.ot { color: #007020; } /* Other */
    code span.pp { color: #bc7a00; } /* Preprocessor */
    code span.sc { color: #4070a0; } /* SpecialChar */
    code span.ss { color: #bb6688; } /* SpecialString */
    code span.st { color: #4070a0; } /* String */
    code span.va { color: #19177c; } /* Variable */
    code span.vs { color: #4070a0; } /* VerbatimString */
    code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
  </style>
  <!--[if lt IE 9]>
    <script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
  <![endif]-->
  <!--
  Manubot generated metadata rendered from header-includes-template.html.
  Suggest improvements at https://github.com/manubot/manubot/blob/master/manubot/process/header-includes-template.html
  -->
  <meta name="dc.format" content="text/html" />
  <meta name="dc.title" content="Open collaborative writing with Manubot" />
  <meta name="citation_title" content="Open collaborative writing with Manubot" />
  <meta property="og:title" content="Open collaborative writing with Manubot" />
  <meta property="twitter:title" content="Open collaborative writing with Manubot" />
  <meta name="dc.date" content="2020-05-25" />
  <meta name="citation_publication_date" content="2020-05-25" />
  <meta name="dc.language" content="en-US" />
  <meta name="citation_language" content="en-US" />
  <meta name="dc.relation.ispartof" content="Manubot" />
  <meta name="dc.publisher" content="Manubot" />
  <meta name="citation_journal_title" content="Manubot" />
  <meta name="citation_technical_report_institution" content="Manubot" />
  <meta name="citation_author" content="Daniel S. Himmelstein" />
  <meta name="citation_author_institution" content="Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America" />
  <meta name="citation_author_orcid" content="0000-0002-3012-7446" />
  <meta name="twitter:creator" content="@dhimmel" />
  <meta name="citation_author" content="Vincent Rubinetti" />
  <meta name="citation_author_institution" content="Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America" />
  <meta name="citation_author_orcid" content="0000-0002-4655-3773" />
  <meta name="twitter:creator" content="@vincerubinetti" />
  <meta name="citation_author" content="David R. Slochower" />
  <meta name="citation_author_institution" content="Skaggs School of Pharmacy and Pharmaceutical Sciences, University of California, San Diego, San Diego, California, United States of America" />
  <meta name="citation_author_orcid" content="0000-0003-3928-5050" />
  <meta name="twitter:creator" content="@drslochower" />
  <meta name="citation_author" content="Dongbo Hu" />
  <meta name="citation_author_institution" content="Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America" />
  <meta name="citation_author_orcid" content="0000-0003-2606-3969" />
  <meta name="twitter:creator" content="@dongbohu" />
  <meta name="citation_author" content="Venkat S. Malladi" />
  <meta name="citation_author_institution" content="Department of Bioinformatics, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America" />
  <meta name="citation_author_institution" content="Bioinformatics Core Facility, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America" />
  <meta name="citation_author_orcid" content="0000-0002-0144-0564" />
  <meta name="twitter:creator" content="@katatonikkat" />
  <meta name="citation_author" content="Casey S. Greene" />
  <meta name="citation_author_institution" content="Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America" />
  <meta name="citation_author_orcid" content="0000-0001-8713-9213" />
  <meta name="twitter:creator" content="@GreeneScientist" />
  <meta name="citation_author" content="Anthony Gitter" />
  <meta name="citation_author_institution" content="Department of Biostatistics and Medical Informatics, University of Wisconsin-Madison, Madison, Wisconsin, United States of America" />
  <meta name="citation_author_institution" content="Morgridge Institute for Research, Madison, Wisconsin, United States of America" />
  <meta name="citation_author_orcid" content="0000-0002-5324-9833" />
  <meta name="twitter:creator" content="@anthonygitter" />
  <link rel="canonical" href="https://greenelab.github.io/meta-review/" />
  <meta property="og:url" content="https://greenelab.github.io/meta-review/" />
  <meta property="twitter:url" content="https://greenelab.github.io/meta-review/" />
  <meta name="citation_fulltext_html_url" content="https://greenelab.github.io/meta-review/" />
  <meta name="citation_pdf_url" content="https://greenelab.github.io/meta-review/manuscript.pdf" />
  <link rel="alternate" type="application/pdf" href="https://greenelab.github.io/meta-review/manuscript.pdf" />
  <link rel="alternate" type="text/html" href="https://greenelab.github.io/meta-review/v/2482af490804891c7f00b248c4e7ef7f356a0730/" />
  <meta name="manubot_html_url_versioned" content="https://greenelab.github.io/meta-review/v/2482af490804891c7f00b248c4e7ef7f356a0730/" />
  <meta name="manubot_pdf_url_versioned" content="https://greenelab.github.io/meta-review/v/2482af490804891c7f00b248c4e7ef7f356a0730/manuscript.pdf" />
  <meta property="og:type" content="article" />
  <meta property="twitter:card" content="summary_large_image" />
  <meta property="og:image" content="https://github.com/greenelab/meta-review/raw/2482af490804891c7f00b248c4e7ef7f356a0730/thumbnail.png" />
  <meta property="twitter:image" content="https://github.com/greenelab/meta-review/raw/2482af490804891c7f00b248c4e7ef7f356a0730/thumbnail.png" />
  <link rel="icon" type="image/png" sizes="192x192" href="https://manubot.org/favicon-192x192.png" />
  <link rel="mask-icon" href="https://manubot.org/safari-pinned-tab.svg" color="#ad1457" />
  <meta name="theme-color" content="#ad1457" />
  <!-- end Manubot generated metadata -->
</head>
<body>
<header id="title-block-header">
<h1 class="title">Open collaborative writing with Manubot</h1>
</header>
<p>A DOI-citable version of this manuscript is available in <em>PLOS Computational Biology</em> at <a href="https://doi.org/10.1371/journal.pcbi.1007128" class="uri">https://doi.org/10.1371/journal.pcbi.1007128</a>.
This version of the manuscript <a href="https://github.com/greenelab/meta-review/compare/b90923a99208c9152cec7929399ab1b7929cb7e2...2482af490804891c7f00b248c4e7ef7f356a0730">contains changes</a> subsequent to the journal publication.</p>
<p><small><em>
This manuscript
(<a href="https://greenelab.github.io/meta-review/v/2482af490804891c7f00b248c4e7ef7f356a0730/">permalink</a>)
was automatically generated
from <a href="https://github.com/greenelab/meta-review/tree/2482af490804891c7f00b248c4e7ef7f356a0730">greenelab/meta-review@2482af4</a>
on May 25, 2020.
</em></small></p>
<h2 id="authors">Authors</h2>
<ul>
<li><p><strong>Daniel S. Himmelstein</strong> <sup><a href="#correspondence">✉</a></sup><br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0002-3012-7446">0000-0002-3012-7446</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/dhimmel">dhimmel</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/dhimmel">dhimmel</a><br>
<small>
Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America
</small></p></li>
<li><p><strong>Vincent Rubinetti</strong> <br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0002-4655-3773">0000-0002-4655-3773</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/vincerubinetti">vincerubinetti</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/vincerubinetti">vincerubinetti</a><br>
<small>
Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America
</small></p></li>
<li><p><strong>David R. Slochower</strong> <br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0003-3928-5050">0000-0003-3928-5050</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/slochower">slochower</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/drslochower">drslochower</a><br>
<small>
Skaggs School of Pharmacy and Pharmaceutical Sciences, University of California, San Diego, San Diego, California, United States of America
</small></p></li>
<li><p><strong>Dongbo Hu</strong> <br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0003-2606-3969">0000-0003-2606-3969</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/dongbohu">dongbohu</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/dongbohu">dongbohu</a><br>
<small>
Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America
</small></p></li>
<li><p><strong>Venkat S. Malladi</strong> <br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0002-0144-0564">0000-0002-0144-0564</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/vsmalladi">vsmalladi</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/katatonikkat">katatonikkat</a><br>
<small>
Department of Bioinformatics, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America; Bioinformatics Core Facility, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America
</small></p></li>
<li><p><strong>Casey S. Greene</strong> <br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0001-8713-9213">0000-0001-8713-9213</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/cgreene">cgreene</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/GreeneScientist">GreeneScientist</a><br>
<small>
Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America
</small></p></li>
<li><p><strong>Anthony Gitter</strong> <sup><a href="#correspondence">✉</a></sup><br>
<img src="images/orcid.svg" class="inline_icon" alt="ORCID icon" />
<a href="https://orcid.org/0000-0002-5324-9833">0000-0002-5324-9833</a>
· <img src="images/github.svg" class="inline_icon" alt="GitHub icon" />
<a href="https://github.com/agitter">agitter</a>
· <img src="images/twitter.svg" class="inline_icon" alt="Twitter icon" />
<a href="https://twitter.com/anthonygitter">anthonygitter</a><br>
<small>
Department of Biostatistics and Medical Informatics, University of Wisconsin-Madison, Madison, Wisconsin, United States of America; Morgridge Institute for Research, Madison, Wisconsin, United States of America
</small></p></li>
</ul>
<div id="correspondence">
<p>✉ — correspondence preferred via <a href="https://github.com/greenelab/meta-review/issues">GitHub Issues</a>.
Otherwise, address correspondence to <a href="mailto:gitter@biostat.wisc.edu" class="email">gitter@biostat.wisc.edu</a> and <a href="mailto:daniel.himmelstein@gmail.com" class="email">daniel.himmelstein@gmail.com</a>.</p>
</div>
<!--


Daniel S. Himmelstein^1[*](#correspondence)^,
Vincent Rubinetti^1^,
David R. Slochower^2^,
Dongbo Hu^1^,
Venkat S. Malladi^3,4^,
Casey S. Greene^1^,
Anthony Gitter^5,6[*](#correspondence)^



1. Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, Pennsylvania, United States of America
2. Skaggs School of Pharmacy and Pharmaceutical Sciences, University of California, San Diego, San Diego, California, United States of America
3. Department of Bioinformatics, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America
4. Bioinformatics Core Facility, University of Texas Southwestern Medical Center, Dallas, Texas, United States of America
5. Department of Biostatistics and Medical Informatics, University of Wisconsin-Madison, Madison, Wisconsin, United States of America
6. Morgridge Institute for Research, Madison, Wisconsin, United States of America


::: {#correspondence}
\* — correspondence preferred via [GitHub Issues](https://github.com/greenelab/meta-review/issues).
Otherwise, address correspondence to <gitter@biostat.wisc.edu> and <daniel.himmelstein@gmail.com>.
:::
-->
<h2 class="page_break_before" id="abstract">Abstract</h2>
<p>Open, collaborative research is a powerful paradigm that can immensely strengthen the scientific process by integrating broad and diverse expertise.
However, traditional research and multi-author writing processes break down at scale.
We present new software named Manubot, available at <a href="https://manubot.org" class="uri">https://manubot.org</a>, to address the challenges of open scholarly writing.
Manubot adopts the contribution workflow used by many large-scale open source software projects to enable collaborative authoring of scholarly manuscripts.
With Manubot, manuscripts are written in Markdown and stored in a Git repository to precisely track changes over time.
By hosting manuscript repositories publicly, such as on GitHub, multiple authors can simultaneously propose and review changes.
A cloud service automatically evaluates proposed changes to catch errors.
Publication with Manubot is continuous:
When a manuscript’s source changes, the rendered outputs are rebuilt and republished to a webpage.
Manubot automates bibliographic tasks by implementing citation by identifier, where users cite persistent identifiers (e.g. DOIs, PubMed IDs, ISBNs, URLs), whose metadata is then retrieved and converted to a user-specified style.
Manubot modernizes publishing to align with the ideals of open science by making it transparent, reproducible, immediate, versioned, collaborative, and free of charge.</p>
<h2 id="author-summary">Author summary</h2>
<p>Traditionally, scholarly manuscripts have been written in private by a predefined team of collaborators.
But now the internet enables realtime open science, where project communication occurs online in a public venue and anyone is able to contribute.
Dispersed teams of online contributors require new tools to jointly prepare manuscripts.</p>
<p>Existing tools fail to scale beyond tens of authors and struggle to support iterative refinement of proposed changes.
Therefore, we created a system called Manubot for writing manuscripts based on collaborative version control.
Manubot adopts the workflow from open source software development, which has enabled hundreds of contributors to simultaneously develop complex codebases such as Python and Linux, and applies it to open collaborative writing.</p>
<p>Manubot also addresses other shortcomings of current publishing tools.
Specifically, all changes to a manuscript are tracked, enabling transparency and better attribution of credit.
Manubot automates many tasks, including creating the bibliography and deploying the manuscript as a webpage.
Manubot webpages preserve old versions and provide a simple yet interactive interface for reading.
As such, Manubot is a suitable foundation for next-generation preprints.
Manuscript readers have ample opportunity to not only provide public peer review but also to contribute improvements, before and after journal publication.</p>
<h2 class="page_break_before" id="introduction">Introduction</h2>
<p>The internet enables science to be shared in real-time at a low cost to a global audience.
This development has decreased the barriers to making science open, while supporting new massively collaborative models of research <span class="citation" data-cites="1DiVJ3t6P">[<a href="#ref-1DiVJ3t6P" role="doc-biblioref">1</a>]</span>.
However, the scientific community requires tools whose workflows encourage openness <span class="citation" data-cites="IWBJQIkl">[<a href="#ref-IWBJQIkl" role="doc-biblioref">2</a>]</span>.
Manuscripts are the cornerstone of scholarly communication, but drafting and publishing manuscripts has traditionally relied on proprietary or offline tools that do not support <em>open scholarly writing</em>, in which anyone is able to contribute and the contribution history is preserved and public.
We introduce <a href="https://manubot.org">Manubot</a>, a new tool and infrastructure for authoring scholarly manuscripts in the open, and report how it was instrumental for the collaborative project that led to its creation.</p>
<p>Based on our experience leading a recent open review <span class="citation" data-cites="16CgM2x0z">[<a href="#ref-16CgM2x0z" role="doc-biblioref">3</a>]</span>, we discuss the advantages and challenges of open collaborative writing, a form of crowdsourcing <span class="citation" data-cites="ZPiRHuLc">[<a href="#ref-ZPiRHuLc" role="doc-biblioref">4</a>]</span>.
Our review manuscript <span class="citation" data-cites="PZMP42Ak">[<a href="#ref-PZMP42Ak" role="doc-biblioref">5</a>]</span> was code-named the Deep Review and surveyed deep learning’s role in biology and precision medicine, a research area undergoing explosive growth.
We initiated the Deep Review in August 2016 by creating a GitHub repository (<a href="https://github.com/greenelab/deep-review" class="uri">https://github.com/greenelab/deep-review</a>) to coordinate and manage contributions.
GitHub is a platform designed for collaborative software development that is adaptable for collaborative writing.
From the start, we made the GitHub repository public under a Creative Commons Attribution License (<a href="https://github.com/greenelab/deep-review/blob/master/LICENSE.md" title="Creative Commons Attribution 4.0 International License">CC BY 4.0</a>).
We encouraged anyone interested to contribute by proposing changes or additions.
Although we invited some specific experts to participate, most authors discovered the manuscript organically through conferences or social media, deciding to contribute without solicitation.
In total, the Deep Review attracted 36 authors, who were not determined in advance, from 20 different institutions in less than two years.</p>
<p>The Deep Review and other studies that subsequently adopted the Manubot platform were unequivocal successes bolstered by the collaborative approach.
However, inviting wide authorship brought many technical and social challenges such as how to fairly distribute credit, coordinate the scientific content, and collaboratively manage extensive reference lists.
The manuscript writing process we developed using the Markdown language, the GitHub platform, and our new Manubot tool for automating manuscript generation addresses these challenges.</p>
<p>Manubot supports citations by adding a persistent identifier like a Digital Object Identifier (DOI) or PubMed Identifier (PMID) directly in the text so that large groups of authors do not have to coordinate reference lists.
When text is changed, Manubot automatically updates the manuscript’s webpage so that all authors can read and edit from the latest version.
Because manuscripts are created from GitHub repositories, Manubot supports a workflow where all edits are reviewed and discussed, ensuring that the collaborative text has a cohesive style and message and that authors receive precise credit for their work.
These and other features support an open collaborative writing process that is not feasible with other writing platforms.</p>
<h2 id="collaborative-writing-platforms">Collaborative writing platforms</h2>
<p>There are many existing collaborative writing platforms (Table <a href="#tbl:platforms">1</a>) <span class="citation" data-cites="AylLD9F8">[<a href="#ref-AylLD9F8" role="doc-biblioref">6</a>]</span>.
In general, platforms with “what you see is what you get” (WYSIWYG) editors, such as Microsoft Word or Google Docs, require the least technical expertise to use.
On the flip side, WYSIWYG platforms can be difficult to customize and incorporate into automated computational workflows.
Traditionally, LaTeX has been used for these needs, since documents are written in plain text and the system is open source and extensible.
Rendering LaTeX documents requires specialized software, but webapps like <a href="https://www.overleaf.com/">Overleaf</a> now enable collaborative authoring of LaTeX documents.
Nonetheless, LaTeX-based systems are limited in that PDF (or similar) is the only fully supported output format.
Alternatively, <a href="https://www.authorea.com/">Authorea</a> is a collaborative writing webapp whose primary output format is HTML.
Authorea allows authors to write in Markdown, a limited subset of LaTeX, or their WYSIWYG HTML editor.</p>
<div id="tbl:platforms" class="tablenos">
<table style="width:100%;">
<caption><span>Table 1:</span> Collaborative writing platforms.
A summary of features that differentiate Manubot from existing collaborative writing platforms.
We assessed features in June 2018 using the free version of each platform and updated our assessment in April 2019 to add the features in the bottom three rows and re-evaluate Authorea and Overleaf.
Some platforms offer additional features through a paid subscription or software.
1) Additional functionality, such as bibliography management and tracking changes, is available by editing the Word document stored in OneDrive with the paid Word desktop application.
2) Conversations about modifications take place on the document as comments, annotations, or unsaved chats.
There is no integrated forum for discussing and editing revisions.
3) In some circumstances, Overleaf Git commits are not modular.
Edits made by distinct authors may be attributed to a single author.
The GitHub Sync feature attributes all edits to the project owner.
</caption>
<colgroup>
<col style="width: 14%" />
<col style="width: 14%" />
<col style="width: 14%" />
<col style="width: 14%" />
<col style="width: 14%" />
<col style="width: 14%" />
<col style="width: 14%" />
</colgroup>
<thead>
<tr class="header">
<th>Feature</th>
<th>Manubot</th>
<th>Authorea</th>
<th>Overleaf v2</th>
<th>Google Docs + Paperpile</th>
<th>Word Online<sup>1</sup></th>
<th>Markdown on GitHub</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Multi-author editing</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
</tr>
<tr class="even">
<td>Propose changes</td>
<td>Yes</td>
<td>No</td>
<td>No</td>
<td>Yes</td>
<td>No</td>
<td>Yes</td>
</tr>
<tr class="odd">
<td>Continuous integration testing</td>
<td>Yes</td>
<td>No</td>
<td>No</td>
<td>No</td>
<td>No</td>
<td>No</td>
</tr>
<tr class="even">
<td>Multi-participant conversation for changes</td>
<td>Yes</td>
<td>No<sup>2</sup></td>
<td>No<sup>2</sup></td>
<td>No<sup>2</sup></td>
<td>No<sup>2</sup></td>
<td>Yes</td>
</tr>
<tr class="odd">
<td>Character-level provenance for text</td>
<td>Yes</td>
<td>Yes</td>
<td>No<sup>3</sup></td>
<td>Requires manual inspection of history</td>
<td>Not after changes are accepted</td>
<td>Yes</td>
</tr>
<tr class="even">
<td>Bibliography management</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>No, requires the Word desktop application</td>
<td>No</td>
</tr>
<tr class="odd">
<td>Citation-by-identifier</td>
<td>Yes</td>
<td>Yes</td>
<td>No</td>
<td>No</td>
<td>No</td>
<td>No</td>
</tr>
<tr class="even">
<td>Editing software</td>
<td>Any text editor</td>
<td>Web interface</td>
<td>Web interface</td>
<td>Web interface</td>
<td>Web interface</td>
<td>Any text editor</td>
</tr>
<tr class="odd">
<td>Document format</td>
<td>Markdown</td>
<td>HTML</td>
<td>LaTeX</td>
<td>Proprietary</td>
<td>Proprietary</td>
<td>Markdown</td>
</tr>
<tr class="even">
<td>Templating</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>No</td>
<td>No</td>
<td>No</td>
</tr>
<tr class="odd">
<td>Technical expertise required</td>
<td>Yes</td>
<td>No</td>
<td>Yes</td>
<td>No</td>
<td>No</td>
<td>Yes</td>
</tr>
<tr class="even">
<td>WYSIWYG mode</td>
<td>No</td>
<td>Yes</td>
<td>Rich text available</td>
<td>Yes</td>
<td>Yes</td>
<td>Preview rendered Markdown</td>
</tr>
<tr class="odd">
<td>Inline comments</td>
<td>Yes using Hypothesis</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>Yes</td>
<td>No</td>
</tr>
<tr class="even">
<td>Viewing changes</td>
<td>Diff of manuscript source</td>
<td>Highlight changes</td>
<td>Compare labeled versions</td>
<td>Highlight changes</td>
<td>No</td>
<td>Diff of manuscript source</td>
</tr>
</tbody>
</table>
</div>
<p>Existing platforms work well for editing text and are widely used for scholarly writing.
However, they often lack features that are important for open collaborative writing, such as versatile version control and multiple permission levels.
For example, Manubot is the only platform listed in Table <a href="#tbl:platforms">1</a> that offers the ability to address thematically related changes together and enables multiple authors to iteratively refine proposed changes.</p>
<h2 id="contribution-workflow">Manubot contribution workflow</h2>
<p>Manubot’s collaborative writing workflow adopts standard software development strategies that enable any contributor to edit any part of the manuscript but enforce discussion and review of all proposed changes.
The GitHub platform supports organizing and editing the manuscript.
Manubot projects use GitHub <em>issues</em> for organization, opening a new issue for each discussion topic.
For example, in a review manuscript like the Deep Review, this includes each primary paper under consideration.
Within a paper’s issue, contributors summarize the research, discuss it (sometimes with participation from the original authors), and assess its relevance to the review.
In a primary research article, issues can instead track progress on specific figures or subsections of text being drafted.
Issues serve as an open to-do list and a forum for debating the main messages of the manuscript.</p>
<p>GitHub and the underlying Git version control system <span class="citation" data-cites="PlcxShQU kEX5dgzK">[<a href="#ref-PlcxShQU" role="doc-biblioref">7</a>,<a href="#ref-kEX5dgzK" role="doc-biblioref">8</a>]</span> also structure the writing process.
The official version of the manuscript is <em>forked</em> by individual contributors, creating a copy they can freely modify.
A contributor then adds and revises files, grouping these changes into <em>commits</em>.
When the changes are ready to be reviewed, the series of commits are submitted as a <em>pull request</em> through GitHub, which notifies other authors of the pending changes.
GitHub’s review interface allows anyone to comment on the changes, globally or at specific lines, asking questions or requesting modifications <span class="citation" data-cites="opQBBK06">[<a href="#ref-opQBBK06" role="doc-biblioref">9</a>]</span>.
Conversations during review can reference other pull requests, issues, or authors, linking the relevant people and content (Figure <a href="#fig:workflow">1</a>).
Reviewing batches of revisions that focus on a single theme is more efficient than independently discussing isolated comments and edits and helps maintain consistent content and tone across different authors and reviewers.
Once all requested modifications are made, the manuscript maintainers, a subset of authors with elevated GitHub permissions, formally approve the pull request and merge the changes into the official version.
The process of writing and revising material can be orchestrated through GitHub with a web browser (as shown in <a href="#vid:usage">S1 Video</a>) or through a local text editor.</p>
<div id="fig:workflow" class="fignos">
<figure>
<img src="images/workflow.svg" class="white" style="width:5in" alt="" /><figcaption><span>Figure 1:</span> <strong>Manubot editing workflow.</strong>
Any reader can contribute to a Manubot manuscript by proposing a change through a pull request.
This example involves three people: a manuscript Maintainer, an existing project Contributor, and an additional Participant in the discussion.
Manuscript text is shown in solid lines on the left of the timeline and discussion on GitHub is shown by squiggly lines to the right of the timeline.
The Contributor opens a GitHub issue to discuss a manuscript modification.
The Maintainer and the Participant provide feedback in the issue, and the Maintainer recommends creating a GitHub pull request to update the text.
The Contributor creates the pull request.
It is reviewed by the Maintainer and the Participant, and the Contributor updates the pull request in response.
Once the pull request is approved, the Maintainer merges the changes into the official version of the manuscript.</figcaption>
</figure>
</div>
<!-- Google Drawing View Link https://docs.google.com/drawings/d/17mst0Z1RMXegeGhM6SwQSJ4nhHgMMh5csyV4MwhsgAY/edit?usp=sharing -->
<p>The Deep Review <a href="https://github.com/greenelab/deep-review/issues/575">issue</a> and <a href="https://github.com/greenelab/deep-review/pull/638">pull request</a> on protein-protein interactions demonstrate this process in practice.
A new contributor identified a relevant research topic that was missing from the review manuscript with examples of how the literature would be summarized, critiqued, and integrated into the review.
A maintainer confirmed that this was a desirable topic and referred to related open issues.
The contributor made the pull request, and two maintainers and another participant made recommendations.
After four rounds of reviews and pull request edits, a maintainer merged the changes.</p>
<p>We found that this workflow was an effective compromise between fully unrestricted editing and a more heavily-structured approach that limited the authors or the sections they could edit.
In addition, authors are associated with their commits, which makes it easy for contributors to receive credit for their work.
Figure <a href="#fig:contrib">2</a> and the GitHub <a href="https://github.com/greenelab/deep-review/graphs/contributors">contributors page</a> summarize all edits and commits from each author, providing aggregated information that is not available on most other collaborative writing platforms.
Because the Manubot writing process tracks the complete history through Git commits, it enables detailed retrospective contribution analysis.
These pull request and contribution tracking examples both come from Deep Review, the largest Manubot project to date, but illustrate the general principles of transparency and collaboration that are shared by all open Manubot manuscripts.</p>
<div id="fig:contrib" class="fignos">
<figure>
<img src="images/deep-review-contribution-ridge.svg" class="white" style="width:100.0%" alt="" /><figcaption><span>Figure 2:</span> <strong>Deep Review contributions by author over time.</strong>
The total words added to the Deep Review by each author is plotted over time (final values in parentheses).
These statistics were extracted from Git commit diffs of the manuscript’s Markdown source.
This figure reveals the composition of written contributions to the manuscript at every point in its history.
The Deep Review was initiated in August 2016, and the first complete manuscript was released as a preprint <span class="citation" data-cites="tJKvnIaZ">[<a href="#ref-tJKvnIaZ" role="doc-biblioref">10</a>]</span> in May 2017.
While the article was under review, we continued to maintain the project and accepted new contributions.
The preprint was updated in January 2018, and the article was accepted by the journal in March 2018 <span class="citation" data-cites="PZMP42Ak">[<a href="#ref-PZMP42Ak" role="doc-biblioref">5</a>]</span>.
As of March 06, 2019, the Deep Review repository accumulated 755 Git commits, 317 merged pull requests, 609 issues, and 819 GitHub stars.
The notebook to generate this figure can be <a href="https://mybinder.org/v2/gh/greenelab/meta-review/binder?filepath=analyses/deep-review-contrib/02.contrib-viz.ipynb">interactively launched using Binder</a> <span class="citation" data-cites="Q20Bxdsr">[<a href="#ref-Q20Bxdsr" role="doc-biblioref">11</a>]</span>, enabling users to explore alternative visualizations or analyses of the source data.</figcaption>
</figure>
</div>
<p>GitHub issues can also be used for formal peer review by independent or journal-selected reviewers.
A reviewer conducting open peer review can create issues using their own GitHub account, as <a href="https://github.com/greenelab/meta-review/issues/124">one reviewer</a> did for this manuscript.
Alternatively, a reviewer can post feedback with a pseudonymous GitHub account or have a trusted third party such as a journal editor post their comments anonymously.
Authors can elect to respond to reviews in the GitHub issues or a public <a href="https://github.com/greenelab/meta-review/blob/v3.0/content/response-to-reviewers.md">response letter</a>, creating open peer review.</p>
<p>Although we developed Manubot with collaborative writing in mind, it can also be helpful for individuals preparing scholarly documents.
Authors may choose to make their changes directly to the <code>master</code> branch, forgoing pull requests and reviews.
This workflow retains many of Manubot’s benefits, such as transparent history, automation, and allowing outside contributors to propose changes.
In cases where outside contributions are unwanted, authors can disable pull requests on GitHub.
It is also possible to use Manubot on a private GitHub repository.
Private manuscripts require some additional customization to disable GitHub Pages and may require a paid continuous integration plan.
See the <a href="#existing-manuscripts">existing manuscripts</a> for examples of the range of contribution workflows and Manubot use cases.</p>
<h2 id="manubot-features">Manubot features</h2>
<p>Manubot is a system for writing scholarly manuscripts via GitHub.
For each manuscript, there is a corresponding Git repository.
The <code>master</code> branch of the repository contains all of the necessary inputs to build the manuscript.
Specifically, a <code>content</code> directory contains one or more Markdown files that define the body of the manuscript as well as a metadata file to set information such as the title, authors, keywords, and language.
Figures can be hosted in the <code>content/images</code> subdirectory or elsewhere and specified by URL.
Repositories contain scripts and other files that define how to build and deploy the manuscript.
Many of these operations are delegated to the <code>manubot</code> <a href="https://github.com/manubot/manubot">Python package</a> or other dependencies such as Pandoc, which converts between document formats, and Travis CI, which builds the manuscript in the cloud.
Manubot pieces together many existing standards and technologies to encapsulate a manuscript in a repository and automatically generate outputs.</p>
<h3 id="markdown">Markdown</h3>
<p>With Manubot, manuscripts are written as plain-text Markdown files.
The <a href="https://spec.commonmark.org/0.28/">Markdown standard</a> itself provides limited yet crucial formatting syntax, including the ability to embed images and format text via bold, italics, hyperlinks, headers, inline code, codeblocks, blockquotes, and numbered or bulleted lists.
In addition, Manubot relies on extensions from <a href="https://pandoc.org/MANUAL.html#pandocs-markdown">Pandoc Markdown</a> to enable citations, tables, captions, and equations specified using the popular TeX math syntax.
Markdown with Pandoc extensions supports most formatting options required for scholarly writing <span class="citation" data-cites="17wKkS4DV">[<a href="#ref-17wKkS4DV" role="doc-biblioref">12</a>]</span> but currently lacks the ability to cross-reference and automatically number figures, tables, and equations.
For this functionality, Manubot includes the <a href="https://github.com/tomduck/pandoc-xnos"><code>pandoc-xnos</code></a> suite of Pandoc filters.
A list of formatting options officially supported by Manubot, at the time of writing, is viewable as <a href="https://github.com/manubot/rootstock/raw/091ca8d85c8ef2d7af16fcc8d2ed3ebcbc187f13/content/02.delete-me.md">raw Markdown</a> and the corresponding <a href="https://manubot.github.io/rootstock/v/091ca8d85c8ef2d7af16fcc8d2ed3ebcbc187f13/">rendered HTML</a>.</p>
<p>By virtue of its readable syntax, Markdown is well suited for version control using Git.
Markdown treats a single line break between text as a space and requires two-or-more consecutive line breaks to denote a new paragraph.
For optimal tracking of Markdown files with Git, we recommend placing each sentence on its own line.
This convention allows Git to display diffs on a per sentence basis, avoids unnecessary reflows associated with line wrapping, and supports easy rearrangement of sentences.</p>
<h3 id="citation-by-identifier">Citation-by-identifier</h3>
<p>Manubot includes an additional layer of citation processing, currently unique to the system.
All citations point to a standard identifier, for which Manubot automatically retrieves bibliographic metadata such as the title, authors, and publication date.
Table <a href="#tbl:citations">2</a> presents the supported identifiers and example citations before and after Manubot processing.
Authors can optionally define citation tags to provide short readable alternatives to the citation identifiers.
Citation metadata is exported to the <a href="http://citeproc-js.readthedocs.io/en/latest/csl-json/markup.html#items">Citation Style Language (CSL) JSON Data Items</a> format, an open standard that is widely supported by reference managers <span class="citation" data-cites="9KfVIq3s K7WVgf8X">[<a href="#ref-9KfVIq3s" role="doc-biblioref">13</a>,<a href="#ref-K7WVgf8X" role="doc-biblioref">14</a>]</span>.
However, sometimes external resources provide Manubot with invalid CSL Data, which can cause errors with downstream citation processors, such as <a href="http://hackage.haskell.org/package/pandoc-citeproc">pandoc-citeproc</a>.
Therefore, Manubot removes invalid fields according to the <a href="https://github.com/citation-style-language/schema">CSL Data specification</a>.
In cases where automatic retrieval of metadata fails or produces incorrect references — which is most common for URL citations — users can manually provide the correct metadata using common reference formats.
Manual metadata also supports references without standard identifiers, such as print-only newspaper articles.</p>
<div id="tbl:citations" class="tablenos">
<table>
<caption><span>Table 2:</span> <strong>Citation types supported by Manubot.</strong>
Manubot allows users to cite different types of persistent identifiers.
Metadata source indicates the primary resource used to retrieve bibliographic metadata.
For certain identifier types, additional metadata sources are queried should the primary fail.
For example, when translation-server ISBN lookup fails, Manubot tries Wikipedia’s <a href="https://www.mediawiki.org/wiki/Citoid">Citoid</a> service followed by the <a href="https://github.com/xlcnd/isbnlib">isbnlib</a> Python package.
When translation-server URL lookup fails, Manubot then tries <a href="http://greycite.knowledgeblog.org/">Greycite</a> <span class="citation" data-cites="GKPtRdAw">[<a href="#ref-GKPtRdAw" role="doc-biblioref">15</a>]</span>.
Raw citations enable citing works when no supported persistent identifiers exist, but require that the user specifies the metadata.
Finally, authors may optionally map a named tag to any of the supported identifier types.
In this example, the tag <code>avasthi-preprints</code> represents the DOI identifier <code>doi:10.7554/eLife.38532</code>.
API: application programming interface
</caption>
<colgroup>
<col style="width: 25%" />
<col style="width: 25%" />
<col style="width: 25%" />
<col style="width: 25%" />
</colgroup>
<thead>
<tr class="header">
<th>Identifier</th>
<th>Metadata source</th>
<th>Example citation</th>
<th>Processed citation</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Digital Object Identifier (DOI)</td>
<td>DOI <a href="https://citation.crosscite.org/docs.html">Content Negotiation</a></td>
<td><code>doi:10.1098/rsif.2017.0387</code></td>
<td><span class="citation" data-cites="PZMP42Ak">[<a href="#ref-PZMP42Ak" role="doc-biblioref">5</a>]</span></td>
</tr>
<tr class="even">
<td>shortDOI</td>
<td>DOI Proxy Server API</td>
<td><code>doi:10/gddkhn</code></td>
<td><span class="citation" data-cites="PZMP42Ak">[<a href="#ref-PZMP42Ak" role="doc-biblioref">5</a>]</span></td>
</tr>
<tr class="odd">
<td>PubMed Identifier (PMID)</td>
<td>NCBI <a href="https://www.ncbi.nlm.nih.gov/books/NBK25501/">E-utilities</a></td>
<td><code>pmid:25851694</code></td>
<td><span class="citation" data-cites="6rA99N5X">[<a href="#ref-6rA99N5X" role="doc-biblioref">16</a>]</span></td>
</tr>
<tr class="even">
<td>PubMed Central Identifier (PMCID)</td>
<td>NCBI <a href="https://api.ncbi.nlm.nih.gov/lit/ctxp/">Literature Citation Exporter</a></td>
<td><code>pmcid:PMC4719068</code></td>
<td><span class="citation" data-cites="ZPiRHuLc">[<a href="#ref-ZPiRHuLc" role="doc-biblioref">4</a>]</span></td>
</tr>
<tr class="odd">
<td>arXiv ID</td>
<td><a href="https://arxiv.org/help/api/index">arXiv API</a></td>
<td><code>arxiv:1502.04015v1</code></td>
<td><span class="citation" data-cites="Y2XyzLMc">[<a href="#ref-Y2XyzLMc" role="doc-biblioref">17</a>]</span></td>
</tr>
<tr class="even">
<td>International Standard Book Number (ISBN)</td>
<td><a href="https://github.com/zotero/translation-server">Zotero translation-server</a></td>
<td><code>isbn:9780262517638</code></td>
<td><span class="citation" data-cites="zBPP9YKu">[<a href="#ref-zBPP9YKu" role="doc-biblioref">18</a>]</span></td>
</tr>
<tr class="odd">
<td>Web address (URL)</td>
<td>Zotero translation-server</td>
<td><code>url:https://lgatto.github.io/open-and-open/</code></td>
<td><span class="citation" data-cites="URixREeW">[<a href="#ref-URixREeW" role="doc-biblioref">19</a>]</span></td>
</tr>
<tr class="even">
<td>Wikidata ID</td>
<td>Zotero translation-server</td>
<td><code>wikidata:Q56458321</code></td>
<td><span class="citation" data-cites="QhC8yJ7V">[<a href="#ref-QhC8yJ7V" role="doc-biblioref">20</a>]</span></td>
</tr>
<tr class="odd">
<td>Raw</td>
<td>Provided by user</td>
<td><code>raw:paywall-movie</code></td>
<td><span class="citation" data-cites="J6sGnQp1">[<a href="#ref-J6sGnQp1" role="doc-biblioref">21</a>]</span></td>
</tr>
<tr class="even">
<td>Tag</td>
<td>Source for tagged identifier</td>
<td><code>tag:avasthi-preprints</code></td>
<td><span class="citation" data-cites="pqBLIXzp">[<a href="#ref-pqBLIXzp" role="doc-biblioref">22</a>]</span></td>
</tr>
</tbody>
</table>
</div>
<p>Manubot formats bibliographies according to a <a href="http://citationstyles.org/">CSL</a> style specification.
Styles define how references are constructed from bibliographic metadata, controlling layout details such as the maximum number of authors to list per reference.
Manubot’s default style emphasizes titles and electronic (rather than print) identifiers and applies numeric-style citations <span class="citation" data-cites="1bGpMqiw">[<a href="#ref-1bGpMqiw" role="doc-biblioref">23</a>]</span>.
Alternatively, users can also choose from thousands of <a href="http://editor.citationstyles.org/searchByName/">predefined styles</a> or build their own <span class="citation" data-cites="w4n6Qtun">[<a href="#ref-w4n6Qtun" role="doc-biblioref">24</a>]</span>.
As a result, adopting the specific bibliographic format required by a journal usually just requires specifying the style’s source URL in the Manubot configuration.</p>
<h3 id="format-conversion">Format conversion</h3>
<p>Manubot uses <a href="https://pandoc.org/">Pandoc</a> to convert manuscripts from Markdown to HTML, PDF, and optionally DOCX outputs.
Pandoc also supports <a href="https://jats.nlm.nih.gov/">Journal Article Tag Suite</a> (JATS), a standard format for scholarly articles that is used by publishers, archives, and text miners <span class="citation" data-cites="LHrRxRb0 AAwqxolU bCyfIm6z">[<a href="#ref-LHrRxRb0" role="doc-biblioref">25</a>,<a href="#ref-AAwqxolU" role="doc-biblioref">26</a>,<a href="#ref-bCyfIm6z" role="doc-biblioref">27</a>]</span>.
Pandoc’s JATS support provides an avenue to integrate Manubot with the larger JATS ecosystem.
In the future, journals may accept submissions in JATS.
For now, Manubot’s DOCX output is usually sufficient for journal submissions that require an editable source document.
Otherwise, authors generally use the PDF output for preprint and initial journal submissions.
The primary Manubot output is HTML intended to be viewed in a web browser.
Accordingly, manuscripts natively support JavaScript and can thus include any web-based interactive visualization, such as those produced using <a href="https://vega.github.io/vega-lite/">Vega-Lite</a>, <a href="https://bokeh.pydata.org">Bokeh</a>, or <a href="https://plot.ly/">Plotly</a> <span class="citation" data-cites="6eVHYGML nyZChH5b">[<a href="#ref-6eVHYGML" role="doc-biblioref">28</a>,<a href="#ref-nyZChH5b" role="doc-biblioref">29</a>]</span>.</p>
<h3 id="interactive-features-and-appearance">Interactive features and appearance</h3>
<p>Manubot comes with several “plugins” that can be included in manuscripts exported as HTML.
These plugins add special interactive features that enhance the user experience of viewing and reading manuscripts (Figure <a href="#fig:plugins">3</a>).
For example, with the “tooltips” plugin enabled, when the user hovers over a link to a reference or figure, a preview of that item pops up above the link, along with controls to navigate between other mentions of that item elsewhere in the document.
The build process can also accommodate different “themes”, which change the general aesthetics and appearance of the exported document (e.g. from a contemporary sans-serif style to a more traditional serif style).
The architecture of the plugins and themes is designed to provide authors with enough flexibility to suit their particular needs and preferences.</p>
<div id="fig:plugins" class="fignos">
<figure>
<img src="images/plugins.png" alt="" /><figcaption><span>Figure 3:</span> <strong>Examples of the various Manubot plugins, illustrating their functionality and usefulness.</strong>
Screenshots were taken from existing manuscripts made with Manubot: <a href="https://greenelab.github.io/scihub-manuscript/v/fd7acb7ed0108c920da56f84819ce13f02f68aa8/">Sci-Hub Coverage Study</a> and <a href="https://trang1618.github.io/tpot-fss-ms/">TPOT-FSS</a>, available under the <a href="https://creativecommons.org/licenses/by/4.0/" title="Creative Commons Attribution 4.0 International License">CC BY 4.0 License</a>.
Clarifying markups are overlaid in purple.</figcaption>
</figure>
</div>
<p>The Manubot “front-end” (layout, look, controls, behavior, etc.) was developed in line with current best practices and user expectations of the modern web.
The plugins use standard technology built in to most major web browsers, allowing them to be relatively lightweight, modular, and easy to configure.</p>
<h3 id="continuous-publication">Continuous publication</h3>
<p>Manubot performs continuous publication:
Every update to a manuscript’s source is automatically reflected in the online outputs.
The approach uses continuous integration (CI) <span class="citation" data-cites="18w6XKsQO Qh7xTLwz lXvpQxeN">[<a href="#ref-18w6XKsQO" role="doc-biblioref">30</a>,<a href="#ref-Qh7xTLwz" role="doc-biblioref">31</a>,<a href="#ref-lXvpQxeN" role="doc-biblioref">32</a>]</span>, specifically via <a href="https://travis-ci.org/">Travis CI</a>, to monitor changes.
When changes occur, the CI service attempts to generate an updated manuscript.
If this process is error free, the CI service timestamps the manuscript and uploads the output files to the GitHub repository.
Because the HTML manuscript is hosted using <a href="https://pages.github.com/">GitHub Pages</a>, the CI service automatically deploys the new manuscript version when it pushes the updated outputs to GitHub.
Using CI to build the manuscript automatically catches many common errors, such as misspelled citations, invalid formatting, or misconfigured software dependencies.</p>
<p>To illustrate, the source GitHub repository for this article is <a href="https://github.com/greenelab/meta-review" class="uri">https://github.com/greenelab/meta-review</a>.
When this repository changes, Travis CI <a href="https://travis-ci.org/greenelab/meta-review">rebuilds</a> the manuscript.
If successful, the output is deployed back to GitHub (to dedicated <a href="https://github.com/greenelab/meta-review/tree/output"><code>output</code></a> and <a href="https://github.com/greenelab/meta-review/tree/gh-pages"><code>gh-pages</code></a> branches).
As a result, <a href="https://greenelab.github.io/meta-review" class="uri">https://greenelab.github.io/meta-review</a> stays up to date with the latest HTML manuscript.
Furthermore, versioned URLs, such as <a href="https://greenelab.github.io/meta-review/v/4b6396bcefd1b9c7ddf39c1d3f0b3eab2dd63f31/" class="uri">https://greenelab.github.io/meta-review/v/4b6396bcefd1b9c7ddf39c1d3f0b3eab2dd63f31/</a>, provide access to previous manuscript versions.</p>
<h3 id="timestamping">Timestamping</h3>
<p>The idea of the “priority of discovery” is important to science, and Vale and Hyman discuss the importance of both disclosure and validation <span class="citation" data-cites="vHuGhm4k">[<a href="#ref-vHuGhm4k" role="doc-biblioref">33</a>]</span>.
In their framework, disclosure occurs when a scientific output is released to the world.
However, for a manuscript that is shared as it is written, being able to establish priority could be challenging.
Manubot supports <a href="https://opentimestamps.org/">OpenTimestamps</a> to timestamp the HTML and PDF outputs on the Bitcoin blockchain.
This procedure allows one to retrospectively prove that a manuscript version existed prior to its blockchain-verifiable timestamp <span class="citation" data-cites="Y2XyzLMc 2K7yQEXT hkguO3lG qh60RjR0 Y4trfW9i">[<a href="#ref-Y2XyzLMc" role="doc-biblioref">17</a>,<a href="#ref-2K7yQEXT" role="doc-biblioref">34</a>,<a href="#ref-hkguO3lG" role="doc-biblioref">35</a>,<a href="#ref-qh60RjR0" role="doc-biblioref">36</a>,<a href="#ref-Y4trfW9i" role="doc-biblioref">37</a>]</span>.
Timestamps protect against attempts to rewrite a manuscript’s history and ensure accurate histories, potentially alleviating certain authorship or priority disputes.
Because all Bitcoin transactions compete for limited space on the blockchain, the fees required to send a single transaction can be high.
OpenTimestamps minimizes fees by encoding many timestamps into a single Bitcoin transaction, enabling the service to be free of charge <span class="citation" data-cites="1DG704X8Q">[<a href="#ref-1DG704X8Q" role="doc-biblioref">38</a>]</span>.
Since transactions can take up to a few days to be made, Manubot initially stores incomplete timestamps and upgrades them in future continuous deployment builds.
We find that this asynchronous design with timestamps precise to the day is suitable for the purposes of scientific writing.</p>
<h3 id="reproducible-manuscripts">Reproducible manuscripts</h3>
<p>Manubot and its dependencies are free of charge and largely open source.
It does rely on gratis services from two proprietary platforms: GitHub and Travis CI.
Fortunately, lock-in to these services is minimal, and <a href="#future">several substitutes</a> already exist.
Manubot provides a substantial step towards end-to-end document reproducibility, where every figure or piece of data in a manuscript can be traced back to its origin <span class="citation" data-cites="sWD9uVuF">[<a href="#ref-sWD9uVuF" role="doc-biblioref">39</a>]</span> and is well-suited for preserving provenance.
For example, figures can be specified using versioned URLs that refer to the code that created them.
In addition, manuscripts can be templated, so that numerical values or tables are inserted directly from the repository that created them.
The Figure <a href="#fig:contrib">2</a> caption provides examples of templates.
Phrases such as “755 Git commits” are written as <code>{{total_commits}} Git commits</code> so that the commit count can be automatically updated.</p>
<h3 id="getting-started">Getting started</h3>
<p>An example repository at <a href="https://github.com/manubot/rootstock" class="uri">https://github.com/manubot/rootstock</a>, referred to as Rootstock, demonstrates Manubot’s features and serves as a template for users to write their own manuscripts with Manubot.
The <a href="https://github.com/manubot/rootstock/blob/091ca8d85c8ef2d7af16fcc8d2ed3ebcbc187f13/SETUP.md">current setup process</a> includes cloning the Rootstock repository, rebranding it to the user’s manuscript, and configuring continuous integration.
The setup process is complex but must only be performed once per manuscript.
Incorporating new Manubot features into an existing manuscript is also possible by pulling the latest commits from Rootstock, which sometimes involves resolving Git conflicts.</p>
<p>Contributing to a manuscript is less technical and can be performed entirely through GitHub’s web interface, as discussed in the <a href="#contribution-workflow">contribution workflow</a> section and demonstrated in <a href="#vid:usage">S1 Video</a>.
Interested readers can practice editing a demo manuscript at <a href="https://github.com/manubot/try-manubot" class="uri">https://github.com/manubot/try-manubot</a>.</p>
<p>At the 2019 <em>Pacific Symposium on Biocomputing</em>, we led a working group where 17 conference participants contributed to a different <a href="https://git.dhimmel.com/psb-manuscript/">demo manuscript</a>.
Based on this experience, we believe most computational scholars have the expertise to contribute to a Manubot manuscript.
Proficiency with Manubot requires familiarity with Markdown, Git, GitHub, and continuous integration.
While these tools do present a barrier to entry, they are also highly applicable outside of Manubot and increasingly part of the standard curriculum for computational scholars.
For example, Markdown is used for documenting Jupyter and R Markdown notebooks.</p>
<h3 id="existing-manuscripts">Existing manuscripts</h3>
<p>Since its creation to facilitate the Deep Review, Manubot has been used to write a variety of scholarly documents.
The <a href="https://github.com/greenelab/scihub-manuscript">Sci-Hub Coverage Study</a> — performed openly on GitHub from its inception — investigated Sci-Hub’s repository of pirated articles <span class="citation" data-cites="IhliSZDo">[<a href="#ref-IhliSZDo" role="doc-biblioref">40</a>]</span>.
Sci-Hub <a href="https://github.com/greenelab/scihub-manuscript/issues/17">reviewed</a> the initial preprint from this study in a series of tweets, pointing out a major error in one of the analyses.
Within hours, the authors used Markdown’s strikethrough formatting in Manubot to cross-out the errant sentences (<a href="https://github.com/greenelab/scihub-manuscript/commit/8fcd0cd665f6fb5f39bed7e26b940aa27d4770ba">commit</a>, <a href="https://greenelab.github.io/scihub-manuscript/v/8fcd0cd665f6fb5f39bed7e26b940aa27d4770ba/">versioned manuscript</a>), thereby alerting readers to the mistake and preventing further propagation of misinformation.
One month later, a larger <a href="https://github.com/greenelab/scihub-manuscript/pull/19">set of revisions</a> explained the error in more detail and was included in a second version of the preprint.
As such, continuous publication via Manubot helped the authors address the error without delay, while retaining a public version history of the process.
This Sci-Hub Coverage Study preprint was the <a href="http://web.archive.org/web/20171221221858/http://www.prepubmed.org/top_preprints/">most viewed</a> 2017 <em>PeerJ Preprint</em>, while the Deep Review was the most viewed 2017 <em>bioRxiv</em> preprint <span class="citation" data-cites="9IrsqXRa">[<a href="#ref-9IrsqXRa" role="doc-biblioref">41</a>]</span>.
Hence, in Manubot’s first year, two of the most popular preprints were written using its collaborative, open, and review-driven authoring process.</p>
<p>Additional research studies are being authored using Manubot, spanning the fields of
<a href="https://vsmalladi.github.io/tfsee-manuscript/">regulatory</a>
<a href="https://simonvh.github.io/gimmemotifs-manuscript/">genomics</a> <span class="citation" data-cites="LGjXBQ7t">[<a href="#ref-LGjXBQ7t" role="doc-biblioref">42</a>]</span>,
<a href="https://zach-hensel.github.io/low-noise-manuscript/">synthetic biology</a> <span class="citation" data-cites="15QSXk9Cl">[<a href="#ref-15QSXk9Cl" role="doc-biblioref">43</a>]</span>,
<a href="https://openclimatedata.github.io/global-emissions/">climate science</a>,
<a href="https://laurentperrinet.github.io/2019-05_illusions-visuelles/">visual perception</a> <span class="citation" data-cites="faKuBkfY">[<a href="#ref-faKuBkfY" role="doc-biblioref">44</a>]</span>,
<a href="https://trang1618.github.io/tpot-fss-ms/">machine learning</a> <span class="citation" data-cites="17a3WKGzu">[<a href="#ref-17a3WKGzu" role="doc-biblioref">45</a>]</span>,
<a href="https://jmonlong.github.io/manu-vgsv/">computational toolkits</a> <span class="citation" data-cites="r2krHy6V">[<a href="#ref-r2krHy6V" role="doc-biblioref">46</a>]</span>,
and <a href="https://yt-project.github.io/yt-3.0-paper/">data visualization</a>.
Manubot is also being used for documents beyond traditional journal publications, such as
<a href="https://benjamin-lee.github.io/deep-rules/">research tips</a>,
<a href="https://indigo-dc.github.io/sqa-baseline/">quality standards</a> <span class="citation" data-cites="WkeOa3Qo">[<a href="#ref-WkeOa3Qo" role="doc-biblioref">47</a>]</span>,
<a href="https://greenelab.github.io/manufund-2018/">grant proposals</a>,
<a href="https://greenelab.github.io/czi-hca-report/">progress reports</a>,
<a href="https://zietzm.github.io/Vagelos2017/">undergraduate research reports</a> <span class="citation" data-cites="15nwuvjrA">[<a href="#ref-15nwuvjrA" role="doc-biblioref">48</a>]</span>,
<a href="https://slochower.github.io/synthetic-motor-literature/">literature reviews</a>, and lab notebooks.
Finally, manuscripts written with other authoring systems have been successfully ported to Manubot, including the <a href="https://git.dhimmel.com/bitcoin-whitepaper/">Bitcoin Whitepaper</a> <span class="citation" data-cites="u9DGTIX">[<a href="#ref-u9DGTIX" role="doc-biblioref">49</a>]</span> and <a href="https://git.dhimmel.com/rephetio-manuscript/">Project Rephetio manuscript</a> <span class="citation" data-cites="O21tn8vf">[<a href="#ref-O21tn8vf" role="doc-biblioref">50</a>]</span>.</p>
<h3 id="citation-utilities">Citation utilities</h3>
<p>The <code>manubot</code> <a href="https://pypi.org/project/manubot/">Python package</a> provides easy access to Manubot’s citation-by-identifier infrastructure, whose functionality extends beyond just Manubot manuscripts.
For example, the <a href="https://kipoi.org/">Kipoi</a> model zoo for genomics <span class="citation" data-cites="tQy0rfF4">[<a href="#ref-tQy0rfF4" role="doc-biblioref">51</a>]</span> uses Manubot’s Python interface to retrieve model authors from persistent identifiers.
In addition, the <code>manubot cite</code> command line utility takes a list of citations and returns either a rendered bibliography or CSL Data Items (i.e. JSON-formatted reference metadata).
For example, the following command outputs a Markdown reference list for the two specified articles according to the bibliographic style of <em>PeerJ</em>:</p>
<div class="sourceCode" id="cb1"><pre class="sourceCode sh"><code class="sourceCode bash"><span id="cb1-1"><a href="#cb1-1"></a><span class="ex">manubot</span> cite --render --format=markdown \</span>
<span id="cb1-2"><a href="#cb1-2"></a>  --csl=https://github.com/citation-style-language/styles/raw/master/peerj.csl \</span>
<span id="cb1-3"><a href="#cb1-3"></a>  pmid:29618526 doi:10.1038/550143a</span></code></pre></div>
<p>Pandoc brands itself as a “universal document converter”, and can convert from any of 32 input formats to any of 51 output formats as of version 2.7.
Thanks to its versatility and active development since 2006, Pandoc enjoys a large userbase across many disciplines and applications.
Its filter interface enables adding custom functionality with community-developed programs.
We are <a href="https://github.com/manubot/manubot/pull/99" title="Create a pandoc-manubot-cite filter for pandoc">prototyping</a> a Manubot-based citation-by-identifier filter.
This filter would allow Pandoc users to cite persistent identifiers as part of their existing Pandoc workflows, without requiring them to adopt other aspects of Manubot.
It could help popularize citation-by-identifier at an influential scale.</p>
<h3 id="future">Future enhancements</h3>
<p>Manubot is still under active development, and we envision major changes in its design and dependencies going forward.
Currently, manuscript repositories must contain a large number of files that do not directly contain manuscript content.
While this enables a high-degree of customization, it also increases complexity.
Therefore, we are investigating whether configuration files with sensible defaults could enable bare-bones repositories that contain manuscript content and little else.</p>
<p>In addition to simplifying the usage, we’re also looking into simplifying the setup.
Presently, setup is complex because users must do advanced command-line operations to clone the Rootstock repository and configure Travis CI.
Although we provide detailed instructions, users often struggle to replicate the long list of setup commands in an appropriate computational environment.
One priority will be to automate setup to a higher degree.
However, this may require switching the services Manubot uses for continuous integration (e.g. from Travis CI to GitHub Actions, CircleCI, Drone, or GitLab CI), environment management (e.g. from Conda to Docker), and repository hosting (e.g. from GitHub to GitLab).
In addition to simplifying setup, such migrations may also present the opportunity to decrease dependency on proprietary services and address other Manubot shortcomings, such as the current inability to view rendered manuscripts produced by pull request builds.</p>
<p>Upgrading a Manubot instance is an opt-in procedure.
Therefore, when we introduce fundamental changes, existing manuscripts continue to function.
However, large Rootstock changes can make upgrading existing manuscripts difficult.
We are happy to provide users <em>pro bono</em> assistance to upgrade or troubleshoot manuscripts.
Users can open an <a href="https://github.com/manubot/rootstock/issues">issue</a> at the Rootstock repository for help.</p>
<p>One strategy to grow Manubot usage is to identify a specific user group or use case for which Manubot can be widely adopted.
For example, a journal may decide to build their publishing workflow around Manubot, such that submissions would consist of a Manubot repository.
This application would be most suitable for journals that currently use GitHub for submissions and publishing, such as the <em>Journal of Open Source Software</em> <span class="citation" data-cites="xpw2aizK">[<a href="#ref-xpw2aizK" role="doc-biblioref">52</a>]</span>.
Manubot could also gain traction as the primary tool used to write collaborative manuscripts within certain communities.
For example, open research projects built from voluntary contributions by geographically-distributed individuals could adopt Manubot.
Likewise, Manubot may excel at enabling collaborative translation of existing manuscripts into other languages.
Another application could be collaborative development of online lessons, documentation, or tutorials.
Projects like <a href="https://software-carpentry.org/">Software Carpentry</a> already host each lesson in a separate GitHub repository and may benefit from Manubot-generated permalinks to historical versions.</p>
<h2 id="authorship">Authorship</h2>
<p>Manubot does not impose any restrictions on authorship.
It allows authors to adhere to the author inclusion and ordering conventions of their field, which vary considerably across disciplines <span class="citation" data-cites="14ahyJcvY">[<a href="#ref-14ahyJcvY" role="doc-biblioref">53</a>]</span>.
Some Manubot projects create a <a href="https://github.com/Benjamin-Lee/deep-rules/blob/cfb7f744573ca0532a19ca1a8e9473a555cf8eb2/contributors.md">table</a> in their GitHub repository to track contributors who did not commit text to the manuscript.
This provides a transparent way to record contributions such as experimental research that generated data for the manuscript and discuss whether they meet that project’s authorship criteria.
Contribution transparency helps prevent ghostwriting <span class="citation" data-cites="RK9sIADd">[<a href="#ref-RK9sIADd" role="doc-biblioref">54</a>]</span> and is especially important in collaborative writing <span class="citation" data-cites="vzcNEQll">[<a href="#ref-vzcNEQll" role="doc-biblioref">55</a>]</span>.
Although we recommend authors provide their ORCID and GitHub username, Manubot also supports pseudonyms, pseudonymous GitHub usernames, and authors without an ORCID or GitHub account.</p>
<p>To determine authorship for the Deep Review, we followed the International Committee of Medical Journal Editors (ICMJE) <a href="http://www.icmje.org/recommendations/browse/roles-and-responsibilities/defining-the-role-of-authors-and-contributors.html">guidelines</a> and used GitHub to track contributions.
ICMJE recommends authors substantially contribute to, draft, approve, and agree to be accountable for the manuscript.
We acknowledged other contributors who did not meet all four criteria, including contributors who provided text but did not review and approve the complete manuscript.
Although these criteria provided a straightforward, equitable way to determine who would be an author, they did not produce a traditionally ordered author list.
In biomedical journals, the convention is that the first and last authors made the most substantial contributions to the manuscript.
This convention can be difficult to reconcile in a collaborative effort.
Using Git, we could quantify the number of commits each author made or the number of sentences an author wrote or edited, but these metrics discount intellectual contributions such as discussing primary literature and reviewing pull requests.
Therefore, we concluded that it is not possible to construct an objective system to compare and weight the different types of contributions and produce an ordered author list <span class="citation" data-cites="e2rpsIbt">[<a href="#ref-e2rpsIbt" role="doc-biblioref">56</a>]</span>.</p>
<p>To address this issue, we generalized the concept of “co-first” authorship, in which two or more authors are denoted as making equal contributions to a paper.
We defined four types of contributions <span class="citation" data-cites="PZMP42Ak">[<a href="#ref-PZMP42Ak" role="doc-biblioref">5</a>]</span>, from major to minor, and reviewed the GitHub discussions and commits to assign authors to these categories.
A randomized algorithm then arbitrarily ordered authors within each contribution category, and we combined the category-specific author lists to produce a traditional ordering.
The randomization procedure was shared with the authors in advance (pre-registered) and run in a deterministic manner.
Given the same author contributions, it always produced the same ordered author list.
We annotated the author list to indicate that author order was partly randomized and emphasize that the order did not indicate one author contributed more than another from the same category.
The Deep Review author ordering procedure illustrates authorship possibilities when all contributions are publicly tracked and recorded that would be difficult with a traditional collaborative writing platform.</p>
<p>Papers with hundreds or thousands of authors are on the rise, such as the article describing the experiments and data that led to the discovery of the Higgs Boson <span class="citation" data-cites="15YfJWkEd">[<a href="#ref-15YfJWkEd" role="doc-biblioref">57</a>]</span> (5000 authors) and the report of the <em>Drosophila</em> genome <span class="citation" data-cites="Szy8sdWq">[<a href="#ref-Szy8sdWq" role="doc-biblioref">58</a>]</span> (1000 authors).
Yet the number of people that participated in writing those papers, as opposed to generating and analyzing the data, is not always clear and is likely to be far below the number of authors <span class="citation" data-cites="NGOit8L2 nahdPyLb">[<a href="#ref-NGOit8L2" role="doc-biblioref">59</a>,<a href="#ref-nahdPyLb" role="doc-biblioref">60</a>]</span>.
Manubot provides the scientists involved in large collaborations the opportunity to actively participate, through a public forum, in the writing process.</p>
<h2 id="discussion">Discussion</h2>
<h3 id="collaborative-review-manuscripts">Collaborative review manuscripts</h3>
<p>The open scholarly writing Manubot enables has particular benefits for review articles, which present the state of the art in a scientific field <span class="citation" data-cites="Rhm4AK0j">[<a href="#ref-Rhm4AK0j" role="doc-biblioref">61</a>]</span>.
Literature reviews are typically written in private by an invited team of colleagues.
In contrast, broadly opening the process to anyone engaged in the topic — such that planning, organizing, writing, and editing occur collaboratively in a public forum where anyone is welcome to participate — can maximize a review’s value.
Open drafting of reviews is especially helpful for capturing state-of-the-art knowledge about rapidly advancing research topics at the intersection of existing disciplines where contributors bring diverse opinions and expertise.</p>
<p>Writing review articles in a public forum allows review authors to engage with the original researchers to clarify their methods and results and present them accurately, as exemplified <a href="https://github.com/greenelab/deep-review/issues/213">here</a>.
Additionally, discussing manuscripts in the open generates valuable pre-publication peer review of preprints <span class="citation" data-cites="pqBLIXzp">[<a href="#ref-pqBLIXzp" role="doc-biblioref">22</a>]</span> or post-publication peer review <span class="citation" data-cites="6rA99N5X jYs2OUFW HflJ6Hy5">[<a href="#ref-6rA99N5X" role="doc-biblioref">16</a>,<a href="#ref-jYs2OUFW" role="doc-biblioref">62</a>,<a href="#ref-HflJ6Hy5" role="doc-biblioref">63</a>]</span>.
Because incentives to provide public peer review of existing literature <span class="citation" data-cites="uw5bep8P">[<a href="#ref-uw5bep8P" role="doc-biblioref">64</a>]</span> are lacking, open collaborative reviews — where authorship is open to anyone who makes a valid contribution — could help spur more post-publication peer review.</p>
<h3 id="additional-collaborative-writing-projects">Additional collaborative writing projects</h3>
<p>The Deep Review was not the first scholarly manuscript written online via an open collaborative process.
This type of manuscript is also known as a Massively Open Online Paper <span class="citation" data-cites="16H8Gdlkx">[<a href="#ref-16H8Gdlkx" role="doc-biblioref">65</a>]</span>.
In 2013, two dozen mathematicians created the 600-page Homotopy Type Theory book, writing collaboratively in LaTeX on GitHub <span class="citation" data-cites="19ly8T0jZ SbkGi9m1">[<a href="#ref-19ly8T0jZ" role="doc-biblioref">66</a>,<a href="#ref-SbkGi9m1" role="doc-biblioref">67</a>]</span>.
Two technical books on cryptocurrency — <a href="https://github.com/bitcoinbook/bitcoinbook">Mastering Bitcoin</a> and <a href="https://github.com/ethereumbook/ethereumbook">Mastering Ethereum</a> — written on GitHub in AsciiDoc format have engaged hundreds of contributors.
Both Homotopy Type Theory and Mastering Bitcoin continue to be maintained years after their initial publication.
A 2017 perspective on the future of peer review was written collaboratively on Overleaf, with contributions from 32 authors <span class="citation" data-cites="52Q1v5nS">[<a href="#ref-52Q1v5nS" role="doc-biblioref">68</a>]</span>.
While debate was raging over tightening the default threshold for statistical significance, nearly 150 scientists contributed to a Google Doc discussion that was condensed into a traditional journal commentary <span class="citation" data-cites="82ZjWq3i HXpnCQu1">[<a href="#ref-82ZjWq3i" role="doc-biblioref">69</a>,<a href="#ref-HXpnCQu1" role="doc-biblioref">70</a>]</span>.
The greatest success to date of open collaborative writing is arguably Wikipedia, whose English version contains over 5.5 million articles.
Wikipedia scaled encyclopedias <a href="https://en.wikipedia.org/wiki/Wikipedia:Size_comparisons">far beyond</a> any privately-written alternative.
These examples illustrate how open collaborative writing can scale scholarly manuscripts where diverse opinion and expertise are paramount beyond what would otherwise be possible.</p>
<p>Open writing also presents new opportunities for distributing scholarly communication.
Though it is still valuable to have versioned drafts of a manuscript with digital identifiers, journal publication may not be the terminal endpoint for collaborative manuscripts.
After releasing the first version of the Deep Review <span class="citation" data-cites="tJKvnIaZ">[<a href="#ref-tJKvnIaZ" role="doc-biblioref">10</a>]</span>, 14 new contributors updated the manuscript (Figure <a href="#fig:contrib">2</a>).
Existing authors continue to discuss new literature, <a href="https://github.com/greenelab/deep-review/">creating a living document</a>.
Manubot provides an ideal platform for perpetual reviews <span class="citation" data-cites="Xs2yPQcr H0XkaC8S">[<a href="#ref-Xs2yPQcr" role="doc-biblioref">71</a>,<a href="#ref-H0XkaC8S" role="doc-biblioref">72</a>]</span>.</p>
<p>Concepts for the future of scholarly publishing extend beyond collaborative writing <span class="citation" data-cites="Bu8VxdWB WDvu1SAV ILhLpgTs">[<a href="#ref-Bu8VxdWB" role="doc-biblioref">73</a>,<a href="#ref-WDvu1SAV" role="doc-biblioref">74</a>,<a href="#ref-ILhLpgTs" role="doc-biblioref">75</a>]</span>.
Pandoc Scholar <span class="citation" data-cites="17wKkS4DV">[<a href="#ref-17wKkS4DV" role="doc-biblioref">12</a>]</span> and Bookdown <span class="citation" data-cites="1BISa1RLr">[<a href="#ref-1BISa1RLr" role="doc-biblioref">76</a>]</span>, which has been used for open writing <span class="citation" data-cites="1CblcIDkq">[<a href="#ref-1CblcIDkq" role="doc-biblioref">77</a>]</span>, both enhance traditional Markdown to better support publishing.
<a href="https://dokie.li/">Dokieli</a> is a clientside editor for HTML articles, which aims for decentralization by adhering to web standards that allow articles and reader annotations to be stored by any participating server <span class="citation" data-cites="q41cceRt">[<a href="#ref-q41cceRt" role="doc-biblioref">78</a>]</span>.
<a href="https://substance.io/texture/">Texture</a> is also a browser-based editor, but uses JATS as the primary storage format.</p>
<p>Several projects in addition to Manubot provide infrastructure for citation-by-identifier.
For example, the <a href="https://github.com/cboettig/knitcitations">knitcitations</a> package enables citation by DOI or URL in R Markdown documents.
Zotero has developed bibliographic metadata extractors, called <a href="https://github.com/zotero/translators/">translators</a>, for over 500 resources.
<a href="https://citation.js.org/">Citation.js</a> converts bibliographic references or identifiers in a variety of formats to CSL JSON <span class="citation" data-cites="1CNWD7ZF">[<a href="#ref-1CNWD7ZF" role="doc-biblioref">79</a>]</span>,
and is used by <a href="http://wikicite.org/wcite/#filter-pwcite">pwcite</a>, a Pandoc filter that enables citing Wikidata IDs.</p>
<p>Examples of continuous integration to automate manuscript generation include <a href="https://github.com/ewanmellor/gh-publisher">gh-publisher</a> and <a href="https://github.com/mfenner/jekyll-travis">jekyll-travis</a>, which was used to produce a <a href="http://book.openingscience.org/">continuously published webpage</a> for the Opening Science book <span class="citation" data-cites="2JcGGG7l ujrfOBM1">[<a href="#ref-2JcGGG7l" role="doc-biblioref">80</a>,<a href="#ref-ujrfOBM1" role="doc-biblioref">81</a>]</span>.
Binder <span class="citation" data-cites="Q20Bxdsr">[<a href="#ref-Q20Bxdsr" role="doc-biblioref">11</a>]</span>, Distill journal articles <span class="citation" data-cites="MHNCSD5I">[<a href="#ref-MHNCSD5I" role="doc-biblioref">82</a>]</span>, Idyll <span class="citation" data-cites="1ESo5MNnB">[<a href="#ref-1ESo5MNnB" role="doc-biblioref">83</a>]</span>, and Stencila <span class="citation" data-cites="BWMf57EM KLKZcPlg">[<a href="#ref-BWMf57EM" role="doc-biblioref">84</a>,<a href="#ref-KLKZcPlg" role="doc-biblioref">85</a>]</span> support manuscripts with interactive graphics and close integration with the underlying code.
As an open source project, Manubot can be extended to adopt best practices from these other emerging platforms.</p>
<p>Several other open science efforts are GitHub-based like our collaborative writing process.
ReScience <span class="citation" data-cites="8o1nWux7">[<a href="#ref-8o1nWux7" role="doc-biblioref">86</a>]</span> as well as titles from <a href="http://www.theoj.org/">Open Journals</a>, such as the <em>Journal of Open Source Software</em> <span class="citation" data-cites="xpw2aizK">[<a href="#ref-xpw2aizK" role="doc-biblioref">52</a>]</span>, rely on GitHub for peer review and hosting.
Distill uses GitHub for transparent peer review and post-publication peer review <span class="citation" data-cites="1ESYVbN4H">[<a href="#ref-1ESYVbN4H" role="doc-biblioref">87</a>]</span>.
GitHub is increasingly used for resource curation <span class="citation" data-cites="vf9t7xMG">[<a href="#ref-vf9t7xMG" role="doc-biblioref">88</a>]</span>, and collaborative scholarly reviews combine literature curation with discussion and interpretation.</p>
<h3 id="limitations">Limitations</h3>
<p>There are potential limitations of our GitHub-based approach.
Because the Deep Review pertained to a computational topic, most of the authors had computational backgrounds, including previous experience with version control workflows and GitHub.
In other disciplines, collaborative writing via GitHub and Manubot could present a steeper barrier to entry and deter participants.
In addition, Git carefully tracks all revisions to the manuscript text but not the surrounding conversations that take place through GitHub issues and pull requests.
These discussions must be archived to ensure that important decisions about the manuscript are preserved and authors receive credit for intellectual contributions that are not directly reflected in the manuscript’s text.
GitHub supports programmatic access to issues, pull requests, and reviews so tracking these conversations is feasible in the future.</p>
<p>In the Deep Review, we established <a href="https://github.com/greenelab/deep-review/blob/v1.0/CONTRIBUTING.md">contributor guidelines</a> that discussed norms in the areas of text contribution, peer review, and authorship, which we identified in advance as potential areas of disagreement.
Our contributor guidelines required verifiable participation for authorship: either directly attributable changes to the text or participation in the discussion on GitHub.
These guidelines did not discuss broader community norms that may have improved inclusiveness.
It is also important to consider how the move to an open contribution model affects under-represented minority members of the scientific community <span class="citation" data-cites="URixREeW">[<a href="#ref-URixREeW" role="doc-biblioref">19</a>]</span>.
Recent work has identified clear social norms and processes as helpful to maintaining a collaborative culture <span class="citation" data-cites="NuDPNceu">[<a href="#ref-NuDPNceu" role="doc-biblioref">89</a>]</span>.
<!-- cited a previous version of doi:10.1080/08874417.2009.11645338 (see https://git.io/vdSSR) -->
Conferences and open source projects have used codes of conduct to establish these norms (e.g. <a href="https://www.contributor-covenant.org/" title="A Code of Conduct for Open Source Projects">Contributor Covenant</a>) <span class="citation" data-cites="bY9cHRxB">[<a href="#ref-bY9cHRxB" role="doc-biblioref">90</a>]</span>.
We would encourage the maintainers of similar projects to consider broader codes of conduct for project participants that build on social as well as academic norms.</p>
<h3 id="manubot-in-the-context-of-open-science">Manubot in the context of open science</h3>
<p>Science is undergoing a transition towards openness.
The internet provides a global information commons, where scholarship can be publicly shared at a minimal cost.
For example, open access publishing provides an economic model that encourages maximal dissemination and reuse of scholarly articles <span class="citation" data-cites="zBPP9YKu PuP45jrB HQfvK1OF">[<a href="#ref-zBPP9YKu" role="doc-biblioref">18</a>,<a href="#ref-PuP45jrB" role="doc-biblioref">91</a>,<a href="#ref-HQfvK1OF" role="doc-biblioref">92</a>]</span>.
More broadly, open licensing solves legal barriers to content reuse, enabling any type of scholarly output to become part of the commons <span class="citation" data-cites="g6WVoxNy 137tbemL9">[<a href="#ref-g6WVoxNy" role="doc-biblioref">93</a>,<a href="#ref-137tbemL9" role="doc-biblioref">94</a>]</span>.
The opportunity to reuse data and code for new investigations, as well as a push for increased reproducibility, has begot a movement to make all research outputs public, unless there are bona fide privacy or security concerns <span class="citation" data-cites="gvyja7v1 rgo1TZr 1A97a4UwU">[<a href="#ref-gvyja7v1" role="doc-biblioref">95</a>,<a href="#ref-rgo1TZr" role="doc-biblioref">96</a>,<a href="#ref-1A97a4UwU" role="doc-biblioref">97</a>]</span>.
New tools and services make it increasingly feasible to publicly share the unabridged methods of a study, especially for computational research, which consists solely of software and data.</p>
<p>Greater openness in both research methods and publishing creates an opportunity to redefine peer review and the role journals play in communicating science <span class="citation" data-cites="52Q1v5nS">[<a href="#ref-52Q1v5nS" role="doc-biblioref">68</a>]</span>.
At the extreme is real-time open science, whereby studies are performed entirely in the open from their inception <span class="citation" data-cites="17EdosXzD">[<a href="#ref-17EdosXzD" role="doc-biblioref">98</a>]</span>.
Many such research projects have now been completed, benefiting from the associated early-stage peer review, additional opportunity for online collaboration, and increased visibility <span class="citation" data-cites="1pWYlPj4 O21tn8vf">[<a href="#ref-O21tn8vf" role="doc-biblioref">50</a>,<a href="#ref-1pWYlPj4" role="doc-biblioref">99</a>]</span>.</p>
<p>Manubot is an ideal authoring protocol for real-time open science, especially for projects that are already using an open source software workflow to manage their research.
While Manubot does require technical expertise, the benefits are manyfold.
Specifically, Manubot demonstrates a system for publishing that is transparent, reproducible, immediate, permissionless, versioned, automated, collaborative, open, linked, provenanced, decentralized, hackable, interactive, annotated, and free of charge.
These attributes empower integrating Manubot with an ecosystem of other community-driven tools to make science as open and collaborative as possible.</p>
<h2 id="code-and-data-availability">Code and data availability</h2>
<p>The source code and data for this manuscript are available at <a href="https://github.com/greenelab/meta-review" class="uri">https://github.com/greenelab/meta-review</a> and archived via Software Heritage identifier <a href="https://archive.softwareheritage.org/swh:1:dir:da789e842d0af90f0fa50de522f0c4caae95e4e3;origin=https://github.com/greenelab/meta-review/"><code>swh:1:dir:da789e842d0af90f0fa50de522f0c4caae95e4e3</code></a>.
Source code for Manubot resides in the following repositories:</p>
<ul>
<li><a href="https://github.com/manubot/manubot" class="uri">https://github.com/manubot/manubot</a> (<a href="https://gitlab.com/manubot/manubot">GitLab mirror</a>, archived at <a href="https://archive.softwareheritage.org/swh:1:dir:506058af283d6d311a92d03b02adb9dca9f139de;origin=https://github.com/manubot/manubot.git/"><code>swh:1:dir:506058af283d6d311a92d03b02adb9dca9f139de</code></a>, packaged on <a href="https://pypi.org/project/manubot/">PyPI</a>)</li>
<li><a href="https://github.com/manubot/rootstock" class="uri">https://github.com/manubot/rootstock</a> (<a href="https://gitlab.com/manubot/rootstock">GitLab mirror</a>, archived at <a href="https://archive.softwareheritage.org/swh:1:dir:4e689f4e2f4b272b8672142f4c81abd3e77392f9;origin=https://github.com/manubot/rootstock.git/"><code>swh:1:dir:4e689f4e2f4b272b8672142f4c81abd3e77392f9</code></a>).</li>
</ul>
<h2 id="supporting-information">Supporting Information</h2>
<p id="vid:usage">
<video
  controls
  width="100%"
  alt="Screen recording of editing with Manubot using GitHub"
  poster="https://ndownloader.figshare.com/files/14874683"
>
<source
    src="https://ndownloader.figshare.com/files/14874821/preview/14874821/video_preview.mp4"
    type="video/mp4"
  >
</video>
<strong>S1 Video: Editing a manuscript on GitHub.</strong>
This screen recording demonstrates how to propose edits to a Manubot manuscript via GitHub.
In the video <span class="citation" data-cites="LaBVNb71">[<a href="#ref-LaBVNb71" role="doc-biblioref">100</a>]</span>, a contributor creates a <a href="https://github.com/manubot/try-manubot/pull/2">pull request</a> to add a sentence to the try-manubot manuscript.
The contributor then revises the proposed change to add a citation, after which it is accepted, merged, and automatically deployed.
</p>
<!-- citekey aliases -->
<h2 id="acknowledgments">Acknowledgments</h2>
<p>We would like to thank the authors of the Deep Review who helped us test collaborative writing with Manubot.
The authors who responded favorably to being acknowledged are Paul-Michael Agapow, Amr M. Alexandari, Brett K. Beaulieu-Jones, Anne E. Carpenter, Travers Ching, Evan M. Cofer, Dave DeCaprio, Brian T. Do, Enrico Ferrero, David J. Harris, Michael M. Hoffman, Alexandr A. Kalinin, Anshul Kundaje, Jack Lanchantin, Christopher A. Lavender, Benjamin J. Lengerich, Zhiyong Lu, Yifan Peng, Yanjun Qi, Gail L. Rosen, Avanti Shrikumar, Srinivas C. Turaga, Gregory P. Way, Laura K. Wiley, Stephen Woloszynek, Wei Xie, Jinbo Xu, and Michael Zietz.
In addition, we thank Ogun Adebali, Evan M. Cofer, and Robert Gieseke for contributing to the Rootstock manuscript.
We are grateful for additional Manubot discussion and testing by Alexander Dunkel, Ansel Halliburton, Benjamin J. Heil, Zach Hensel, Alexandra J. Lee, YoSon Park, Achintya Rao, and other GitHub users.
We thank John MacFarlane and Nikolay Yakimov for assistance with Pandoc and the global Binder team for advice on Binder.
Finally, we thank C. Titus Brown and the other anonymous reviewers for their help improving this manuscript.</p>
<!--
## Funding

DSH, DH, VR, and CSG were supported by [Grant G-2018-11163](https://sloan.org/grant-detail/8501) from the Alfred P. Sloan Foundation and [Grant GBMF4552](https://www.moore.org/grant-detail?grantId=GBMF4552) from the Gordon and Betty Moore Foundation.
VSM was supported by Grant RP150596 from the Cancer Prevention and Research Institute of Texas.
The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.
-->
<h2 class="page_break_before" id="references">References</h2>
<!-- Explicitly insert bibliography here -->
<div id="refs" class="references hanging-indent" role="doc-bibliography">
<div id="ref-1DiVJ3t6P">
<p>1. <strong>Reinventing Discovery</strong><br />
Michael Nielsen<br />
<em>Princeton University Press</em> (2011-01-31) <a href="https://doi.org/gfx2dm">https://doi.org/gfx2dm</a><br />
DOI: <a href="https://doi.org/10.1515/9781400839452">10.1515/9781400839452</a> · ISBN: <a href="https://worldcat.org/isbn/9781400839452">9781400839452</a></p>
</div>
<div id="ref-IWBJQIkl">
<p>2. <strong>Open Science by Design: Realizing a Vision for 21st Century Research</strong><br />
National Academies of Sciences, Engineering, and Medicine<br />
<em>National Academies Press</em> (2018-08-09) <a href="https://doi.org/gfxzc4">https://doi.org/gfxzc4</a><br />
DOI: <a href="https://doi.org/10.17226/25116">10.17226/25116</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/30212065">30212065</a> · ISBN: <a href="https://worldcat.org/isbn/9780309476249">9780309476249</a></p>
</div>
<div id="ref-16CgM2x0z">
<p>3. <strong>TechBlog: “Manubot” powers a crowdsourced “deep-learning” review</strong><br />
Jeffrey Perkel<br />
<em>Naturejobs</em> (2018-02-20) <a href="http://blogs.nature.com/naturejobs/2018/02/20/techblog-manubot-powers-a-crowdsourced-deep-learning-review/">http://blogs.nature.com/naturejobs/2018/02/20/techblog-manubot-powers-a-crowdsourced-deep-learning-review/</a></p>
</div>
<div id="ref-ZPiRHuLc">
<p>4. <strong>Crowdsourcing in biomedicine: challenges and opportunities</strong><br />
Ritu Khare, Benjamin M Good, Robert Leaman, Andrew I Su, Zhiyong Lu<br />
<em>Briefings in bioinformatics</em> (2016-01) <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4719068/">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4719068/</a><br />
DOI: <a href="https://doi.org/10.1093/bib/bbv021">10.1093/bib/bbv021</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/25888696">25888696</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4719068">PMC4719068</a></p>
</div>
<div id="ref-PZMP42Ak">
<p>5. <strong>Opportunities and obstacles for deep learning in biology and medicine</strong><br />
Travers Ching, Daniel S. Himmelstein, Brett K. Beaulieu-Jones, Alexandr A. Kalinin, Brian T. Do, Gregory P. Way, Enrico Ferrero, Paul-Michael Agapow, Michael Zietz, Michael M. Hoffman, … Casey S. Greene<br />
<em>Journal of The Royal Society Interface</em> (2018-04) <a href="https://doi.org/gddkhn">https://doi.org/gddkhn</a><br />
DOI: <a href="https://doi.org/10.1098/rsif.2017.0387">10.1098/rsif.2017.0387</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29618526">29618526</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5938574">PMC5938574</a></p>
</div>
<div id="ref-AylLD9F8">
<p>6. <strong>Scientific writing: the online cooperative</strong><br />
Jeffrey M. Perkel<br />
<em>Nature</em> (2014-10-01) <a href="https://doi.org/gbqsnd">https://doi.org/gbqsnd</a><br />
DOI: <a href="https://doi.org/10.1038/514127a">10.1038/514127a</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/25279924">25279924</a></p>
</div>
<div id="ref-PlcxShQU">
<p>7. <strong>A Quick Introduction to Version Control with Git and GitHub</strong><br />
John D. Blischak, Emily R. Davenport, Greg Wilson<br />
<em>PLOS Computational Biology</em> (2016-01-19) <a href="https://doi.org/gbqsnf">https://doi.org/gbqsnf</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pcbi.1004668">10.1371/journal.pcbi.1004668</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/26785377">26785377</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4718703">PMC4718703</a></p>
</div>
<div id="ref-kEX5dgzK">
<p>8. <strong>Ten Simple Rules for Taking Advantage of Git and GitHub</strong><br />
Yasset Perez-Riverol, Laurent Gatto, Rui Wang, Timo Sachsenberg, Julian Uszkoreit, Felipe da Veiga Leprevost, Christian Fufezan, Tobias Ternent, Stephen J. Eglen, Daniel S. Katz, … Juan Antonio Vizcaíno<br />
<em>PLOS Computational Biology</em> (2016-07-14) <a href="https://doi.org/gbrb39">https://doi.org/gbrb39</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pcbi.1004947">10.1371/journal.pcbi.1004947</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27415786">27415786</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4945047">PMC4945047</a></p>
</div>
<div id="ref-opQBBK06">
<p>9. <strong>Opportunities And Obstacles For Deep Learning In Biology And Medicine</strong><br />
Johnny Israeli<br />
<em>Towards Data Science</em> (2017-05-31) <a href="https://towardsdatascience.com/opportunities-and-obstacles-for-deep-learning-in-biology-and-medicine-6ec914fe18c2">https://towardsdatascience.com/opportunities-and-obstacles-for-deep-learning-in-biology-and-medicine-6ec914fe18c2</a></p>
</div>
<div id="ref-tJKvnIaZ">
<p>10. <strong>Opportunities And Obstacles For Deep Learning In Biology And Medicine</strong><br />
Travers Ching, Daniel S. Himmelstein, Brett K. Beaulieu-Jones, Alexandr A. Kalinin, Brian T. Do, Gregory P. Way, Enrico Ferrero, Paul-Michael Agapow, Michael Zietz, Michael M Hoffman, … Casey S. Greene<br />
<em>bioRxiv</em> (2017-05-28) <a href="https://doi.org/gbpvh5">https://doi.org/gbpvh5</a><br />
DOI: <a href="https://doi.org/10.1101/142760">10.1101/142760</a></p>
</div>
<div id="ref-Q20Bxdsr">
<p>11. <strong>Binder 2.0 - Reproducible, interactive, sharable environments for science at scale</strong><br />
Project Jupyter, Matthias Bussonnier, Jessica Forde, Jeremy Freeman, Brian Granger, Tim Head, Chris Holdgraf, Kyle Kelley, Gladys Nalvarte, Andrew Osheroff, … Carol Willing<br />
<em>Proceedings of the 17th Python in Science Conference</em> (2018) <a href="https://doi.org/gfwcm6">https://doi.org/gfwcm6</a><br />
DOI: <a href="https://doi.org/10.25080/majora-4af1f417-011">10.25080/majora-4af1f417-011</a></p>
</div>
<div id="ref-17wKkS4DV">
<p>12. <strong>Formatting Open Science: agilely creating multiple document formats for academic manuscripts with Pandoc Scholar</strong><br />
Albert Krewinkel, Robert Winkler<br />
<em>PeerJ Computer Science</em> (2017-05-08) <a href="https://doi.org/gbrb4c">https://doi.org/gbrb4c</a><br />
DOI: <a href="https://doi.org/10.7717/peerj-cs.112">10.7717/peerj-cs.112</a></p>
</div>
<div id="ref-9KfVIq3s">
<p>13. <strong>Reference Management</strong><br />
Martin Fenner, Kaja Scheliga, Sönke Bartling<br />
<em>Opening Science</em> (2013-12-17) <a href="https://doi.org/gbxtc8">https://doi.org/gbxtc8</a><br />
DOI: <a href="https://doi.org/10.1007/978-3-319-00026-8_8">10.1007/978-3-319-00026-8_8</a></p>
</div>
<div id="ref-K7WVgf8X">
<p>14. <strong>Comparison of Select Reference Management Tools</strong><br />
Yingting Zhang<br />
<em>Medical Reference Services Quarterly</em> (2012-01) <a href="https://doi.org/hpv">https://doi.org/hpv</a><br />
DOI: <a href="https://doi.org/10.1080/02763869.2012.641841">10.1080/02763869.2012.641841</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/22289095">22289095</a></p>
</div>
<div id="ref-GKPtRdAw">
<p>15. <strong>Twenty-Five Shades of Greycite: Semantics for referencing and
preservation</strong><br />
Phillip Lord, Lindsay Marshall<br />
<em>arXiv</em> (2013-04-26) <a href="https://arxiv.org/abs/1304.7151v1">https://arxiv.org/abs/1304.7151v1</a></p>
</div>
<div id="ref-6rA99N5X">
<p>16. <strong>Reviewing post-publication peer review.</strong><br />
Paul Knoepfler<br />
<em>Trends in genetics : TIG</em> (2015-04-04) <a href="https://www.ncbi.nlm.nih.gov/pubmed/25851694">https://www.ncbi.nlm.nih.gov/pubmed/25851694</a><br />
DOI: <a href="https://doi.org/10.1016/j.tig.2015.03.006">10.1016/j.tig.2015.03.006</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/25851694">25851694</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4472664">PMC4472664</a></p>
</div>
<div id="ref-Y2XyzLMc">
<p>17. <strong>Decentralized Trusted Timestamping using the Crypto Currency Bitcoin</strong><br />
Bela Gipp, Norman Meuschke, André Gernandt<br />
<em>arXiv</em> (2015-02-13) <a href="https://arxiv.org/abs/1502.04015v1">https://arxiv.org/abs/1502.04015v1</a></p>
</div>
<div id="ref-zBPP9YKu">
<p>18. <strong>Open access</strong><br />
Peter Suber<br />
<em>MIT Press</em> (2012)<br />
ISBN: <a href="https://worldcat.org/isbn/9780262517638">9780262517638</a></p>
</div>
<div id="ref-URixREeW">
<p>19. <strong>Open science and open science</strong><br />
~/<br />
(2017-06-05) <a href="https://lgatto.github.io/open-and-open/">https://lgatto.github.io/open-and-open/</a></p>
</div>
<div id="ref-QhC8yJ7V">
<p>20. <strong>Plan S: Accelerating the transition to full and immediate Open Access to scientific publications</strong><br />
cOAlition S<br />
(2018-09-04) <a href="https://www.wikidata.org/wiki/Q56458321">https://www.wikidata.org/wiki/Q56458321</a></p>
</div>
<div id="ref-J6sGnQp1">
<p>21. <strong>Paywall: The Business of Scholarship</strong>(2018-10) <a href="https://paywallthemovie.com/paywall">https://paywallthemovie.com/paywall</a></p>
</div>
<div id="ref-pqBLIXzp">
<p>22. <strong>Journal clubs in the time of preprints</strong><br />
Prachee Avasthi, Alice Soragni, Joshua N Bembenek<br />
<em>eLife</em> (2018-06-11) <a href="https://doi.org/gdm89h">https://doi.org/gdm89h</a><br />
DOI: <a href="https://doi.org/10.7554/elife.38532">10.7554/elife.38532</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29889024">29889024</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5995539">PMC5995539</a></p>
</div>
<div id="ref-1bGpMqiw">
<p>23. <strong>On author versus numeric citation styles</strong><br />
Daniel Himmelstein<br />
<em>Satoshi Village</em> (2018-03-12) <a href="https://blog.dhimmel.com/citation-styles/">https://blog.dhimmel.com/citation-styles/</a></p>
</div>
<div id="ref-w4n6Qtun">
<p>24. <strong>TechBlog: Create the perfect bibliography with the CSL Editor</strong><br />
Jeffrey Perkel<br />
<em>Naturejobs</em> (2017-05-03) <a href="http://blogs.nature.com/naturejobs/2017/05/03/techblog-create-the-perfect-bibliography-with-the-csl-editor/">http://blogs.nature.com/naturejobs/2017/05/03/techblog-create-the-perfect-bibliography-with-the-csl-editor/</a></p>
</div>
<div id="ref-LHrRxRb0">
<p>25. <strong>ANSI/NISO Z39.96-2019, JATS: Journal Article Tag Suite, version 1.2</strong><br />
National Information Standards Organization<br />
<em>NISO</em> (2019-02-08) <a href="https://www.niso.org/publications/z3996-2019-jats">https://www.niso.org/publications/z3996-2019-jats</a></p>
</div>
<div id="ref-AAwqxolU">
<p>26. <strong>Journal Article Tag Suite 1.0: National Information Standards Organization standard of journal extensible markup language</strong><br />
Sun Huh<br />
<em>Science Editing</em> (2014-08-18) <a href="https://doi.org/gbxtdk">https://doi.org/gbxtdk</a><br />
DOI: <a href="https://doi.org/10.6087/kcse.2014.1.99">10.6087/kcse.2014.1.99</a></p>
</div>
<div id="ref-bCyfIm6z">
<p>27. <strong>NISO Z39.96-201x, JATS: Journal Article Tag Suite</strong><br />
Mark H. Needleman<br />
<em>Serials Review</em> (2012-09) <a href="https://doi.org/gbxtdj">https://doi.org/gbxtdj</a><br />
DOI: <a href="https://doi.org/10.1080/00987913.2012.10765464">10.1080/00987913.2012.10765464</a></p>
</div>
<div id="ref-6eVHYGML">
<p>28. <strong>Data visualization tools drive interactivity and reproducibility in online publishing</strong><br />
Jeffrey M. Perkel<br />
<em>Nature</em> (2018-02-01) <a href="https://doi.org/gfw6g3">https://doi.org/gfw6g3</a><br />
DOI: <a href="https://doi.org/10.1038/d41586-018-01322-9">10.1038/d41586-018-01322-9</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29388968">29388968</a></p>
</div>
<div id="ref-nyZChH5b">
<p>29. <strong>Vega-Lite: A Grammar of Interactive Graphics</strong><br />
Arvind Satyanarayan, Dominik Moritz, Kanit Wongsuphasawat, Jeffrey Heer<br />
<em>IEEE Transactions on Visualization and Computer Graphics</em> (2017-01) <a href="https://doi.org/f92f32">https://doi.org/f92f32</a><br />
DOI: <a href="https://doi.org/10.1109/tvcg.2016.2599030">10.1109/tvcg.2016.2599030</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27875150">27875150</a></p>
</div>
<div id="ref-18w6XKsQO">
<p>30. <strong>Collaborative software development made easy</strong><br />
Andrew Silver<br />
<em>Nature</em> (2017-10) <a href="https://doi.org/cdvr">https://doi.org/cdvr</a><br />
DOI: <a href="https://doi.org/10.1038/550143a">10.1038/550143a</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/28980652">28980652</a></p>
</div>
<div id="ref-Qh7xTLwz">
<p>31. <strong>Reproducibility of computational workflows is automated using continuous analysis</strong><br />
Brett K Beaulieu-Jones, Casey S Greene<br />
<em>Nature Biotechnology</em> (2017-03-13) <a href="https://doi.org/f9ttx6">https://doi.org/f9ttx6</a><br />
DOI: <a href="https://doi.org/10.1038/nbt.3780">10.1038/nbt.3780</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/28288103">28288103</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6103790">PMC6103790</a></p>
</div>
<div id="ref-lXvpQxeN">
<p>32. <strong>Developing a modern data workflow for evolving data</strong><br />
Glenda M Yenni, Erica M Christensen, Ellen K Bledsoe, Sarah R Supp, Renata M Diaz, Ethan P White, SK Morgan Ernest<br />
<em>bioRxiv</em> (2018-06-12) <a href="https://doi.org/gdqbzn">https://doi.org/gdqbzn</a><br />
DOI: <a href="https://doi.org/10.1101/344804">10.1101/344804</a></p>
</div>
<div id="ref-vHuGhm4k">
<p>33. <strong>Priority of discovery in the life sciences</strong><br />
Ronald D Vale, Anthony A Hyman<br />
<em>eLife</em> (2016-06-16) <a href="https://doi.org/gcx6gx">https://doi.org/gcx6gx</a><br />
DOI: <a href="https://doi.org/10.7554/elife.16931">10.7554/elife.16931</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27310529">27310529</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4911212">PMC4911212</a></p>
</div>
<div id="ref-2K7yQEXT">
<p>34. <strong>Proof of prespecified endpoints in medical research with the bitcoin blockchain – The Grey Literature</strong><a href="https://blog.bgcarlisle.com/2014/08/25/proof-of-prespecified-endpoints-in-medical-research-with-the-bitcoin-blockchain/">https://blog.bgcarlisle.com/2014/08/25/proof-of-prespecified-endpoints-in-medical-research-with-the-bitcoin-blockchain/</a></p>
</div>
<div id="ref-hkguO3lG">
<p>35. <strong>The most interesting case of scientific irreproducibility?</strong><br />
Daniel Himmelstein<br />
<em>Satoshi Village</em> (2017-03-08) <a href="https://blog.dhimmel.com/irreproducible-timestamps/">https://blog.dhimmel.com/irreproducible-timestamps/</a></p>
</div>
<div id="ref-qh60RjR0">
<p>36. <strong>Bitcoin for the biological literature</strong><br />
Douglas Heaven<br />
<em>Nature</em> (2019-02) <a href="https://doi.org/gft5gp">https://doi.org/gft5gp</a><br />
DOI: <a href="https://doi.org/10.1038/d41586-019-00447-9">10.1038/d41586-019-00447-9</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/30718888">30718888</a></p>
</div>
<div id="ref-Y4trfW9i">
<p>37. <strong>Bitcoin: A Peer-to-Peer Electronic Cash System</strong><br />
Satoshi Nakamoto<br />
<em>Manubot</em> (2019-11-20) <a href="https://git.dhimmel.com/bitcoin-whitepaper/">https://git.dhimmel.com/bitcoin-whitepaper/</a></p>
</div>
<div id="ref-1DG704X8Q">
<p>38. <strong>OpenTimestamps: Scalable, Trust-Minimized, Distributed Timestamping with Bitcoin</strong><br />
Peter Todd<br />
<em>Peter Todd</em> (2016-09-15) <a href="https://petertodd.org/2016/opentimestamps-announcement">https://petertodd.org/2016/opentimestamps-announcement</a></p>
</div>
<div id="ref-sWD9uVuF">
<p>39. <strong>eLife supports development of open technology stack for publishing reproducible manuscripts online</strong><br />
Emily Packer<br />
<em>eLife Press Pack</em> (2017-09-07) <a href="https://elifesciences.org/for-the-press/e6038800/elife-supports-development-of-open-technology-stack-for-publishing-reproducible-manuscripts-online">https://elifesciences.org/for-the-press/e6038800/elife-supports-development-of-open-technology-stack-for-publishing-reproducible-manuscripts-online</a></p>
</div>
<div id="ref-IhliSZDo">
<p>40. <strong>Sci-Hub provides access to nearly all scholarly literature</strong><br />
Daniel S Himmelstein, Ariel Rodriguez Romero, Jacob G Levernier, Thomas Anthony Munro, Stephen Reid McLaughlin, Bastian Greshake Tzovaras, Casey S Greene<br />
<em>eLife</em> (2018-03-01) <a href="https://doi.org/ckcj">https://doi.org/ckcj</a><br />
DOI: <a href="https://doi.org/10.7554/elife.32822">10.7554/elife.32822</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29424689">29424689</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5832410">PMC5832410</a></p>
</div>
<div id="ref-9IrsqXRa">
<p>41. <strong>2017 in news: The science events that shaped the year</strong><br />
Ewen Callaway, Davide Castelvecchi, David Cyranoski, Elizabeth Gibney, Heidi Ledford, Jane J. Lee, Lauren Morello, Nicky Phillips, Quirin Schiermeier, Jeff Tollefson, … Alexandra Witze<br />
<em>Nature</em> (2017-12-21) <a href="https://doi.org/chnh">https://doi.org/chnh</a><br />
DOI: <a href="https://doi.org/10.1038/d41586-017-08493-x">10.1038/d41586-017-08493-x</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29293246">29293246</a></p>
</div>
<div id="ref-LGjXBQ7t">
<p>42. <strong>GimmeMotifs: an analysis framework for transcription factor motif analysis</strong><br />
Niklas Bruse, Simon J. van Heeringen<br />
<em>bioRxiv</em> (2018-11-20) <a href="https://doi.org/gfxrkc">https://doi.org/gfxrkc</a><br />
DOI: <a href="https://doi.org/10.1101/474403">10.1101/474403</a></p>
</div>
<div id="ref-15QSXk9Cl">
<p>43. <strong>Plasmids for Independently Tunable, Low-Noise Expression of Two Genes</strong><br />
João P. N. Silva, Soraia Vidigal Lopes, Diogo J. Grilo, Zach Hensel<br />
<em>mSphere</em> (2019-05-29) <a href="https://doi.org/gf3vhf">https://doi.org/gf3vhf</a><br />
DOI: <a href="https://doi.org/10.1128/msphere.00340-19">10.1128/msphere.00340-19</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/31142623">31142623</a></p>
</div>
<div id="ref-faKuBkfY">
<p>44. <strong>Illusions et hallucinations visuelles : une porte sur la perception</strong><br />
Laurent Perrinet<br />
<em>The Conversation</em> (2019-06-06) <a href="https://theconversation.com/illusions-et-hallucinations-visuelles-une-porte-sur-la-perception-117389">https://theconversation.com/illusions-et-hallucinations-visuelles-une-porte-sur-la-perception-117389</a></p>
</div>
<div id="ref-17a3WKGzu">
<p>45. <strong>Scaling tree-based automated machine learning to biomedical big data with a feature set selector</strong><br />
Trang T Le, Weixuan Fu, Jason H Moore<br />
<em>Bioinformatics</em> (2019-06-04) <a href="https://doi.org/gf3tds">https://doi.org/gf3tds</a><br />
DOI: <a href="https://doi.org/10.1093/bioinformatics/btz470">10.1093/bioinformatics/btz470</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/31165141">31165141</a></p>
</div>
<div id="ref-r2krHy6V">
<p>46. <strong>Genotyping structural variants in pangenome graphs using the vg toolkit</strong><br />
Glenn Hickey, David Heller, Jean Monlong, Jonas Andreas Sibbesen, Jouni Siren, Jordan Eizenga, Eric Dawson, Erik Garrison, Adam Novak, Benedict Paten<br />
<em>bioRxiv</em> (2019-06-01) <a href="https://doi.org/gf3jfm">https://doi.org/gf3jfm</a><br />
DOI: <a href="https://doi.org/10.1101/654566">10.1101/654566</a></p>
</div>
<div id="ref-WkeOa3Qo">
<p>47. <strong>A set of common software quality assurance baseline criteria for research projects</strong><br />
Pablo Orviz, Álvaro López García, Doina Cristina Duma, Giacinto Donvito, Mario David, Jorge Gomes<br />
(2017) <a href="https://digital.csic.es/handle/10261/160086">https://digital.csic.es/handle/10261/160086</a></p>
</div>
<div id="ref-15nwuvjrA">
<p>48. <strong>Vagelos Report Summer 2017</strong><br />
Michael Zietz<br />
<em>Figshare</em> (2017-08-25) <a href="https://doi.org/gbr3pf">https://doi.org/gbr3pf</a><br />
DOI: <a href="https://doi.org/10.6084/m9.figshare.5346577">10.6084/m9.figshare.5346577</a></p>
</div>
<div id="ref-u9DGTIX">
<p>49. <strong>How I used the Manubot to reproduce the Bitcoin Whitepaper</strong><br />
Daniel Himmelstein<br />
<em>Steem</em> (2017-09-20) <a href="https://busy.org/@dhimmel/how-i-used-the-manubot-to-reproduce-the-bitcoin-whitepaper">https://busy.org/@dhimmel/how-i-used-the-manubot-to-reproduce-the-bitcoin-whitepaper</a></p>
</div>
<div id="ref-O21tn8vf">
<p>50. <strong>Systematic integration of biomedical knowledge prioritizes drugs for repurposing</strong><br />
Daniel Scott Himmelstein, Antoine Lizee, Christine Hessler, Leo Brueggeman, Sabrina L Chen, Dexter Hadley, Ari Green, Pouya Khankhanian, Sergio E Baranzini<br />
<em>eLife</em> (2017-09-22) <a href="https://doi.org/cdfk">https://doi.org/cdfk</a><br />
DOI: <a href="https://doi.org/10.7554/elife.26726">10.7554/elife.26726</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/28936969">28936969</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5640425">PMC5640425</a></p>
</div>
<div id="ref-tQy0rfF4">
<p>51. <strong>The Kipoi repository accelerates community exchange and reuse of predictive models for genomics</strong><br />
Žiga Avsec, Roman Kreuzhuber, Johnny Israeli, Nancy Xu, Jun Cheng, Avanti Shrikumar, Abhimanyu Banerjee, Daniel S. Kim, Thorsten Beier, Lara Urban, … Julien Gagneur<br />
<em>Nature Biotechnology</em> (2019-05-28) <a href="https://doi.org/gf3fmq">https://doi.org/gf3fmq</a><br />
DOI: <a href="https://doi.org/10.1038/s41587-019-0140-0">10.1038/s41587-019-0140-0</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/31138913">31138913</a></p>
</div>
<div id="ref-xpw2aizK">
<p>52. <strong>Journal of Open Source Software (JOSS): design and first-year review</strong><br />
Arfon M. Smith, Kyle E. Niemeyer, Daniel S. Katz, Lorena A. Barba, George Githinji, Melissa Gymrek, Kathryn D. Huff, Christopher R. Madan, Abigail Cabunoc Mayes, Kevin M. Moerman, … Jacob T. Vanderplas<br />
<em>PeerJ Computer Science</em> (2018-02-12) <a href="https://doi.org/gc5sjf">https://doi.org/gc5sjf</a><br />
DOI: <a href="https://doi.org/10.7717/peerj-cs.147">10.7717/peerj-cs.147</a></p>
</div>
<div id="ref-14ahyJcvY">
<p>53. <strong>A Systematic Review of Research on the Meaning, Ethics and Practices of Authorship across Scholarly Disciplines</strong><br />
Ana Marušić, Lana Bošnjak, Ana Jerončić<br />
<em>PLoS ONE</em> (2011-09-08) <a href="https://doi.org/fp5drd">https://doi.org/fp5drd</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pone.0023477">10.1371/journal.pone.0023477</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/21931600">21931600</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3169533">PMC3169533</a></p>
</div>
<div id="ref-RK9sIADd">
<p>54. <strong>What Should Be Done To Tackle Ghostwriting in the Medical Literature?</strong><br />
Peter C Gøtzsche, Jerome P Kassirer, Karen L Woolley, Elizabeth Wager, Adam Jacobs, Art Gertel, Cindy Hamilton<br />
<em>PLoS Medicine</em> (2009-02-03) <a href="https://doi.org/bnzbx7">https://doi.org/bnzbx7</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pmed.1000023">10.1371/journal.pmed.1000023</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/19192943">19192943</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2634793">PMC2634793</a></p>
</div>
<div id="ref-vzcNEQll">
<p>55. <strong>Ten simple rules for collaboratively writing a multi-authored paper</strong><br />
Marieke A. Frassl, David P. Hamilton, Blaize A. Denfeld, Elvira de Eyto, Stephanie E. Hampton, Philipp S. Keller, Sapna Sharma, Abigail S. L. Lewis, Gesa A. Weyhenmeyer, Catherine M. O’Reilly, … Núria Catalán<br />
<em>PLOS Computational Biology</em> (2018-11-15) <a href="https://doi.org/gfj8wf">https://doi.org/gfj8wf</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pcbi.1006508">10.1371/journal.pcbi.1006508</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/30439938">30439938</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6237291">PMC6237291</a></p>
</div>
<div id="ref-e2rpsIbt">
<p>56. <strong>Revisiting authorship, and JOSS software publications</strong><br />
C. Titus Brown<br />
<em>Living in an Ivory Basement</em> (2019-01-16) <a href="http://ivory.idyll.org/blog/2019-authorship-revisiting.html">http://ivory.idyll.org/blog/2019-authorship-revisiting.html</a></p>
</div>
<div id="ref-15YfJWkEd">
<p>57. <strong>Combined Measurement of the Higgs Boson Mass in pp Collisions at sqrt[s]=7 and 8 TeV with the ATLAS and CMS Experiments</strong><br />
G. Aad, B. Abbott, J. Abdallah, O. Abdinov, R. Aben, M. Abolins, O. S. AbouZeid, H. Abramowicz, H. Abreu, R. Abreu, … <br />
<em>Physical Review Letters</em> (2015-05-14) <a href="https://doi.org/f3nr73">https://doi.org/f3nr73</a><br />
DOI: <a href="https://doi.org/10.1103/physrevlett.114.191803">10.1103/physrevlett.114.191803</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/26024162">26024162</a></p>
</div>
<div id="ref-Szy8sdWq">
<p>58. <strong>Drosophila Muller F Elements Maintain a Distinct Set of Genomic Properties Over 40 Million Years of Evolution</strong><br />
Wilson Leung, Christopher D. Shaffer, Laura K. Reed, Sheryl T. Smith, William Barshop, William Dirkes, Matthew Dothager, Paul Lee, Jeannette Wong, David Xiong, … Sarah C. R. Elgin<br />
<em>G3: Genes|Genomes|Genetics</em> (2015-03-04) <a href="https://doi.org/gfw5vf">https://doi.org/gfw5vf</a><br />
DOI: <a href="https://doi.org/10.1534/g3.114.015966">10.1534/g3.114.015966</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/25740935">25740935</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4426361">PMC4426361</a></p>
</div>
<div id="ref-NGOit8L2">
<p>59. <strong>Fruit-fly paper has 1,000 authors</strong><br />
Chris Woolston<br />
<em>Nature</em> (2015-05-13) <a href="https://doi.org/gckfqm">https://doi.org/gckfqm</a><br />
DOI: <a href="https://doi.org/10.1038/521263f">10.1038/521263f</a></p>
</div>
<div id="ref-nahdPyLb">
<p>60. <strong>Physics paper sets record with more than 5,000 authors</strong><br />
Davide Castelvecchi<br />
<em>Nature</em> (2015-05-15) <a href="https://doi.org/4sn">https://doi.org/4sn</a><br />
DOI: <a href="https://doi.org/10.1038/nature.2015.17567">10.1038/nature.2015.17567</a></p>
</div>
<div id="ref-Rhm4AK0j">
<p>61. <strong>Ten Simple Rules for Writing a Literature Review</strong><br />
Marco Pautasso<br />
<em>PLoS Computational Biology</em> (2013-07-18) <a href="https://doi.org/gcx9dm">https://doi.org/gcx9dm</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pcbi.1003149">10.1371/journal.pcbi.1003149</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/23874189">23874189</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3715443">PMC3715443</a></p>
</div>
<div id="ref-jYs2OUFW">
<p>62. <strong>A Stronger Post-Publication Culture Is Needed for Better Science</strong><br />
Hilda Bastian<br />
<em>PLoS Medicine</em> (2014-12-30) <a href="https://doi.org/gdm9cj">https://doi.org/gdm9cj</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pmed.1001772">10.1371/journal.pmed.1001772</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/25548904">25548904</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4280106">PMC4280106</a></p>
</div>
<div id="ref-HflJ6Hy5">
<p>63. <strong>Post-Publication Peer Review: Opening Up Scientific Conversation</strong><br />
Jane Hunter<br />
<em>Frontiers in Computational Neuroscience</em> (2012) <a href="https://doi.org/gdm9cm">https://doi.org/gdm9cm</a><br />
DOI: <a href="https://doi.org/10.3389/fncom.2012.00063">10.3389/fncom.2012.00063</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/22969719">22969719</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3431010">PMC3431010</a></p>
</div>
<div id="ref-uw5bep8P">
<p>64. <strong>Post-publication peer review, in all its guises, is here to stay</strong><br />
Michael Markie<br />
<em>Insights the UKSG journal</em> (2015-07-07) <a href="https://doi.org/gdm9ck">https://doi.org/gdm9ck</a><br />
DOI: <a href="https://doi.org/10.1629/uksg.245">10.1629/uksg.245</a></p>
</div>
<div id="ref-16H8Gdlkx">
<p>65. <strong>Introducing Massively Open Online Papers (MOOPs)</strong><br />
Jonathan Tennant, Natalia Z Bielczyk, Bastian Greshake Tzovaras, Paola Masuzzo, Tobias Steiner<br />
(2019-07-02) <a href="https://doi.org/gf7k53">https://doi.org/gf7k53</a><br />
DOI: <a href="https://doi.org/10.31222/osf.io/et8ak">10.31222/osf.io/et8ak</a></p>
</div>
<div id="ref-19ly8T0jZ">
<p>66. <strong>The HoTT Book</strong><br />
Homotopy Type Theory<br />
(2013-03-12) <a href="https://homotopytypetheory.org/book/">https://homotopytypetheory.org/book/</a></p>
</div>
<div id="ref-SbkGi9m1">
<p>67. <strong>Mathematics and Computation | The HoTT book</strong><a href="http://math.andrej.com/2013/06/20/the-hott-book/">http://math.andrej.com/2013/06/20/the-hott-book/</a></p>
</div>
<div id="ref-52Q1v5nS">
<p>68. <strong>A multi-disciplinary perspective on emergent and future innovations in peer review</strong><br />
Jonathan P. Tennant, Jonathan M. Dugan, Daniel Graziotin, Damien C. Jacques, François Waldner, Daniel Mietchen, Yehia Elkhatib, Lauren B. Collister, Christina K. Pikas, Tom Crick, … Julien Colomb<br />
<em>F1000Research</em> (2017-11-01) <a href="https://doi.org/gc5tcv">https://doi.org/gc5tcv</a><br />
DOI: <a href="https://doi.org/10.12688/f1000research.12037.2">10.12688/f1000research.12037.2</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/29188015">29188015</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5686505">PMC5686505</a></p>
</div>
<div id="ref-82ZjWq3i">
<p>69. <strong>Nearly 100 scientists spent 2 months on Google Docs to redefine the p-value. Here’s what they came up with</strong><br />
Jop Vrieze<br />
<em>Science</em> (2018-01-18) <a href="https://doi.org/gc5tct">https://doi.org/gc5tct</a><br />
DOI: <a href="https://doi.org/10.1126/science.aat0471">10.1126/science.aat0471</a></p>
</div>
<div id="ref-HXpnCQu1">
<p>70. <strong>Justify your alpha</strong><br />
Daniel Lakens, Federico G. Adolfi, Casper J. Albers, Farid Anvari, Matthew A. J. Apps, Shlomo E. Argamon, Thom Baguley, Raymond B. Becker, Stephen D. Benning, Daniel E. Bradford, … Rolf A. Zwaan<br />
<em>Nature Human Behaviour</em> (2018-02-26) <a href="https://doi.org/gcz8f3">https://doi.org/gcz8f3</a><br />
DOI: <a href="https://doi.org/10.1038/s41562-018-0311-x">10.1038/s41562-018-0311-x</a></p>
</div>
<div id="ref-Xs2yPQcr">
<p>71. <strong>A proposal for regularly updated review/survey articles: “Perpetual
Reviews”</strong><br />
David L. Mobley, Daniel M. Zuckerman<br />
<em>arXiv</em> (2015-02-03) <a href="https://arxiv.org/abs/1502.01329v2">https://arxiv.org/abs/1502.01329v2</a></p>
</div>
<div id="ref-H0XkaC8S">
<p>72. <strong>Why we need the Living Journal of Computational Molecular Science</strong><br />
David L. Mobley, Michael R. Shirts, Daniel M. Zuckerman<br />
<em>Living Journal of Computational Molecular Science</em> (2017-08-22) <a href="https://www.livecomsjournal.org/article/2031-why-we-need-the-living-journal-of-computational-molecular-science">https://www.livecomsjournal.org/article/2031-why-we-need-the-living-journal-of-computational-molecular-science</a><br />
DOI: <a href="https://doi.org/10.33011/livecoms.1.1.2031">10.33011/livecoms.1.1.2031</a></p>
</div>
<div id="ref-Bu8VxdWB">
<p>73. <strong>The “Paper” of the Future</strong><br />
Alyssa Goodman, Josh Peek, Alberto Accomazzi, Chris Beaumont, Christine L. Borgman, How-Huan Hope Chen, Merce Crosas, Christopher Erdmann, August Muench, Alberto Pepe, Curtis Wong<br />
<em>Authorea</em> <a href="https://doi.org/gf3c59">https://doi.org/gf3c59</a><br />
DOI: <a href="https://doi.org/10.22541/au.148769949.92783646">10.22541/au.148769949.92783646</a></p>
</div>
<div id="ref-WDvu1SAV">
<p>74. <strong>The arXiv of the future will not look like the arXiv</strong><br />
Alberto Pepe, Matteo Cantiello, Josh Nicholson<br />
<em>Authorea</em> <a href="https://doi.org/gdqbz3">https://doi.org/gdqbz3</a><br />
DOI: <a href="https://doi.org/10.22541/au.149693987.70506124">10.22541/au.149693987.70506124</a></p>
</div>
<div id="ref-ILhLpgTs">
<p>75. <strong>TechBlog: C. Titus Brown: Predicting the paper of the future</strong><br />
C. Titus Brown<br />
<em>Naturejobs</em> (2017-06-01) <a href="http://blogs.nature.com/naturejobs/2017/06/01/techblog-c-titus-brown-predicting-the-paper-of-the-future/">http://blogs.nature.com/naturejobs/2017/06/01/techblog-c-titus-brown-predicting-the-paper-of-the-future/</a></p>
</div>
<div id="ref-1BISa1RLr">
<p>76. <strong>bookdown</strong><br />
Yihui Xie<br />
<em>Chapman &amp;Hall/CRC The R Series</em> (2016-12-21) <a href="https://doi.org/gdqbz2">https://doi.org/gdqbz2</a><br />
DOI: <a href="https://doi.org/10.1201/9781315204963">10.1201/9781315204963</a></p>
</div>
<div id="ref-1CblcIDkq">
<p>77. <strong>Orchestrating a community-developed computational workshop and accompanying training materials</strong><br />
Sean Davis, Marcel Ramos, Lori Shepherd, Nitesh Turaga, Ludwig Geistlinger, Martin T. Morgan, Benjamin Haibe-Kains, Levi Waldron<br />
<em>F1000Research</em> (2018-10-17) <a href="https://doi.org/gfzrwh">https://doi.org/gfzrwh</a><br />
DOI: <a href="https://doi.org/10.12688/f1000research.16516.1">10.12688/f1000research.16516.1</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/30473781">30473781</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6234736">PMC6234736</a></p>
</div>
<div id="ref-q41cceRt">
<p>78. <strong>Decentralised Authoring, Annotations and Notifications for a Read-Write Web with dokieli</strong><br />
Sarven Capadisli, Amy Guy, Ruben Verborgh, Christoph Lange, Sören Auer, Tim Berners-Lee<br />
(2017-03-18) <a href="https://csarven.ca/dokieli-rww">https://csarven.ca/dokieli-rww</a></p>
</div>
<div id="ref-1CNWD7ZF">
<p>79. <strong>Citation.js: a format-independent, modular bibliography tool for the browser and command line</strong><br />
Lars G. Willighagen<br />
<em>PeerJ Computer Science</em> (2019-08-12) <a href="https://doi.org/gf7k5g">https://doi.org/gf7k5g</a><br />
DOI: <a href="https://doi.org/10.7717/peerj-cs.214">10.7717/peerj-cs.214</a></p>
</div>
<div id="ref-2JcGGG7l">
<p>80. <strong>Continuous Publishing</strong><br />
Gobbledygook<br />
<a href="http://blog.martinfenner.org/2014/03/10/continuous-publishing/">http://blog.martinfenner.org/2014/03/10/continuous-publishing/</a><br />
DOI: <a href="https://doi.org//2014/03/10/continuous-publishing">/2014/03/10/continuous-publishing</a></p>
</div>
<div id="ref-ujrfOBM1">
<p>81. <strong>Opening Science</strong><br />
Sönke Bartling, Sascha Friesike (editors)<br />
<em>Springer International Publishing</em> (2014) <a href="https://doi.org/gdqbzz">https://doi.org/gdqbzz</a><br />
DOI: <a href="https://doi.org/10.1007/978-3-319-00026-8">10.1007/978-3-319-00026-8</a></p>
</div>
<div id="ref-MHNCSD5I">
<p>82. <strong>The Building Blocks of Interpretability</strong><br />
Chris Olah, Arvind Satyanarayan, Ian Johnson, Shan Carter, Ludwig Schubert, Katherine Ye, Alexander Mordvintsev<br />
<em>Distill</em> (2018-03-06) <a href="https://doi.org/gdvhz5">https://doi.org/gdvhz5</a><br />
DOI: <a href="https://doi.org/10.23915/distill.00010">10.23915/distill.00010</a></p>
</div>
<div id="ref-1ESo5MNnB">
<p>83. <strong>Announcing idyll.pub</strong><br />
Matthew Conlen, Andrew Osheroff<br />
<em>Idyll</em> (2018-06-26) <a href="https://idyll.pub/post/announcing-idyll-pub-0a3eff0661df3446a915700d/">https://idyll.pub/post/announcing-idyll-pub-0a3eff0661df3446a915700d/</a></p>
</div>
<div id="ref-BWMf57EM">
<p>84. <strong>Stencila – an office suite for reproducible research</strong><br />
Michael Aufreiter, Aleksandra Pawlik, Nokome Bentley<br />
<em>eLife Labs</em> (2018-07-02) <a href="https://elifesciences.org/labs/c496b8bb/stencila-an-office-suite-for-reproducible-research">https://elifesciences.org/labs/c496b8bb/stencila-an-office-suite-for-reproducible-research</a></p>
</div>
<div id="ref-KLKZcPlg">
<p>85. <strong>Introducing eLife’s first computationally reproducible article</strong><br />
Giuliano Maciocci, Michael Aufreiter, Nokome Bentley<br />
<em>eLife Labs</em> (2019-02-20) <a href="https://elifesciences.org/labs/ad58f08d/introducing-elife-s-first-computationally-reproducible-article">https://elifesciences.org/labs/ad58f08d/introducing-elife-s-first-computationally-reproducible-article</a></p>
</div>
<div id="ref-8o1nWux7">
<p>86. <strong>Sustainable computational science: the ReScience initiative</strong><br />
Nicolas P. Rougier, Konrad Hinsen, Frédéric Alexandre, Thomas Arildsen, Lorena A. Barba, Fabien C. Y. Benureau, C. Titus Brown, Pierre de Buyl, Ozan Caglayan, Andrew P. Davison, … Tiziano Zito<br />
<em>PeerJ Computer Science</em> (2017-12-18) <a href="https://doi.org/gcx5kf">https://doi.org/gcx5kf</a><br />
DOI: <a href="https://doi.org/10.7717/peerj-cs.142">10.7717/peerj-cs.142</a></p>
</div>
<div id="ref-1ESYVbN4H">
<p>87. <strong>Distill Update 2018</strong><br />
Distill Editors<br />
<em>Distill</em> (2018-08-14) <a href="https://doi.org/gfbzs9">https://doi.org/gfbzs9</a><br />
DOI: <a href="https://doi.org/10.23915/distill.00013">10.23915/distill.00013</a></p>
</div>
<div id="ref-vf9t7xMG">
<p>88. <strong>The appropriation of GitHub for curation</strong><br />
Yu Wu, Na Wang, Jessica Kropczynski, John M. Carroll<br />
<em>PeerJ Computer Science</em> (2017-10-09) <a href="https://doi.org/gb3bxk">https://doi.org/gb3bxk</a><br />
DOI: <a href="https://doi.org/10.7717/peerj-cs.134">10.7717/peerj-cs.134</a></p>
</div>
<div id="ref-NuDPNceu">
<p>89. <strong>Innovating Collaborative Content Creation: The Role of Altruism and Wiki Technology</strong><br />
Christian Wagner, Pattarawan Prasarnphanich<br />
<em>2007 40th Annual Hawaii International Conference on System Sciences (HICSS’07)</em> (2007) <a href="https://doi.org/b6vqgx">https://doi.org/b6vqgx</a><br />
DOI: <a href="https://doi.org/10.1109/hicss.2007.277">10.1109/hicss.2007.277</a></p>
</div>
<div id="ref-bY9cHRxB">
<p>90. <strong>Code of conduct in open source projects</strong><br />
Parastou Tourani, Bram Adams, Alexander Serebrenik<br />
<em>2017 IEEE 24th International Conference on Software Analysis, Evolution and Reengineering (SANER)</em> (2017-02) <a href="https://doi.org/gfzbs6">https://doi.org/gfzbs6</a><br />
DOI: <a href="https://doi.org/10.1109/saner.2017.7884606">10.1109/saner.2017.7884606</a></p>
</div>
<div id="ref-PuP45jrB">
<p>91. <strong>The academic, economic and societal impacts of Open Access: an evidence-based review</strong><br />
Jonathan P. Tennant, François Waldner, Damien C. Jacques, Paola Masuzzo, Lauren B. Collister, Chris. H. J. Hartgerink<br />
<em>F1000Research</em> (2016-09-21) <a href="https://doi.org/gbqrbc">https://doi.org/gbqrbc</a><br />
DOI: <a href="https://doi.org/10.12688/f1000research.8460.3">10.12688/f1000research.8460.3</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27158456">27158456</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4837983">PMC4837983</a></p>
</div>
<div id="ref-HQfvK1OF">
<p>92. <strong>How open science helps researchers succeed</strong><br />
Erin C McKiernan, Philip E Bourne, C Titus Brown, Stuart Buck, Amye Kenall, Jennifer Lin, Damon McDougall, Brian A Nosek, Karthik Ram, Courtney K Soderberg, … Tal Yarkoni<br />
<em>eLife</em> (2016-07-07) <a href="https://doi.org/gbqsng">https://doi.org/gbqsng</a><br />
DOI: <a href="https://doi.org/10.7554/elife.16800">10.7554/elife.16800</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27387362">27387362</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4973366">PMC4973366</a></p>
</div>
<div id="ref-g6WVoxNy">
<p>93. <strong>The Legal Framework for Reproducible Scientific Research: Licensing and Copyright</strong><br />
Victoria Stodden<br />
<em>Computing in Science &amp; Engineering</em> (2009-01) <a href="https://doi.org/b7tskf">https://doi.org/b7tskf</a><br />
DOI: <a href="https://doi.org/10.1109/mcse.2009.19">10.1109/mcse.2009.19</a></p>
</div>
<div id="ref-137tbemL9">
<p>94. <strong>Legal confusion threatens to slow data science</strong><br />
Simon Oxenham<br />
<em>Nature</em> (2016-08-03) <a href="https://doi.org/bndt">https://doi.org/bndt</a><br />
DOI: <a href="https://doi.org/10.1038/536016a">10.1038/536016a</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27488781">27488781</a></p>
</div>
<div id="ref-gvyja7v1">
<p>95. <strong>Enhancing reproducibility for computational methods</strong><br />
V. Stodden, M. McNutt, D. H. Bailey, E. Deelman, Y. Gil, B. Hanson, M. A. Heroux, J. P. A. Ioannidis, M. Taufer<br />
<em>Science</em> (2016-12-08) <a href="https://doi.org/gbr42b">https://doi.org/gbr42b</a><br />
DOI: <a href="https://doi.org/10.1126/science.aah6168">10.1126/science.aah6168</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/27940837">27940837</a></p>
</div>
<div id="ref-rgo1TZr">
<p>96. <strong>The case for open computer programs</strong><br />
Darrel C. Ince, Leslie Hatton, John Graham-Cumming<br />
<em>Nature</em> (2012-02-22) <a href="https://doi.org/hqg">https://doi.org/hqg</a><br />
DOI: <a href="https://doi.org/10.1038/nature10836">10.1038/nature10836</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/22358837">22358837</a></p>
</div>
<div id="ref-1A97a4UwU">
<p>97. <strong>The Open Knowledge Foundation: Open Data Means Better Science</strong><br />
Jennifer C. Molloy<br />
<em>PLoS Biology</em> (2011-12-06) <a href="https://doi.org/g3b">https://doi.org/g3b</a><br />
DOI: <a href="https://doi.org/10.1371/journal.pbio.1001195">10.1371/journal.pbio.1001195</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/22162946">22162946</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3232214">PMC3232214</a></p>
</div>
<div id="ref-17EdosXzD">
<p>98. <strong>This revolution will be digitized: online tools for radical collaboration</strong><br />
C. Patil, V. Siegel<br />
<em>Disease Models &amp; Mechanisms</em> (2009-04-30) <a href="https://doi.org/fvjhcj">https://doi.org/fvjhcj</a><br />
DOI: <a href="https://doi.org/10.1242/dmm.003285">10.1242/dmm.003285</a> · PMID: <a href="https://www.ncbi.nlm.nih.gov/pubmed/19407323">19407323</a> · PMCID: <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2675795">PMC2675795</a></p>
</div>
<div id="ref-1pWYlPj4">
<p>99. <strong>Publishing the research process</strong><br />
Daniel Mietchen, Ross Mounce, Lyubomir Penev<br />
<em>Research Ideas and Outcomes</em> (2015-12-17) <a href="https://doi.org/f3mn7d">https://doi.org/f3mn7d</a><br />
DOI: <a href="https://doi.org/10.3897/rio.1.e7547">10.3897/rio.1.e7547</a></p>
</div>
<div id="ref-LaBVNb71">
<p>100. <strong>How to edit a manuscript on GitHub with Manubot</strong><br />
David Slochower, Daniel Himmelstein<br />
<em>Figshare</em> (2019) <a href="https://doi.org/gfzb6b">https://doi.org/gfzb6b</a><br />
DOI: <a href="https://doi.org/10.6084/m9.figshare.7946192.v2">10.6084/m9.figshare.7946192.v2</a></p>
</div>
</div>
<!-- default theme -->

<style>
    /* import google fonts */
    @import url("https://fonts.googleapis.com/css?family=Open+Sans:400,600,700");
    @import url("https://fonts.googleapis.com/css?family=Source+Code+Pro");

    /* -------------------------------------------------- */
    /* global */
    /* -------------------------------------------------- */

    /* all elements */
    * {
        /* force sans-serif font unless specified otherwise */
        font-family: "Open Sans", "Helvetica", sans-serif;

        /* prevent text inflation on some mobile browsers */
        -webkit-text-size-adjust: none !important;
        -moz-text-size-adjust: none !important;
        -o-text-size-adjust: none !important;
        text-size-adjust: none !important;
    }

    @media only screen {
        /* "page" element */
        body {
            position: relative;
            box-sizing: border-box;
            font-size: 12pt;
            line-height: 1.5;
            max-width: 8.5in;
            margin: 20px auto;
            padding: 40px;
            border-radius: 5px;
            border: solid 1px #bdbdbd;
            box-shadow: 0 0 20px rgba(0, 0, 0, 0.05);
            background: #ffffff;
        }
    }

    /* when on screen < 8.5in wide */
    @media only screen and (max-width: 8.5in) {
        /* "page" element */
        body {
            padding: 20px;
            margin: 0;
            border-radius: 0;
            border: none;
            box-shadow: 0 0 20px rgba(0, 0, 0, 0.05) inset;
            background: none;
        }
    }

    /* -------------------------------------------------- */
    /* headings */
    /* -------------------------------------------------- */

    /* all headings */
    h1,
    h2,
    h3,
    h4,
    h5,
    h6 {
        margin: 20px 0;
        padding: 0;
        font-weight: bold;
    }

    /* biggest heading */
    h1 {
        margin: 40px 0;
        text-align: center;
    }

    /* second biggest heading */
    h2 {
        margin-top: 30px;
        padding-bottom: 5px;
        border-bottom: solid 1px #bdbdbd;
    }

    /* -------------------------------------------------- */
    /* manuscript header */
    /* -------------------------------------------------- */

    /* manuscript title */
    header > h1 {
        margin: 0;
    }

    /* manuscript title caption text (ie "automatically generated on") */
    header + p {
        text-align: center;
        margin-top: 10px;
    }

    /* -------------------------------------------------- */
    /* text elements */
    /* -------------------------------------------------- */

    /* links */
    a {
        color: #2196f3;
        overflow-wrap: break-word;
    }

    /* normal links (not empty, not button link, not syntax highlighting link) */
    a:not(:empty):not(.button):not(.sourceLine) {
        padding-left: 1px;
        padding-right: 1px;
    }

    /* superscripts and subscripts */
    sub,
    sup {
        /* prevent from affecting line height */
        line-height: 0;
    }

    /* unordered and ordered lists*/
    ul,
    ol {
        padding-left: 20px;
    }

    /* class for styling text semibold */
    .semibold {
        font-weight: 600;
    }

    /* class for styling elements horizontally left aligned */
    .left {
        display: block;
        text-align: left;
        margin-left: auto;
        margin-right: 0;
        justify-content: left;
    }

    /* class for styling elements horizontally centered */
    .center {
        display: block;
        text-align: center;
        margin-left: auto;
        margin-right: auto;
        justify-content: center;
    }

    /* class for styling elements horizontally right aligned */
    .right {
        display: block;
        text-align: right;
        margin-left: 0;
        margin-right: auto;
        justify-content: right;
    }

    /* -------------------------------------------------- */
    /* section elements */
    /* -------------------------------------------------- */

    /* horizontal divider line */
    hr {
        border: none;
        height: 1px;
        background: #bdbdbd;
    }

    /* paragraphs, horizontal dividers, figures, tables, code */
    p,
    hr,
    figure,
    table,
    pre {
        /* treat all as "paragraphs", with consistent vertical margins */
        margin-top: 20px;
        margin-bottom: 20px;
    }

    /* -------------------------------------------------- */
    /* figures */
    /* -------------------------------------------------- */

    /* figure */
    figure {
        max-width: 100%;
        margin-left: auto;
        margin-right: auto;
    }

    /* figure caption */
    figcaption {
        padding: 0;
        padding-top: 10px;
    }

    /* figure image element */
    figure img {
        max-width: 100%;
        display: block;
        margin-left: auto;
        margin-right: auto;
    }

    /* figure auto-number */
    img + figcaption > span:first-of-type {
        font-weight: bold;
        margin-right: 5px;
    }

    /* -------------------------------------------------- */
    /* tables */
    /* -------------------------------------------------- */

    /* table */
    table {
        border-collapse: collapse;
        border-spacing: 0;
        width: 100%;
        margin-left: auto;
        margin-right: auto;
    }

    /* table cells */
    th,
    td {
        border: solid 1px #bdbdbd;
        padding: 10px;
        /* squash table if too wide for page by forcing line breaks */
        overflow-wrap: break-word;
        word-break: break-word;
    }

    /* header row and even rows */
    th,
    tr:nth-child(2n) {
        background-color: #fafafa;
    }

    /* odd rows */
    tr:nth-child(2n + 1) {
        background-color: #ffffff;
    }

    /* table caption */
    caption {
        text-align: left;
        padding: 0;
        padding-bottom: 10px;
    }

    /* table auto-number */
    table > caption > span:first-of-type,
    div.table_wrapper > table > caption > span:first-of-type {
        font-weight: bold;
        margin-right: 5px;
    }

    /* -------------------------------------------------- */
    /* code */
    /* -------------------------------------------------- */

    /* multi-line code block */
    pre {
        padding: 10px;
        background-color: #eeeeee;
        color: #000000;
        border-radius: 5px;
        break-inside: avoid;
        text-align: left;
    }

    /* inline code, ie code within normal text */
    :not(pre) > code {
        padding: 0 4px;
        background-color: #eeeeee;
        color: #000000;
        border-radius: 5px;
    }

    /* code text */
    /* apply all children, to reach syntax highlighting sub-elements */
    code,
    code * {
        /* force monospace font */
        font-family: "Source Code Pro", "Courier New", monospace;
    }

    /* -------------------------------------------------- */
    /* quotes */
    /* -------------------------------------------------- */

    /* quoted text */
    blockquote {
        margin: 0;
        padding: 0;
        border-left: 4px solid #bdbdbd;
        padding-left: 16px;
        break-inside: avoid;
    }

    /* -------------------------------------------------- */
    /* banners */
    /* -------------------------------------------------- */

    /* info banners */
    .banner {
        box-sizing: border-box;
        display: block;
        position: relative;
        width: 100%;
        margin-top: 20px;
        margin-bottom: 20px;
        padding: 20px;
        text-align: center;
    }

    /* paragraph in banner */
    .banner > p {
        margin: 0;
    }

    /* -------------------------------------------------- */
    /* highlight colors */
    /* -------------------------------------------------- */

    .white {
        background: #ffffff;
    }
    .lightgrey {
        background: #eeeeee;
    }
    .grey {
        background: #757575;
    }
    .darkgrey {
        background: #424242;
    }
    .black {
        background: #000000;
    }
    .lightred {
        background: #ffcdd2;
    }
    .lightyellow {
        background: #ffecb3;
    }
    .lightgreen {
        background: #dcedc8;
    }
    .lightblue {
        background: #e3f2fd;
    }
    .lightpurple {
        background: #f3e5f5;
    }
    .red {
        background: #f44336;
    }
    .orange {
        background: #ff9800;
    }
    .yellow {
        background: #ffeb3b;
    }
    .green {
        background: #4caf50;
    }
    .blue {
        background: #2196f3;
    }
    .purple {
        background: #9c27b0;
    }
    .white,
    .lightgrey,
    .lightred,
    .lightyellow,
    .lightgreen,
    .lightblue,
    .lightpurple,
    .orange,
    .yellow,
    .white a,
    .lightgrey a,
    .lightred a,
    .lightyellow a,
    .lightgreen a,
    .lightblue a,
    .lightpurple a,
    .orange a,
    .yellow a {
        color: #000000;
    }
    .grey,
    .darkgrey,
    .black,
    .red,
    .green,
    .blue,
    .purple,
    .grey a,
    .darkgrey a,
    .black a,
    .red a,
    .green a,
    .blue a,
    .purple a {
        color: #ffffff;
    }

    /* -------------------------------------------------- */
    /* buttons */
    /* -------------------------------------------------- */

    /* class for styling links like buttons */
    .button {
        display: inline-flex;
        justify-content: center;
        align-items: center;
        margin: 5px;
        padding: 10px 20px;
        font-size: 0.75em;
        font-weight: 600;
        text-transform: uppercase;
        text-decoration: none;
        letter-spacing: 1px;
        background: none;
        color: #2196f3;
        border: solid 1px #bdbdbd;
        border-radius: 5px;
    }

    /* buttons when hovered */
    .button:hover:not([disabled]),
    .icon_button:hover:not([disabled]) {
        cursor: pointer;
        background: #f5f5f5;
    }

    /* buttons when disabled */
    .button[disabled],
    .icon_button[disabled] {
        opacity: 0.35;
        pointer-events: none;
    }

    /* class for styling buttons containg only single icon */
    .icon_button {
        display: inline-flex;
        justify-content: center;
        align-items: center;
        text-decoration: none;
        margin: 0;
        padding: 0;
        background: none;
        border-radius: 5px;
        border: none;
        width: 20px;
        height: 20px;
        min-width: 20px;
        min-height: 20px;
    }

    /* icon button inner svg image */
    .icon_button > svg {
        height: 16px;
    }

    /* -------------------------------------------------- */
    /* icons */
    /* -------------------------------------------------- */

    /* class for styling icons inline with text */
    .inline_icon {
        height: 1em;
        position: relative;
        top: 0.125em;
    }

    /* -------------------------------------------------- */
    /* print control */
    /* -------------------------------------------------- */

    @media print {
        @page {
            /* suggested printing margin */
            margin: 0.5in;
        }

        /* document and "page" elements */
        html, body {
            margin: 0;
            padding: 0;
            width: 100%;
            height: 100%;
        }

        /* "page" element */
        body {
            font-size: 11pt !important;
            line-height: 1.35;
        }

        /* all headings */
        h1,
        h2,
        h3,
        h4,
        h5,
        h6 {
            margin: 15px 0;
        }

        /* heading 1 */
        h1 {
            font-size: 1.75em;
        }

        /* heading 2 */
        h2 {
            font-size: 1.25em;
            margin-top: 0;
        }

        /* heading 3 */
        h3 {
            font-size: 1.10em;
        }

        /* figures and tables */
        figure, table {
            font-size: 0.85em;
        }

        /* table cells */
        th,
        td {
            padding: 5px;
        }

        /* shrink font awesome icons */
        i.fas,
        i.fab,
        i.far,
        i.fal {
            transform: scale(0.85);
        }

        /* decrease banner margins */
        .banner {
            margin-top: 15px;
            margin-bottom: 15px;
            padding: 15px;
        }

        /* class for centering an element vertically on its own page */
        .page_center {
            margin: auto;
            width: 100%;
            height: 100%;
            display: flex;
            align-items: center;
            vertical-align: middle;
            break-before: page;
            break-after: page;
        }

        /* always insert a page break before the element */
        .page_break_before {
            break-before: page;
        }

        /* always insert a page break after the element */
        .page_break_after {
            break-after: page;
        }

        /* avoid page break before the element */
        .page_break_before_avoid {
            break-before: avoid;
        }

        /* avoid page break after the element */
        .page_break_after_avoid {
            break-after: avoid;
        }

        /* avoid page break inside the element */
        .page_break_inside_avoid {
            break-inside: avoid;
        }
    }

    /* -------------------------------------------------- */
    /* override pandoc css quirks */
    /* -------------------------------------------------- */

    .sourceCode {
        /* prevent unsightly overflow in wide code blocks */
        overflow: auto !important;
    }

    div.sourceCode {
        /* prevent background fill on top-most code block  container */
        background: none !important;
    }

    .sourceCode * {
        /* force consistent line spacing */
        line-height: 1.5 !important;
    }

    div.sourceCode {
        /* style code block margins same as <pre> element */
        margin-top: 20px;
        margin-bottom: 20px;
    }

    /* -------------------------------------------------- */
    /* tablenos */
    /* -------------------------------------------------- */

    /* tablenos wrapper */
    .tablenos {
        /* show scrollbar on tables if necessary to prevent overflow */
        width: 100%;
        margin: 20px 0;
    }

    .tablenos > table {
        /* move margins from table to table_wrapper to allow margin collapsing */
        margin: 0;
    }

    @media only screen {
        /* tablenos wrapper */
        .tablenos {
            /* show scrollbar on tables if necessary to prevent overflow */
            overflow-x: auto !important;
        }

        .tablenos th,
        .tablenos td {
            overflow-wrap: unset !important;
            word-break: unset !important;
        }

        /* table in wrapper */
        .tablenos table,
        .tablenos table * {
            /* don't break table words */
            overflow-wrap: normal !important;
        }
    }

    /* -------------------------------------------------- */
    /* mathjax */
    /* -------------------------------------------------- */

    /* mathjax containers */
    .math.display > span:not(.MathJax_Preview) {
        /* turn inline element (no dimensions) into block (allows fixed width and thus scrolling) */
        display: flex !important;
        overflow-x: auto !important;
        overflow-y: hidden !important;
        justify-content: center;
        align-items: center;
        margin: 0 !important;
    }

    /* right click menu */
    .MathJax_Menu {
        border-radius: 5px !important;
        border: solid 1px #bdbdbd !important;
        box-shadow: none !important;
    }

    /* equation auto-number */
    span[id^="eq:"] > span.math.display + span {
        font-weight: 600;
    }

    /* equation */
    span[id^="eq:"] > span.math.display > span {
        /* nudge to make room for equation auto-number and anchor */
        margin-right: 60px !important;
    }

    /* -------------------------------------------------- */
    /* anchors plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* anchor button */
        .anchor {
            opacity: 0;
            margin-left: 5px;
        }

        /* anchor buttons within <h2>'s */
        h2 .anchor {
            margin-left: 10px;
        }

        /* anchor buttons when hovered/focused and anything containing an anchor button when hovered */
        *:hover > .anchor,
        .anchor:hover,
        .anchor:focus {
            opacity: 1;
        }

        /* anchor button when hovered */
        .anchor:hover {
            cursor: pointer;
        }
    }

    /* always show anchor button on devices with no mouse/hover ability */
    @media (hover: none) {
        .anchor {
            opacity: 1;
        }
    }

    /* always hide anchor button on print */
    @media only print {
        .anchor {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* accordion plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* accordion arrow button */
        .accordion_arrow {
            margin-right: 10px;
        }

        /* arrow icon when <h2> data-collapsed attribute true */
        h2[data-collapsed="true"] > .accordion_arrow > svg {
            transform: rotate(-90deg);
        }

        /* all elements (except <h2>'s) when data-collapsed attribute true */
        *:not(h2)[data-collapsed="true"] {
            display: none;
        }

        /* accordion arrow button when hovered and <h2>'s when hovered */
        .accordion_arrow:hover,
        h2[data-collapsed="true"]:hover,
        h2[data-collapsed="false"]:hover {
            cursor: pointer;
        }
    }

    /* always hide accordion arrow button on print */
    @media only print {
        .accordion_arrow {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* tooltips plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* tooltip container */
        #tooltip {
            position: absolute;
            width: 50%;
            min-width: 240px;
            max-width: 75%;
            z-index: 1;
        }

        /* tooltip content */
        #tooltip_content {
            margin-bottom: 5px;
            padding: 20px;
            border-radius: 5px;
            border: solid 1px #bdbdbd;
            box-shadow: 0 0 20px rgba(0, 0, 0, 0.05);
            background: #ffffff;
            overflow-wrap: break-word;
        }

        /* tooltip copy of paragraphs and figures */
        #tooltip_content > p,
        #tooltip_content > figure {
            margin: 0;
            max-height: 320px;
            overflow-y: auto;
        }

        /* tooltip copy of <img> */
        #tooltip_content > figure > img {
            max-height: 260px;
        }

        /* navigation bar */
        #tooltip_nav_bar {
            margin-top: 10px;
            text-align: center;
        }

        /* navigation bar previous/next buton */
        #tooltip_nav_bar > .icon_button {
            position: relative;
            top: 3px;
        }

        /* navigation bar previous button */
        #tooltip_nav_bar > .icon_button:first-of-type {
            margin-right: 5px;
        }

        /* navigation bar next button */
        #tooltip_nav_bar > .icon_button:last-of-type {
            margin-left: 5px;
        }
    }

    /* always hide tooltip on print */
    @media only print {
        #tooltip {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* jump to first plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* jump button */
        .jump_arrow {
            position: relative;
            top: 0.125em;
            margin-right: 5px;
        }
    }

    /* always hide jump button on print */
    @media only print {
        .jump_arrow {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* link highlight plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* anything with data-highlighted attribute true */
        [data-highlighted="true"] {
            background: #ffeb3b;
        }

        /* anything with data-selected attribute true */
        [data-selected="true"] {
            background: #ff8a65 !important;
        }

        /* animation definition for glow */
        @keyframes highlight_glow {
            0% {
                background: none;
            }
            10% {
                background: #bbdefb;
            }
            100% {
                background: none;
            }
        }

        /* anything with data-glow attribute true */
        [data-glow="true"] {
            animation: highlight_glow 2s;
        }
    }

    /* -------------------------------------------------- */
    /* table of contents plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* toc panel */
        #toc_panel {
            box-sizing: border-box;
            position: fixed;
            top: 0;
            left: 0;
            background: #ffffff;
            box-shadow: 0 0 20px rgba(0, 0, 0, 0.05);
            z-index: 2;
        }

        /* toc panel when closed */
        #toc_panel[data-open="false"] {
            min-width: 60px;
            width: 60px;
            height: 60px;
            border-right: solid 1px #bdbdbd;
            border-bottom: solid 1px #bdbdbd;
        }

        /* toc panel when open */
        #toc_panel[data-open="true"] {
            min-width: 260px;
            max-width: 480px;
            /* keep panel edge consistent distance away from "page" edge */
            width: calc(((100vw - 8.5in) / 2) - 30px - 40px);
            bottom: 0;
            border-right: solid 1px #bdbdbd;
        }

        /* toc panel header */
        #toc_header {
            box-sizing: border-box;
            display: flex;
            flex-direction: row;
            align-items: center;
            height: 60px;
            margin: 0;
            padding: 20px;
        }

        /* toc panel header when hovered */
        #toc_header:hover {
            cursor: pointer;
        }

        /* toc panel header when panel open */
        #toc_panel[data-open="true"] > #toc_header {
            border-bottom: solid 1px #bdbdbd;
        }

        /* toc open/close header button */
        #toc_button {
            margin-right: 20px;
        }

        /* hide toc list and header text when closed */
        #toc_panel[data-open="false"] > #toc_header > *:not(#toc_button),
        #toc_panel[data-open="false"] > #toc_list {
            display: none;
        }

        /* toc list of entries */
        #toc_list {
            box-sizing: border-box;
            width: 100%;
            padding: 20px;
            position: absolute;
            top: calc(60px + 1px);
            bottom: 0;
            overflow: auto;
        }

        /* toc entry, link to section in document */
        .toc_link {
            display: block;
            padding: 5px;
            position: relative;
            font-weight: 600;
            text-decoration: none;
        }

        /* toc entry when hovered or when "viewed" */
        .toc_link:hover,
        .toc_link[data-viewing="true"] {
            background: #f5f5f5;
        }

        /* toc entry, level 1 indentation */
        .toc_link[data-level="1"] {
            margin-left: 0;
        }

        /* toc entry, level 2 indentation */
        .toc_link[data-level="2"] {
            margin-left: 20px;
        }

        /* toc entry, level 3 indentation */
        .toc_link[data-level="3"] {
            margin-left: 40px;
        }

        /* toc entry, level 4 indentation */
        .toc_link[data-level="4"] {
            margin-left: 60px;
        }

        /* toc entry bullets */
        #toc_panel[data-bullets="true"] .toc_link[data-level]:before {
            position: absolute;
            left: -15px;
            top: -1px;
            font-size: 1.5em;
        }

        /* toc entry, level 2 bullet */
        #toc_panel[data-bullets="true"] .toc_link[data-level="2"]:before {
            content: "\2022";
        }

        /* toc entry, level 3 bullet */
        #toc_panel[data-bullets="true"] .toc_link[data-level="3"]:before {
            content: "\25AB";
        }

        /* toc entry, level 4 bullet */
        #toc_panel[data-bullets="true"] .toc_link[data-level="4"]:before {
            content: "-";
        }
    }

    /* when on screen < 8.5in wide */
    @media only screen and (max-width: 8.5in) {
        /* push <body> ("page") element down to make room for toc icon */
        .toc_body_nudge {
            padding-top: 60px;
        }

        /* toc icon when panel closed and not hovered */
        #toc_panel[data-open="false"]:not(:hover) {
            background: rgba(255, 255, 255, 0.75);
        }
    }

    /* always hide toc panel on print */
    @media only print {
        #toc_panel {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* lightbox plugin */
    /* -------------------------------------------------- */

    @media only screen {
        /* regular <img> in document when hovered */
        .lightbox_document_img:hover {
            cursor: pointer;
        }

        .body_no_scroll {
            overflow: hidden !important;
        }

        /* screen overlay */
        #lightbox_overlay {
            display: flex;
            flex-direction: column;
            position: fixed;
            left: 0;
            top: 0;
            right: 0;
            bottom: 0;
            background: rgba(0, 0, 0, 0.75);
            z-index: 3;
        }

        /* middle area containing lightbox image */
        #lightbox_image_container {
            flex-grow: 1;
            display: flex;
            justify-content: center;
            align-items: center;
            overflow: hidden;
            position: relative;
            padding: 20px;
        }

        /* bottom area containing caption */
        #lightbox_bottom_container {
            display: flex;
            justify-content: center;
            align-items: center;
            height: 100px;
            min-height: 100px;
            max-height: 100px;
            background: rgba(0, 0, 0, 0.5);
        }

        /* image number info text box */
        #lightbox_number_info {
            position: absolute;
            color: #ffffff;
            font-weight: 600;
            left: 2px;
            top: 0;
            z-index: 4;
        }

        /* zoom info text box */
        #lightbox_zoom_info {
            position: absolute;
            color: #ffffff;
            font-weight: 600;
            right: 2px;
            top: 0;
            z-index: 4;
        }

        /* copy of image caption */
        #lightbox_caption {
            box-sizing: border-box;
            display: inline-block;
            width: 100%;
            max-height: 100%;
            padding: 10px 0;
            text-align: center;
            overflow-y: auto;
            color: #ffffff;
        }

        /* navigation previous/next button */
        .lightbox_button {
            width: 100px;
            height: 100%;
            min-width: 100px;
            min-height: 100%;
            color: #ffffff;
        }

        /* navigation previous/next button when hovered */
        .lightbox_button:hover {
            background: none !important;
        }

        /* navigation button icon */
        .lightbox_button > svg {
            height: 25px;
        }

        /* figure auto-number */
        #lightbox_caption > span:first-of-type {
            font-weight: bold;
            margin-right: 5px;
        }

        /* lightbox image when hovered */
        #lightbox_img:hover {
            cursor: grab;
        }

        /* lightbox image when grabbed */
        #lightbox_img:active {
            cursor: grabbing;
        }
    }

    /* when on screen < 480px wide */
    @media only screen and (max-width: 480px) {
        /* make navigation buttons skinnier on small screens to make more room for caption text */
        .lightbox_button {
            width: 50px;
            min-width: 50px;
        }
    }

    /* always hide lightbox on print */
    @media only print {
        #lightbox_overlay {
            display: none;
        }
    }

    /* -------------------------------------------------- */
    /* hypothesis (annotations) plugin */
    /* -------------------------------------------------- */

    /* side panel */
    .annotator-frame {
        width: 280px !important;
        z-index: 0 !important;
    }

    /* match highlight color to rest of theme */
    .annotator-highlights-always-on .annotator-hl {
        background-color: #ffeb3b !important;
    }

    /* match focused color to rest of theme */
    .annotator-hl.annotator-hl-focused {
        background-color: #ff8a65 !important;
    }

    /* match bucket bar color to rest of theme */
    .annotator-bucket-bar {
        background: #f5f5f5 !important;
    }

    /* always hide toolbar and tooltip on print */
    @media only print {
        .annotator-frame {
            display: none !important;
        }

        hypothesis-adder {
            display: none !important;
        }
    }
</style>
<!-- anchors plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin adds an anchor next to each of a certain type
        // of element that provides a human-readable url to that specific
        // item/position in the document (eg "manuscript.html#abstract"). It
        // also makes it such that scrolling out of view of a target removes
        // its identifier from the url.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'anchors';

        // default plugin options
        const options = {
            // which types of elements to add anchors next to, in
            // "document.querySelector" format
            typesQuery: 'h1, h2, h3, [id^="fig:"], [id^="tbl:"], [id^="eq:"]',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            // add anchor to each element of specified types
            const elements = document.querySelectorAll(options.typesQuery);
            for (const element of elements)
                addAnchor(element);

            // attach scroll listener to window
            window.addEventListener('scroll', onScroll);
        }

        // when window is scrolled
        function onScroll() {
            // if url has hash and user has scrolled out of view of hash
            // target, remove hash from url
            const tolerance = 100;
            const target = getHashTarget();
            if (target) {
                if (
                    target.getBoundingClientRect().top >
                        window.innerHeight + tolerance ||
                    target.getBoundingClientRect().bottom < 0 - tolerance
                )
                    history.pushState(null, null, ' ');
            }
        }

        // add anchor to element
        function addAnchor(element) {
            let addTo; // element to add anchor button to

            // if figure or table, modify withId and addTo to get expected
            // elements
            if (element.id.indexOf('fig:') === 0) {
                addTo = element.querySelector('figcaption');
            } else if (element.id.indexOf('tbl:') === 0) {
                addTo = element.querySelector('caption');
            } else if (element.id.indexOf('eq:') === 0) {
                addTo = element.querySelector('.eqnos-number');
            }

            addTo = addTo || element;
            const id = element.id || null;

            // do not add anchor if element doesn't have assigned id.
            // id is generated by pandoc and is assumed to be unique and
            // human-readable
            if (!id)
                return;

            // create anchor button
            const anchor = document.createElement('a');
            anchor.innerHTML = document.querySelector('.icon_link').innerHTML;
            anchor.title = 'Link to this part of the document';
            anchor.classList.add('icon_button', 'anchor');
            anchor.dataset.ignore = 'true';
            anchor.href = '#' + id;
            addTo.appendChild(anchor);
        }

        // get element that is target of link or url hash
        function getHashTarget() {
            const hash = window.location.hash;
            const id = hash.slice(1);
            let target = document.querySelector('[id="' + id + '"]');
            if (!target)
                return;

            // if figure or table, modify target to get expected element
            if (id.indexOf('fig:') === 0)
                target = target.querySelector('figure');
            if (id.indexOf('tbl:') === 0)
                target = target.querySelector('table');

            return target;
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- link icon -->

<template class="icon_link">
    <!-- modified from: https://fontawesome.com/icons/link -->
    <svg width="16" height="16" viewBox="0 0 512 512">
        <path
            fill="currentColor"
            d="M326.612 185.391c59.747 59.809 58.927 155.698.36 214.59-.11.12-.24.25-.36.37l-67.2 67.2c-59.27 59.27-155.699 59.262-214.96 0-59.27-59.26-59.27-155.7 0-214.96l37.106-37.106c9.84-9.84 26.786-3.3 27.294 10.606.648 17.722 3.826 35.527 9.69 52.721 1.986 5.822.567 12.262-3.783 16.612l-13.087 13.087c-28.026 28.026-28.905 73.66-1.155 101.96 28.024 28.579 74.086 28.749 102.325.51l67.2-67.19c28.191-28.191 28.073-73.757 0-101.83-3.701-3.694-7.429-6.564-10.341-8.569a16.037 16.037 0 0 1-6.947-12.606c-.396-10.567 3.348-21.456 11.698-29.806l21.054-21.055c5.521-5.521 14.182-6.199 20.584-1.731a152.482 152.482 0 0 1 20.522 17.197zM467.547 44.449c-59.261-59.262-155.69-59.27-214.96 0l-67.2 67.2c-.12.12-.25.25-.36.37-58.566 58.892-59.387 154.781.36 214.59a152.454 152.454 0 0 0 20.521 17.196c6.402 4.468 15.064 3.789 20.584-1.731l21.054-21.055c8.35-8.35 12.094-19.239 11.698-29.806a16.037 16.037 0 0 0-6.947-12.606c-2.912-2.005-6.64-4.875-10.341-8.569-28.073-28.073-28.191-73.639 0-101.83l67.2-67.19c28.239-28.239 74.3-28.069 102.325.51 27.75 28.3 26.872 73.934-1.155 101.96l-13.087 13.087c-4.35 4.35-5.769 10.79-3.783 16.612 5.864 17.194 9.042 34.999 9.69 52.721.509 13.906 17.454 20.446 27.294 10.606l37.106-37.106c59.271-59.259 59.271-155.699.001-214.959z"
        ></path>
    </svg>
</template>
<!-- accordion plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin allows sections of content under <h2> headings
        // to be collapsible.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'accordion';

        // default plugin options
        const options = {
            // whether to always start expanded ('false'), always start
            // collapsed ('true'), or start collapsed when screen small ('auto')
            startCollapsed: 'auto',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            // run through each <h2> heading
            const headings = document.querySelectorAll('h2');
            for (const heading of headings) {
                addArrow(heading);

                // start expanded/collapsed based on option
                if (
                    options.startCollapsed === 'true' ||
                    (options.startCollapsed === 'auto' && isSmallScreen())
                )
                    collapseHeading(heading);
                else
                    expandHeading(heading);
            }

            // attach hash change listener to window
            window.addEventListener('hashchange', onHashChange);
        }

        // when hash (eg manuscript.html#introduction) changes
        function onHashChange() {
            const target = getHashTarget();
            if (target)
                goToElement(target);
        }

        // add arrow to heading
        function addArrow(heading) {
            // add arrow button
            const arrow = document.createElement('button');
            arrow.innerHTML = document.querySelector(
                '.icon_angle_down'
            ).innerHTML;
            arrow.classList.add('icon_button', 'accordion_arrow');
            heading.insertBefore(arrow, heading.firstChild);

            // attach click listener to heading and button
            heading.addEventListener('click', onHeadingClick);
            arrow.addEventListener('click', onArrowClick);
        }

        // determine if on mobile-like device with small screen
        function isSmallScreen() {
            return Math.min(window.innerWidth, window.innerHeight) < 480;
        }

        // scroll to and focus element
        function goToElement(element, offset) {
            // expand accordion section if collapsed
            expandElement(element);
            const y =
                getRectInView(element).top -
                getRectInView(document.documentElement).top -
                (offset || 0);
            // trigger any function listening for "onscroll" event
            window.dispatchEvent(new Event('scroll'));
            window.scrollTo(0, y);
            document.activeElement.blur();
            element.focus();
        }

        // get element that is target of hash
        function getHashTarget(link) {
            const hash = link ? link.hash : window.location.hash;
            const id = hash.slice(1);
            let target = document.querySelector('[id="' + id + '"]');
            if (!target)
                return;

            // if figure or table, modify target to get expected element
            if (id.indexOf('fig:') === 0)
                target = target.querySelector('figure');
            if (id.indexOf('tbl:') === 0)
                target = target.querySelector('table');

            return target;
        }

        // when <h2> heading is clicked
        function onHeadingClick(event) {
            // only collapse if <h2> itself is target of click (eg, user did
            // not click on anchor within <h2>)
            if (event.target === this)
                toggleCollapse(this);
        }

        // when arrow button is clicked
        function onArrowClick() {
            toggleCollapse(this.parentNode);
        }

        // collapse section if expanded, expand if collapsed
        function toggleCollapse(heading) {
            if (heading.dataset.collapsed === 'false')
                collapseHeading(heading);
            else
                expandHeading(heading);
        }

        // elements to exclude from collapse, such as table of contents panel,
        // hypothesis panel, etc
        const exclude = '#toc_panel, div.annotator-frame, #lightbox_overlay';

        // collapse section
        function collapseHeading(heading) {
            heading.setAttribute('data-collapsed', 'true');
            const children = getChildren(heading);
            for (const child of children)
                child.setAttribute('data-collapsed', 'true');
        }

        // expand section
        function expandHeading(heading) {
            heading.setAttribute('data-collapsed', 'false');
            const children = getChildren(heading);
            for (const child of children)
                child.setAttribute('data-collapsed', 'false');
        }

        // get list of elements between this <h2> and next <h2> or <h1>
        // ("children" of the <h2> section)
        function getChildren(heading) {
            return nextUntil(heading, 'h2, h1', exclude);
        }

        // get position/dimensions of element or viewport
        function getRectInView(element) {
            let rect = {};
            rect.left = 0;
            rect.top = 0;
            rect.right = document.documentElement.clientWidth;
            rect.bottom = document.documentElement.clientHeight;
            let style = {};

            if (element instanceof HTMLElement) {
                rect = element.getBoundingClientRect();
                style = window.getComputedStyle(element);
            }

            const margin = {};
            margin.left = parseFloat(style.marginLeftWidth) || 0;
            margin.top = parseFloat(style.marginTopWidth) || 0;
            margin.right = parseFloat(style.marginRightWidth) || 0;
            margin.bottom = parseFloat(style.marginBottomWidth) || 0;

            const border = {};
            border.left = parseFloat(style.borderLeftWidth) || 0;
            border.top = parseFloat(style.borderTopWidth) || 0;
            border.right = parseFloat(style.borderRightWidth) || 0;
            border.bottom = parseFloat(style.borderBottomWidth) || 0;

            const newRect = {};
            newRect.left = rect.left + margin.left + border.left;
            newRect.top = rect.top + margin.top + border.top;
            newRect.right = rect.right + margin.right + border.right;
            newRect.bottom = rect.bottom + margin.bottom + border.bottom;
            newRect.width = newRect.right - newRect.left;
            newRect.height = newRect.bottom - newRect.top;

            return newRect;
        }

        // get list of elements after a start element up to element matching
        // query
        function nextUntil(element, query, exclude) {
            const elements = [];
            while (element = element.nextElementSibling, element) {
                if (element.matches(query))
                    break;
                if (!element.matches(exclude))
                    elements.push(element);
            }
            return elements;
        }

        // get closest element before specified element that matches query
        function firstBefore(element, query) {
            while (
                element &&
                element !== document.body &&
                !element.matches(query)
            )
                element = element.previousElementSibling || element.parentNode;

            return element;
        }

        // check if element is part of collapsed heading
        function isCollapsed(element) {
            while (element && element !== document.body) {
                if (element.dataset.collapsed === 'true')
                    return true;
                element = element.parentNode;
            }
            return false;
        }

        // expand heading containing element if necesary
        function expandElement(element) {
            if (isCollapsed(element)) {
                const heading = firstBefore(element, 'h2');
                if (heading)
                    heading.click();
            }
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- angle down icon -->

<template class="icon_angle_down">
    <!-- modified from: https://fontawesome.com/icons/angle-down -->
    <svg width="16" height="16" viewBox="0 0 448 512">
        <path
            fill="currentColor"
            d="M207.029 381.476L12.686 187.132c-9.373-9.373-9.373-24.569 0-33.941l22.667-22.667c9.357-9.357 24.522-9.375 33.901-.04L224 284.505l154.745-154.021c9.379-9.335 24.544-9.317 33.901.04l22.667 22.667c9.373 9.373 9.373 24.569 0 33.941L240.971 381.476c-9.373 9.372-24.569 9.372-33.942 0z"
        ></path>
    </svg>
</template>
<!-- tooltips plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin makes it such that when the user hovers or
        // focuses a link to a citation or figure, a tooltip appears with a
        // preview of the reference content, along with arrows to navigate
        // between instances of the same reference in the document.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'tooltips';

        // default plugin options
        const options = {
            // whether user must click off to close tooltip instead of just
            // un-hovering
            clickClose: 'false',
            // delay (in ms) between opening and closing tooltip
            delay: '100',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            const links = getLinks();
            for (const link of links) {
                // attach hover and focus listeners to link
                link.addEventListener('mouseover', onLinkHover);
                link.addEventListener('mouseleave', onLinkUnhover);
                link.addEventListener('focus', onLinkFocus);
                link.addEventListener('touchend', onLinkTouch);
            }

            // attach mouse, key, and resize listeners to window
            window.addEventListener('mousedown', onClick);
            window.addEventListener('touchstart', onClick);
            window.addEventListener('keyup', onKeyUp);
            window.addEventListener('resize', onResize);
        }

        // when link is hovered
        function onLinkHover() {
            // function to open tooltip
            const delayOpenTooltip = function() {
                openTooltip(this);
            }.bind(this);

            // run open function after delay
            this.openTooltipTimer = window.setTimeout(
                delayOpenTooltip,
                options.delay
            );
        }

        // when mouse leaves link
        function onLinkUnhover() {
            // cancel opening tooltip
            window.clearTimeout(this.openTooltipTimer);

            // don't close on unhover if option specifies
            if (options.clickClose === 'true')
                return;

            // function to close tooltip
            const delayCloseTooltip = function() {
                // if tooltip open and if mouse isn't over tooltip, close
                const tooltip = document.getElementById('tooltip');
                if (tooltip && !tooltip.matches(':hover'))
                    closeTooltip();
            };

            // run close function after delay
            this.closeTooltipTimer = window.setTimeout(
                delayCloseTooltip,
                options.delay
            );
        }

        // when link is focused (tabbed to)
        function onLinkFocus(event) {
            openTooltip(this);
        }

        // when link is touched on touch screen
        function onLinkTouch(event) {
            // attempt to force hover state on first tap always, and trigger
            // regular link click (and navigation) on second tap
            if (event.target === document.activeElement)
                event.target.click();
            else {
                document.activeElement.blur();
                event.target.focus();
            }
            if (event.cancelable)
                event.preventDefault();
            event.stopPropagation();
            return false;
        }

        // when mouse is clicked anywhere in window
        function onClick(event) {
            closeTooltip();
        }

        // when key pressed
        function onKeyUp(event) {
            if (!event || !event.key)
                return;

            switch (event.key) {
                // trigger click of prev button
                case 'ArrowLeft':
                    const prevButton = document.getElementById(
                        'tooltip_prev_button'
                    );
                    if (prevButton)
                        prevButton.click();
                    break;
                // trigger click of next button
                case 'ArrowRight':
                    const nextButton = document.getElementById(
                        'tooltip_next_button'
                    );
                    if (nextButton)
                        nextButton.click();
                    break;
                // close on esc
                case 'Escape':
                    closeTooltip();
                    break;
            }
        }

        // when window is resized or zoomed
        function onResize() {
            closeTooltip();
        }

        // get all links of types we wish to handle
        function getLinks() {
            const queries = [];
            // exclude buttons, anchor links, toc links, etc
            const exclude =
                ':not(.button):not(.icon_button):not(.anchor):not(.toc_link)';
            queries.push('a[href^="#ref-"]' + exclude); // citation links
            queries.push('a[href^="#fig:"]' + exclude); // figure links
            const query = queries.join(', ');
            return document.querySelectorAll(query);
        }

        // get links with same target, get index of link in set, get total
        // same links
        function getSameLinks(link) {
            const sameLinks = [];
            const links = getLinks();
            for (const otherLink of links) {
                if (
                    otherLink.getAttribute('href') === link.getAttribute('href')
                )
                    sameLinks.push(otherLink);
            }

            return {
                elements: sameLinks,
                index: sameLinks.indexOf(link),
                total: sameLinks.length
            };
        }

        // open tooltip
        function openTooltip(link) {
            // delete tooltip if it exists, start fresh
            closeTooltip();

            // make tooltip element
            const tooltip = makeTooltip(link);

            // if source couldn't be found and tooltip not made, exit
            if (!tooltip)
                return;

            // make navbar elements
            const navBar = makeNavBar(link);
            if (navBar)
                tooltip.firstElementChild.appendChild(navBar);

            // attach tooltip to page
            document.body.appendChild(tooltip);

            // position tooltip
            const position = function() {
                positionTooltip(link);
            };
            position();

            // if tooltip contains images, position again after they've loaded
            const imgs = tooltip.querySelectorAll('img');
            for (const img of imgs)
                img.addEventListener('load', position);
        }

        // close (delete) tooltip
        function closeTooltip() {
            const tooltip = document.getElementById('tooltip');
            if (tooltip)
                tooltip.remove();
        }

        // make tooltip
        function makeTooltip(link) {
            // get target element that link points to
            const source = getSource(link);

            // if source can't be found, exit
            if (!source)
                return;

            // create new tooltip
            const tooltip = document.createElement('div');
            tooltip.id = 'tooltip';
            const tooltipContent = document.createElement('div');
            tooltipContent.id = 'tooltip_content';
            tooltip.appendChild(tooltipContent);

            // make copy of source node and put in tooltip
            const sourceCopy = makeCopy(source);
            tooltipContent.appendChild(sourceCopy);

            // attach mouse event listeners
            tooltip.addEventListener('click', onTooltipClick);
            tooltip.addEventListener('mousedown', onTooltipClick);
            tooltip.addEventListener('touchstart', onTooltipClick);
            tooltip.addEventListener('mouseleave', onTooltipUnhover);

            // (for interaction with lightbox plugin)
            // transfer click on tooltip copied img to original img
            const sourceImg = source.querySelector('img');
            const sourceCopyImg = sourceCopy.querySelector('img');
            if (sourceImg && sourceCopyImg) {
                const clickImg = function() {
                    sourceImg.click();
                    closeTooltip();
                };
                sourceCopyImg.addEventListener('click', clickImg);
            }

            return tooltip;
        }

        // make carbon copy of html dom element
        function makeCopy(source) {
            const sourceCopy = source.cloneNode(true);

            // delete elements marked with ignore (eg anchor and jump buttons)
            const deleteFromCopy = sourceCopy.querySelectorAll(
                '[data-ignore="true"]'
            );
            for (const element of deleteFromCopy)
                element.remove();

            // delete certain element attributes
            const attributes = [
                'id',
                'data-collapsed',
                'data-selected',
                'data-highlighted',
                'data-glow'
            ];
            for (const attribute of attributes) {
                sourceCopy.removeAttribute(attribute);
                const elements = sourceCopy.querySelectorAll(
                    '[' + attribute + ']'
                );
                for (const element of elements)
                    element.removeAttribute(attribute);
            }

            return sourceCopy;
        }

        // when tooltip is clicked
        function onTooltipClick(event) {
            // when user clicks on tooltip, stop click from transferring
            // outside of tooltip (eg, click off to close tooltip, or eg click
            // off to unhighlight same refs)
            event.stopPropagation();
        }

        // when tooltip is unhovered
        function onTooltipUnhover(event) {
            if (options.clickClose === 'true')
                return;

            // make sure new mouse/touch/focus no longer over tooltip or any
            // element within it
            const tooltip = document.getElementById('tooltip');
            if (!tooltip)
                return;
            if (this.contains(event.relatedTarget))
                return;

            closeTooltip();
        }

        // make nav bar to go betwen prev/next instances of same reference
        function makeNavBar(link) {
            // find other links to the same source
            const sameLinks = getSameLinks(link);

            // don't show nav bar when singular reference
            if (sameLinks.total <= 1)
                return;

            // find prev/next links with same target
            const prevLink = getPrevLink(link, sameLinks);
            const nextLink = getNextLink(link, sameLinks);

            // create nav bar
            const navBar = document.createElement('div');
            navBar.id = 'tooltip_nav_bar';
            const text = sameLinks.index + 1 + ' of ' + sameLinks.total;

            // create nav bar prev/next buttons
            const prevButton = document.createElement('button');
            const nextButton = document.createElement('button');
            prevButton.id = 'tooltip_prev_button';
            nextButton.id = 'tooltip_next_button';
            prevButton.title =
                'Jump to the previous occurence of this item in the document [←]';
            nextButton.title =
                'Jump to the next occurence of this item in the document [→]';
            prevButton.classList.add('icon_button');
            nextButton.classList.add('icon_button');
            prevButton.innerHTML = document.querySelector(
                '.icon_caret_left'
            ).innerHTML;
            nextButton.innerHTML = document.querySelector(
                '.icon_caret_right'
            ).innerHTML;
            navBar.appendChild(prevButton);
            navBar.appendChild(document.createTextNode(text));
            navBar.appendChild(nextButton);

            // attach click listeners to buttons
            prevButton.addEventListener('click', function() {
                onPrevNextClick(link, prevLink);
            });
            nextButton.addEventListener('click', function() {
                onPrevNextClick(link, nextLink);
            });

            return navBar;
        }

        // get previous link with same target
        function getPrevLink(link, sameLinks) {
            if (!sameLinks)
                sameLinks = getSameLinks(link);
            // wrap index to other side if < 1
            let index;
            if (sameLinks.index - 1 >= 0)
                index = sameLinks.index - 1;
            else
                index = sameLinks.total - 1;
            return sameLinks.elements[index];
        }

        // get next link with same target
        function getNextLink(link, sameLinks) {
            if (!sameLinks)
                sameLinks = getSameLinks(link);
            // wrap index to other side if > total
            let index;
            if (sameLinks.index + 1 <= sameLinks.total - 1)
                index = sameLinks.index + 1;
            else
                index = 0;
            return sameLinks.elements[index];
        }

        // get element that is target of link or url hash
        function getSource(link) {
            const hash = link ? link.hash : window.location.hash;
            const id = hash.slice(1);
            let target = document.querySelector('[id="' + id + '"]');
            if (!target)
                return;

            // if ref or figure, modify target to get expected element
            if (id.indexOf('ref-') === 0)
                target = target.querySelector('p');
            else if (id.indexOf('fig:') === 0)
                target = target.querySelector('figure');

            return target;
        }

        // when prev/next arrow button is clicked
        function onPrevNextClick(link, prevNextLink) {
            if (link && prevNextLink)
                goToElement(prevNextLink, window.innerHeight * 0.5);
        }

        // scroll to and focus element
        function goToElement(element, offset) {
            // expand accordion section if collapsed
            expandElement(element);
            const y =
                getRectInView(element).top -
                getRectInView(document.documentElement).top -
                (offset || 0);
            // trigger any function listening for "onscroll" event
            window.dispatchEvent(new Event('scroll'));
            window.scrollTo(0, y);
            document.activeElement.blur();
            element.focus();
        }

        // determine position to place tooltip based on link position in
        // viewport and tooltip size
        function positionTooltip(link, left, top) {
            const tooltipElement = document.getElementById('tooltip');
            if (!tooltipElement)
                return;

            // get convenient vars for position/dimensions of
            // link/tooltip/page/view
            link = getRectInPage(link);
            const tooltip = getRectInPage(tooltipElement);
            const view = getRectInPage();

            // horizontal positioning
            if (left)
                // use explicit value
                left = left;
            else if (link.left + tooltip.width < view.right)
                // fit tooltip to right of link
                left = link.left;
            else if (link.right - tooltip.width > view.left)
                // fit tooltip to left of link
                left = link.right - tooltip.width;
            // center tooltip in view
            else
                left = (view.right - view.left) / 2 - tooltip.width / 2;

            // vertical positioning
            if (top)
                // use explicit value
                top = top;
            else if (link.top - tooltip.height > view.top)
                // fit tooltip above link
                top = link.top - tooltip.height;
            else if (link.bottom + tooltip.height < view.bottom)
                // fit tooltip below link
                top = link.bottom;
            else {
                // center tooltip in view
                top = view.top + view.height / 2 - tooltip.height / 2;
                // nudge off of link to left/right if possible
                if (link.right + tooltip.width < view.right)
                    left = link.right;
                else if (link.left - tooltip.width > view.left)
                    left = link.left - tooltip.width;
            }

            tooltipElement.style.left = left + 'px';
            tooltipElement.style.top = top + 'px';
        }

        // get position/dimensions of element or viewport
        function getRectInView(element) {
            let rect = {};
            rect.left = 0;
            rect.top = 0;
            rect.right = document.documentElement.clientWidth;
            rect.bottom = document.documentElement.clientHeight;
            let style = {};

            if (element instanceof HTMLElement) {
                rect = element.getBoundingClientRect();
                style = window.getComputedStyle(element);
            }

            const margin = {};
            margin.left = parseFloat(style.marginLeftWidth) || 0;
            margin.top = parseFloat(style.marginTopWidth) || 0;
            margin.right = parseFloat(style.marginRightWidth) || 0;
            margin.bottom = parseFloat(style.marginBottomWidth) || 0;

            const border = {};
            border.left = parseFloat(style.borderLeftWidth) || 0;
            border.top = parseFloat(style.borderTopWidth) || 0;
            border.right = parseFloat(style.borderRightWidth) || 0;
            border.bottom = parseFloat(style.borderBottomWidth) || 0;

            const newRect = {};
            newRect.left = rect.left + margin.left + border.left;
            newRect.top = rect.top + margin.top + border.top;
            newRect.right = rect.right + margin.right + border.right;
            newRect.bottom = rect.bottom + margin.bottom + border.bottom;
            newRect.width = newRect.right - newRect.left;
            newRect.height = newRect.bottom - newRect.top;

            return newRect;
        }

        // get position of element relative to page
        function getRectInPage(element) {
            const rect = getRectInView(element);
            const body = getRectInView(document.body);

            const newRect = {};
            newRect.left = rect.left - body.left;
            newRect.top = rect.top - body.top;
            newRect.right = rect.right - body.left;
            newRect.bottom = rect.bottom - body.top;
            newRect.width = rect.width;
            newRect.height = rect.height;

            return newRect;
        }

        // (for interaction with accordion plugin)
        // get closest element before specified element that matches query
        function firstBefore(element, query) {
            while (
                element &&
                element !== document.body &&
                !element.matches(query)
            )
                element = element.previousElementSibling || element.parentNode;

            return element;
        }

        // (for interaction with accordion plugin)
        // check if element is part of collapsed heading
        function isCollapsed(element) {
            while (element && element !== document.body) {
                if (element.dataset.collapsed === 'true')
                    return true;
                element = element.parentNode;
            }
            return false;
        }

        // (for interaction with accordion plugin)
        // expand heading containing element if necesary
        function expandElement(element) {
            if (isCollapsed(element)) {
                const heading = firstBefore(element, 'h2');
                if (heading)
                    heading.click();
            }
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- caret left icon -->

<template class="icon_caret_left">
    <!-- modified from: https://fontawesome.com/icons/caret-left -->
    <svg width="16" height="16" viewBox="0 0 192 512">
        <path
            fill="currentColor"
            d="M192 127.338v257.324c0 17.818-21.543 26.741-34.142 14.142L29.196 270.142c-7.81-7.81-7.81-20.474 0-28.284l128.662-128.662c12.599-12.6 34.142-3.676 34.142 14.142z"
        ></path>
    </svg>
</template>

<!-- caret right icon -->

<template class="icon_caret_right">
    <!-- modified from: https://fontawesome.com/icons/caret-right -->
    <svg width="16" height="16" viewBox="0 0 192 512">
        <path
            fill="currentColor"
            d="M0 384.662V127.338c0-17.818 21.543-26.741 34.142-14.142l128.662 128.662c7.81 7.81 7.81 20.474 0 28.284L34.142 398.804C21.543 411.404 0 402.48 0 384.662z"
        ></path>
    </svg>
</template>
<!-- jump to first plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin adds a button next to each reference entry,
        // figure, and table that jumps the page to the first occurrence of a
        // link to that item in the manuscript.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'jumpToFirst';

        // default plugin options
        const options = {
            // whether to add buttons next to reference entries
            references: 'true',
            // whether to add buttons next to figures
            figures: 'true',
            // whether to add buttons next to tables
            tables: 'true',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            if (options.references !== 'false')
                makeReferenceButtons();
            if (options.figures !== 'false')
                makeFigureButtons();
            if (options.tables !== 'false')
                makeTableButtons();
        }

        // when jump button clicked
        function onButtonClick() {
            const first = getFirstOccurrence(this.dataset.id);
            if (!first)
                return;

            // update url hash so navigating "back" in history will return
            // user to jump button
            window.location.hash = this.dataset.id;
            // scroll to link
            window.setTimeout(function() {
                goToElement(first, window.innerHeight * 0.5);
            }, 0);
        }

        // get first occurence of link to item in document
        function getFirstOccurrence(id) {
            let query = 'a';
            query += '[href="#' + id + '"]';
            // exclude buttons, anchor links, toc links, etc
            query +=
                ':not(.button):not(.icon_button):not(.anchor):not(.toc_link)';
            return document.querySelector(query);
        }

        // add button next to each reference entry
        function makeReferenceButtons() {
            const references = document.querySelectorAll('div[id^="ref-"]');
            for (const reference of references) {
                // get reference id and element to add button to
                const id = reference.id;
                const container = reference.firstElementChild;
                const first = getFirstOccurrence(id);

                // if can't find link to reference, ignore
                if (!first)
                    continue;

                // make jump button
                let button = document.createElement('button');
                button.classList.add('icon_button', 'jump_arrow');
                button.title =
                    'Jump to the first occurence of this reference in the document';
                button.innerHTML = document.querySelector(
                    '.icon_angle_double_up'
                ).innerHTML;
                button.dataset.id = id;
                button.dataset.ignore = 'true';
                container.innerHTML = button.outerHTML + container.innerHTML;
                button = container.firstElementChild;
                button.addEventListener('click', onButtonClick);
            }
        }

        // add button next to each figure
        function makeFigureButtons() {
            const figures = document.querySelectorAll('[id^="fig:"]');
            for (const figure of figures) {
                // get figure id and element to add button to
                const id = figure.id;
                const container = figure.querySelector('figcaption') || figure;
                const first = getFirstOccurrence(id);

                // if can't find link to figure, ignore
                if (!first)
                    continue;

                // make jump button
                const button = document.createElement('button');
                button.classList.add('icon_button', 'jump_arrow');
                button.title =
                    'Jump to the first occurence of this figure in the document';
                button.innerHTML = document.querySelector(
                    '.icon_angle_double_up'
                ).innerHTML;
                button.dataset.id = id;
                button.dataset.ignore = 'true';
                container.insertBefore(button, container.firstElementChild);
                button.addEventListener('click', onButtonClick);
            }
        }

        // add button next to each figure
        function makeTableButtons() {
            const tables = document.querySelectorAll('[id^="tbl:"]');
            for (const table of tables) {
                // get ref id and element to add button to
                const id = table.id;
                const container = table.querySelector('caption') || table;
                const first = getFirstOccurrence(id);

                // if can't find link to table, ignore
                if (!first)
                    continue;

                // make jump button
                const button = document.createElement('button');
                button.classList.add('icon_button', 'jump_arrow');
                button.title =
                    'Jump to the first occurence of this table in the document';
                button.innerHTML = document.querySelector(
                    '.icon_angle_double_up'
                ).innerHTML;
                button.dataset.id = id;
                button.dataset.ignore = 'true';
                container.insertBefore(button, container.firstElementChild);
                button.addEventListener('click', onButtonClick);
            }
        }

        // scroll to and focus element
        function goToElement(element, offset) {
            // expand accordion section if collapsed
            expandElement(element);
            const y =
                getRectInView(element).top -
                getRectInView(document.documentElement).top -
                (offset || 0);
            // trigger any function listening for "onscroll" event
            window.dispatchEvent(new Event('scroll'));
            window.scrollTo(0, y);
            document.activeElement.blur();
            element.focus();
        }

        // get position/dimensions of element or viewport
        function getRectInView(element) {
            let rect = {};
            rect.left = 0;
            rect.top = 0;
            rect.right = document.documentElement.clientWidth;
            rect.bottom = document.documentElement.clientHeight;
            let style = {};

            if (element instanceof HTMLElement) {
                rect = element.getBoundingClientRect();
                style = window.getComputedStyle(element);
            }

            const margin = {};
            margin.left = parseFloat(style.marginLeftWidth) || 0;
            margin.top = parseFloat(style.marginTopWidth) || 0;
            margin.right = parseFloat(style.marginRightWidth) || 0;
            margin.bottom = parseFloat(style.marginBottomWidth) || 0;

            const border = {};
            border.left = parseFloat(style.borderLeftWidth) || 0;
            border.top = parseFloat(style.borderTopWidth) || 0;
            border.right = parseFloat(style.borderRightWidth) || 0;
            border.bottom = parseFloat(style.borderBottomWidth) || 0;

            const newRect = {};
            newRect.left = rect.left + margin.left + border.left;
            newRect.top = rect.top + margin.top + border.top;
            newRect.right = rect.right + margin.right + border.right;
            newRect.bottom = rect.bottom + margin.bottom + border.bottom;
            newRect.width = newRect.right - newRect.left;
            newRect.height = newRect.bottom - newRect.top;

            return newRect;
        }

        // get closest element before specified element that matches query
        function firstBefore(element, query) {
            while (
                element &&
                element !== document.body &&
                !element.matches(query)
            )
                element = element.previousElementSibling || element.parentNode;

            return element;
        }

        // check if element is part of collapsed heading
        function isCollapsed(element) {
            while (element && element !== document.body) {
                if (element.dataset.collapsed === 'true')
                    return true;
                element = element.parentNode;
            }
            return false;
        }

        // (for interaction with accordion plugin)
        // expand heading containing element if necesary
        function expandElement(element) {
            if (isCollapsed(element)) {
                const heading = firstBefore(element, 'h2');
                if (heading)
                    heading.click();
            }
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- angle double up icon -->

<template class="icon_angle_double_up">
    <!-- modified from: https://fontawesome.com/icons/angle-double-up -->
    <svg width="16" height="16" viewBox="0 0 320 512">
        <path
            fill="currentColor"
            d="M177 255.7l136 136c9.4 9.4 9.4 24.6 0 33.9l-22.6 22.6c-9.4 9.4-24.6 9.4-33.9 0L160 351.9l-96.4 96.4c-9.4 9.4-24.6 9.4-33.9 0L7 425.7c-9.4-9.4-9.4-24.6 0-33.9l136-136c9.4-9.5 24.6-9.5 34-.1zm-34-192L7 199.7c-9.4 9.4-9.4 24.6 0 33.9l22.6 22.6c9.4 9.4 24.6 9.4 33.9 0l96.4-96.4 96.4 96.4c9.4 9.4 24.6 9.4 33.9 0l22.6-22.6c9.4-9.4 9.4-24.6 0-33.9l-136-136c-9.2-9.4-24.4-9.4-33.8 0z"
        ></path>
    </svg>
</template>
<!-- link highlight plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin makes it such that when a user hovers or
        // focuses a link, other links that have the same target will be
        // highlighted. It also makes it such that when clicking a link, the
        // target of the link (eg reference, figure, table) is briefly
        // highlighted.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'linkHighlight';

        // default plugin options
        const options = {
            // whether to also highlight links that go to external urls
            externalLinks: 'false',
            // whether user must click off to unhighlight instead of just
            // un-hovering
            clickUnhighlight: 'false',
            // whether to also highlight links that are unique
            highlightUnique: 'true',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            const links = getLinks();
            for (const link of links) {
                // attach mouse and focus listeners to link
                link.addEventListener('mouseenter', onLinkFocus);
                link.addEventListener('focus', onLinkFocus);
                link.addEventListener('mouseleave', onLinkUnhover);
            }

            // attach click and hash change listeners to window
            window.addEventListener('click', onClick);
            window.addEventListener('touchstart', onClick);
            window.addEventListener('hashchange', onHashChange);

            // run hash change on window load in case user has navigated
            // directly to hash
            onHashChange();
        }

        // when link is focused (tabbed to) or hovered
        function onLinkFocus() {
            highlight(this);
        }

        // when link is unhovered
        function onLinkUnhover() {
            if (options.clickUnhighlight !== 'true')
                unhighlightAll();
        }

        // when the mouse is clicked anywhere in window
        function onClick(event) {
            unhighlightAll();
        }

        // when hash (eg manuscript.html#introduction) changes
        function onHashChange() {
            const target = getHashTarget();
            if (target)
                glowElement(target);
        }

        // get element that is target of link or url hash
        function getHashTarget(link) {
            const hash = link ? link.hash : window.location.hash;
            const id = hash.slice(1);
            let target = document.querySelector('[id="' + id + '"]');
            if (!target)
                return;

            return target;
        }

        // start glow sequence on an element
        function glowElement(element) {
            const startGlow = function() {
                onGlowEnd();
                element.dataset.glow = 'true';
                element.addEventListener('animationend', onGlowEnd);
            };
            const onGlowEnd = function() {
                element.removeAttribute('data-glow');
                element.removeEventListener('animationend', onGlowEnd);
            };
            startGlow();
        }

        // highlight link and all others with same target
        function highlight(link) {
            // force unhighlight all to start fresh
            unhighlightAll();

            // get links with same target
            if (!link)
                return;
            const sameLinks = getSameLinks(link);

            // if link unique and option is off, exit and don't highlight
            if (sameLinks.length <= 1 && options.highlightUnique !== 'true')
                return;

            // highlight all same links, and "select" (special highlight) this
            // one
            for (const sameLink of sameLinks) {
                if (sameLink === link)
                    sameLink.setAttribute('data-selected', 'true');
                else
                    sameLink.setAttribute('data-highlighted', 'true');
            }
        }

        // unhighlight all links
        function unhighlightAll() {
            const links = getLinks();
            for (const link of links) {
                link.setAttribute('data-selected', 'false');
                link.setAttribute('data-highlighted', 'false');
            }
        }

        // get links with same target
        function getSameLinks(link) {
            const results = [];
            const links = getLinks();
            for (const otherLink of links) {
                if (
                    otherLink.getAttribute('href') === link.getAttribute('href')
                )
                    results.push(otherLink);
            }
            return results;
        }

        // get all links of types we wish to handle
        function getLinks() {
            let query = 'a';
            if (options.externalLinks !== 'true')
                query += '[href^="#"]';
            // exclude buttons, anchor links, toc links, etc
            query +=
                ':not(.button):not(.icon_button):not(.anchor):not(.toc_link)';
            return document.querySelectorAll(query);
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>
<!-- table of contents plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin provides a "table of contents" (toc) panel on
        // the side of the document that allows the user to conveniently
        // navigate between sections of the document.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'tableOfContents';

        // default plugin options
        const options = {
            // which types of elements to add links for, in
            // "document.querySelector" format
            typesQuery: 'h1, h2, h3',
            // whether default behavior is to be closed ('false'), open
            // ('true'), or only open when screen wide enough to fit panel
            // ('auto'). note: still always starts closed when page loads.
            open: 'auto',
            // if list item is more than this many characters, text will be
            // truncated
            charLimit: '50',
            // whether or not to show bullets next to each toc item
            bullets: 'false',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            // make toc panel and populate with entries (links to document
            // sections)
            const panel = makePanel();
            if (!panel)
                return;
            makeEntries(panel);
            document.body.insertBefore(panel, document.body.firstChild);

            closePanel();

            // attach click, scroll, and hash change listeners to window
            window.addEventListener('click', onClick);
            window.addEventListener('touchstart', onClick);
            window.addEventListener('scroll', onScroll);
            window.addEventListener('hashchange', onScroll);
            window.addEventListener('keyup', onKeyUp);
            onScroll();

            // add class to push document body down out of way of toc button
            document.body.classList.add('toc_body_nudge');
        }

        // determine if screen wide enough to fit toc panel
        function isSmallScreen() {
            // in default theme:
            // 816px = 8.5in = width of "page" (<body>) element
            // 260px = min width of toc panel (*2 for both sides of <body>)
            return window.innerWidth < 816 + 260 * 2;
        }

        // open/close panel based on option and screen size
        function openOrClosePanel() {
            if (
                options.open === 'true' ||
                (options.open === 'auto' && !isSmallScreen())
            )
                openPanel();
            else
                closePanel();
        }

        // when mouse is clicked anywhere in window
        function onClick() {
            const panel = document.getElementById('toc_panel');
            if (!panel)
                return;

            if (panel.dataset.open === 'true')
                openOrClosePanel();
        }

        // when window is scrolled or hash changed
        function onScroll() {
            highlightViewed();
        }

        // when key pressed
        function onKeyUp(event) {
            if (!event || !event.key)
                return;

            // close on esc
            if (event.key === 'Escape')
                closePanel();
        }

        // find entry of currently viewed document section in toc and highlight
        function highlightViewed() {
            const firstId = getFirstInView(options.typesQuery);

            // get toc entries (links), unhighlight all, then highlight viewed
            const list = document.getElementById('toc_list');
            if (!firstId || !list)
                return;
            const links = list.querySelectorAll('a');
            for (const link of links)
                link.dataset.viewing = 'false';
            const link = list.querySelector('a[href="#' + firstId + '"]');
            if (!link)
                return;
            link.dataset.viewing = 'true';
        }

        // get first or previous toc listed element in top half of view
        function getFirstInView(query) {
            // get all elements matching query and with id
            const elements = document.querySelectorAll(query);
            const elementsWithIds = [];
            for (const element of elements) {
                if (element.id)
                    elementsWithIds.push(element);
            }


            // get first or previous element in top half of view
            for (let i = 0; i < elementsWithIds.length; i++) {
                const element = elementsWithIds[i];
                const prevElement = elementsWithIds[Math.max(0, i - 1)];
                if (element.getBoundingClientRect().top >= 0) {
                    if (
                        element.getBoundingClientRect().top <
                        window.innerHeight / 2
                    )
                        return element.id;
                    else
                        return prevElement.id;
                }
            }
        }

        // make panel
        function makePanel() {
            // create panel
            const panel = document.createElement('div');
            panel.id = 'toc_panel';
            if (options.bullets === 'true')
                panel.dataset.bullets = 'true';

            // create header
            const header = document.createElement('div');
            header.id = 'toc_header';

            // create toc button
            const button = document.createElement('button');
            button.id = 'toc_button';
            button.innerHTML = document.querySelector(
                '.icon_th_list'
            ).innerHTML;
            button.classList.add('icon_button');

            // create header text
            const text = document.createElement('h3');
            text.innerHTML = 'Table of Contents';

            // create container for toc list
            const list = document.createElement('div');
            list.id = 'toc_list';

            // attach click listeners
            panel.addEventListener('click', onPanelClick);
            header.addEventListener('click', onHeaderClick);
            button.addEventListener('click', onButtonClick);

            // attach elements
            header.appendChild(button);
            header.appendChild(text);
            panel.appendChild(header);
            panel.appendChild(list);

            return panel;
        }

        // create toc entries (links) to each element of the specified types
        function makeEntries(panel) {
            const elements = document.querySelectorAll(options.typesQuery);
            for (const element of elements) {
                // do not add link if element doesn't have assigned id
                if (!element.id)
                    continue;

                // create link/list item
                const link = document.createElement('a');
                link.classList.add('toc_link');
                switch (element.tagName.toLowerCase()) {
                    case 'h1':
                        link.dataset.level = '1';
                        break;
                    case 'h2':
                        link.dataset.level = '2';
                        break;
                    case 'h3':
                        link.dataset.level = '3';
                        break;
                    case 'h4':
                        link.dataset.level = '4';
                        break;
                }
                link.title = element.innerText;
                let text = element.innerText;
                if (text.length > options.charLimit)
                    text = text.slice(0, options.charLimit) + '...';
                link.innerHTML = text;
                link.href = '#' + element.id;
                link.addEventListener('click', onLinkClick);

                // attach link
                panel.querySelector('#toc_list').appendChild(link);
            }
        }

        // when panel is clicked
        function onPanelClick(event) {
            // stop click from propagating to window/document and closing panel
            event.stopPropagation();
        }

        // when header itself is clicked
        function onHeaderClick(event) {
            togglePanel();
        }

        // when button is clicked
        function onButtonClick(event) {
            togglePanel();
            // stop header underneath button from also being clicked
            event.stopPropagation();
        }

        // when link is clicked
        function onLinkClick() {
            openOrClosePanel();
        }

        // open panel if closed, close if opened
        function togglePanel() {
            const panel = document.getElementById('toc_panel');
            if (!panel)
                return;

            if (panel.dataset.open === 'true')
                closePanel();
            else
                openPanel();
        }

        // open panel
        function openPanel() {
            const panel = document.getElementById('toc_panel');
            if (panel)
                panel.dataset.open = 'true';
        }

        // close panel
        function closePanel() {
            const panel = document.getElementById('toc_panel');
            if (panel)
                panel.dataset.open = 'false';
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- th list icon -->

<template class="icon_th_list">
    <!-- modified from: https://fontawesome.com/icons/th-list -->
    <svg width="16" height="16" viewBox="0 0 512 512" tabindex="-1">
        <path
            fill="currentColor"
            d="M96 96c0 26.51-21.49 48-48 48S0 122.51 0 96s21.49-48 48-48 48 21.49 48 48zM48 208c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48zm0 160c-26.51 0-48 21.49-48 48s21.49 48 48 48 48-21.49 48-48-21.49-48-48-48zm96-236h352c8.837 0 16-7.163 16-16V76c0-8.837-7.163-16-16-16H144c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16zm0 160h352c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H144c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16zm0 160h352c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H144c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16z"
            tabindex="-1"
        ></path>
    </svg>
</template>
<!-- lightbox plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin makes it such that when a user clicks on an
        // image, the image fills the screen and the user can pan/drag/zoom
        // the image and navigate between other images in the document.

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'lightbox';

        // default plugin options
        const options = {
            // list of possible zoom/scale factors
            zoomSteps:
                '0.1, 0.25, 0.333333, 0.5, 0.666666, 0.75, 1,' +
                '1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, 5, 6, 7, 8',
            // whether to fit image to view ('fit'), display at 100% and shrink
            // if necessary ('shrink'), or always display at 100% ('100')
            defaultZoom: 'fit',
            // whether to zoom in/out toward center of view ('true') or mouse
            // ('false')
            centerZoom: 'false',
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            // run through each <img> element
            const imgs = document.querySelectorAll('figure > img');
            let count = 1;
            for (const img of imgs) {
                img.classList.add('lightbox_document_img');
                img.dataset.number = count;
                img.dataset.total = imgs.length;
                img.addEventListener('click', openLightbox);
                count++;
            }

            // attach mouse and key listeners to window
            window.addEventListener('mousemove', onWindowMouseMove);
            window.addEventListener('keyup', onKeyUp);
        }

        // when mouse is moved anywhere in window
        function onWindowMouseMove(event) {
            window.mouseX = event.clientX;
            window.mouseY = event.clientY;
        }

        // when key pressed
        function onKeyUp(event) {
            if (!event || !event.key)
                return;

            switch (event.key) {
                // trigger click of prev button
                case 'ArrowLeft':
                    const prevButton = document.getElementById(
                        'lightbox_prev_button'
                    );
                    if (prevButton)
                        prevButton.click();
                    break;
                // trigger click of next button
                case 'ArrowRight':
                    const nextButton = document.getElementById(
                        'lightbox_next_button'
                    );
                    if (nextButton)
                        nextButton.click();
                    break;
                // close on esc
                case 'Escape':
                    closeLightbox();
                    break;
            }
        }

        // open lightbox
        function openLightbox() {
            const lightbox = makeLightbox(this);
            if (!lightbox)
                return;

            blurBody(lightbox);
            document.body.appendChild(lightbox);
        }

        // make lightbox
        function makeLightbox(img) {
            // delete lightbox if it exists, start fresh
            closeLightbox();

            // create screen overlay containing lightbox
            const overlay = document.createElement('div');
            overlay.id = 'lightbox_overlay';

            // create image info boxes
            const numberInfo = document.createElement('div');
            const zoomInfo = document.createElement('div');
            numberInfo.id = 'lightbox_number_info';
            zoomInfo.id = 'lightbox_zoom_info';

            // create container for image
            const imageContainer = document.createElement('div');
            imageContainer.id = 'lightbox_image_container';
            const lightboxImg = makeLightboxImg(
                img,
                imageContainer,
                numberInfo,
                zoomInfo
            );
            imageContainer.appendChild(lightboxImg);

            // create bottom container for caption and navigation buttons
            const bottomContainer = document.createElement('div');
            bottomContainer.id = 'lightbox_bottom_container';
            const caption = makeCaption(img);
            const prevButton = makePrevButton(img);
            const nextButton = makeNextButton(img);
            bottomContainer.appendChild(prevButton);
            bottomContainer.appendChild(caption);
            bottomContainer.appendChild(nextButton);

            // attach top middle and bottom to overlay
            overlay.appendChild(numberInfo);
            overlay.appendChild(zoomInfo);
            overlay.appendChild(imageContainer);
            overlay.appendChild(bottomContainer);

            return overlay;
        }

        // make <img> object that is intuitively draggable and zoomable
        function makeLightboxImg(
            sourceImg,
            container,
            numberInfoBox,
            zoomInfoBox
        ) {
            // create copy of source <img>
            const img = sourceImg.cloneNode(true);
            img.classList.remove('lightbox_document_img');
            img.removeAttribute('id');
            img.removeAttribute('width');
            img.removeAttribute('height');
            img.style.position = 'unset';
            img.style.margin = '0';
            img.style.padding = '0';
            img.style.width = '';
            img.style.height = '';
            img.style.minWidth = '';
            img.style.minHeight = '';
            img.style.maxWidth = '';
            img.style.maxHeight = '';
            img.id = 'lightbox_img';

            // build sorted list of unique zoomSteps, always including a 100%
            let zoomSteps = [];
            const optionsZooms = options.zoomSteps.split(/[^0-9.]/);
            for (const optionZoom of optionsZooms) {
                const newZoom = parseFloat(optionZoom);
                if (newZoom && !zoomSteps.includes(newZoom))
                    zoomSteps.push(newZoom);
            }
            if (!zoomSteps.includes(1))
                zoomSteps.push(1);
            zoomSteps = zoomSteps.sort(function sortNumber(a, b) {
                return a - b;
            });

            // <img> object property variables
            let zoom = 1;
            let translateX = 0;
            let translateY = 0;
            let clickMouseX = undefined;
            let clickMouseY = undefined;
            let clickTranslateX = undefined;
            let clickTranslateY = undefined;

            updateNumberInfo();

            // update image numbers displayed in info box
            function updateNumberInfo() {
                numberInfoBox.innerHTML =
                    sourceImg.dataset.number + ' of ' + sourceImg.dataset.total;
            }

            // update zoom displayed in info box
            function updateZoomInfo() {
                let zoomInfo = zoom * 100;
                if (!Number.isInteger(zoomInfo))
                    zoomInfo = zoomInfo.toFixed(2);
                zoomInfoBox.innerHTML = zoomInfo + '%';
            }

            // move to closest zoom step above current zoom
            const zoomIn = function() {
                for (const zoomStep of zoomSteps) {
                    if (zoomStep > zoom) {
                        zoom = zoomStep;
                        break;
                    }
                }
                updateTransform();
            };

            // move to closest zoom step above current zoom
            const zoomOut = function() {
                zoomSteps.reverse();
                for (const zoomStep of zoomSteps) {
                    if (zoomStep < zoom) {
                        zoom = zoomStep;
                        break;
                    }
                }
                zoomSteps.reverse();

                updateTransform();
            };

            // update display of <img> based on scale/translate properties
            const updateTransform = function() {
                // set transform
                img.style.transform =
                    'translate(' +
                    (translateX || 0) +
                    'px,' +
                    (translateY || 0) +
                    'px) scale(' +
                    (zoom || 1) +
                    ')';

                // get new width/height after scale
                const rect = img.getBoundingClientRect();
                // limit translate
                translateX = Math.max(translateX, -rect.width / 2);
                translateX = Math.min(translateX, rect.width / 2);
                translateY = Math.max(translateY, -rect.height / 2);
                translateY = Math.min(translateY, rect.height / 2);

                // set transform
                img.style.transform =
                    'translate(' +
                    (translateX || 0) +
                    'px,' +
                    (translateY || 0) +
                    'px) scale(' +
                    (zoom || 1) +
                    ')';

                updateZoomInfo();
            };

            // fit <img> to container
            const fit = function() {
                // no x/y offset, 100% zoom by default
                translateX = 0;
                translateY = 0;
                zoom = 1;

                // widths of <img> and container
                const imgWidth = img.naturalWidth;
                const imgHeight = img.naturalHeight;
                const containerWidth = parseFloat(
                    window.getComputedStyle(container).width
                );
                const containerHeight = parseFloat(
                    window.getComputedStyle(container).height
                );

                // how much zooming is needed to fit <img> to container
                const xRatio = imgWidth / containerWidth;
                const yRatio = imgHeight / containerHeight;
                const maxRatio = Math.max(xRatio, yRatio);
                const newZoom = 1 / maxRatio;

                // fit <img> to container according to option
                if (options.defaultZoom === 'shrink') {
                    if (maxRatio > 1)
                        zoom = newZoom;
                } else if (options.defaultZoom === 'fit')
                    zoom = newZoom;

                updateTransform();
            };

            // when mouse wheel is rolled anywhere in container
            const onContainerWheel = function(event) {
                if (!event)
                    return;

                // let ctrl + mouse wheel to zoom behave as normal
                if (event.ctrlKey)
                    return;

                // prevent normal scroll behavior
                event.preventDefault();
                event.stopPropagation();

                // point around which to scale img
                const viewRect = container.getBoundingClientRect();
                const viewX = (viewRect.left + viewRect.right) / 2;
                const viewY = (viewRect.top + viewRect.bottom) / 2;
                const originX = options.centerZoom === 'true' ? viewX : mouseX;
                const originY = options.centerZoom === 'true' ? viewY : mouseY;

                // get point on image under origin
                const oldRect = img.getBoundingClientRect();
                const oldPercentX = (originX - oldRect.left) / oldRect.width;
                const oldPercentY = (originY - oldRect.top) / oldRect.height;

                // increment/decrement zoom
                if (event.deltaY < 0)
                    zoomIn();
                if (event.deltaY > 0)
                    zoomOut();

                // get offset between previous image point and origin
                const newRect = img.getBoundingClientRect();
                const offsetX =
                    originX - (newRect.left + newRect.width * oldPercentX);
                const offsetY =
                    originY - (newRect.top + newRect.height * oldPercentY);

                // translate image to keep image point under origin
                translateX += offsetX;
                translateY += offsetY;

                // perform translate
                updateTransform();
            };

            // when container is clicked
            function onContainerClick(event) {
                // if container itself is target of click, and not other
                // element above it
                if (event.target === this)
                    closeLightbox();
            }

            // when mouse button is pressed on image
            const onImageMouseDown = function(event) {
                // store original mouse position relative to image
                clickMouseX = window.mouseX;
                clickMouseY = window.mouseY;
                clickTranslateX = translateX;
                clickTranslateY = translateY;
                event.stopPropagation();
                event.preventDefault();
            };

            // when mouse button is released anywhere in window
            const onWindowMouseUp = function(event) {
                // reset original mouse position
                clickMouseX = undefined;
                clickMouseY = undefined;
                clickTranslateX = undefined;
                clickTranslateY = undefined;

                // remove global listener if lightbox removed from document
                if (!document.body.contains(container))
                    window.removeEventListener('mouseup', onWindowMouseUp);
            };

            // when mouse is moved anywhere in window
            const onWindowMouseMove = function(event) {
                if (
                    clickMouseX === undefined ||
                    clickMouseY === undefined ||
                    clickTranslateX === undefined ||
                    clickTranslateY === undefined
                )
                    return;

                // offset image based on original and current mouse position
                translateX = clickTranslateX + window.mouseX - clickMouseX;
                translateY = clickTranslateY + window.mouseY - clickMouseY;
                updateTransform();
                event.preventDefault();

                // remove global listener if lightbox removed from document
                if (!document.body.contains(container))
                    window.removeEventListener('mousemove', onWindowMouseMove);
            };

            // when window is resized
            const onWindowResize = function(event) {
                fit();

                // remove global listener if lightbox removed from document
                if (!document.body.contains(container))
                    window.removeEventListener('resize', onWindowResize);
            };

            // attach the necessary event listeners
            img.addEventListener('dblclick', fit);
            img.addEventListener('mousedown', onImageMouseDown);
            container.addEventListener('wheel', onContainerWheel);
            container.addEventListener('mousedown', onContainerClick);
            container.addEventListener('touchstart', onContainerClick);
            window.addEventListener('mouseup', onWindowMouseUp);
            window.addEventListener('mousemove', onWindowMouseMove);
            window.addEventListener('resize', onWindowResize);

            // run fit() after lightbox atttached to document and <img> Loaded
            // so needed container and img dimensions available
            img.addEventListener('load', fit);

            return img;
        }

        // make caption
        function makeCaption(img) {
            const caption = document.createElement('div');
            caption.id = 'lightbox_caption';
            const captionSource = img.nextElementSibling;
            if (captionSource.tagName.toLowerCase() === 'figcaption') {
                const captionCopy = makeCopy(captionSource);
                caption.innerHTML = captionCopy.innerHTML;
            }

            caption.addEventListener('touchstart', function(event) {
                event.stopPropagation();
            });

            return caption;
        }

        // make carbon copy of html dom element
        function makeCopy(source) {
            const sourceCopy = source.cloneNode(true);

            // delete elements marked with ignore (eg anchor and jump buttons)
            const deleteFromCopy = sourceCopy.querySelectorAll(
                '[data-ignore="true"]'
            );
            for (const element of deleteFromCopy)
                element.remove();

            // delete certain element attributes
            const attributes = [
                'id',
                'data-collapsed',
                'data-selected',
                'data-highlighted',
                'data-glow'
            ];
            for (const attribute of attributes) {
                sourceCopy.removeAttribute(attribute);
                const elements = sourceCopy.querySelectorAll(
                    '[' + attribute + ']'
                );
                for (const element of elements)
                    element.removeAttribute(attribute);
            }

            return sourceCopy;
        }

        // make button to jump to previous image in document
        function makePrevButton(img) {
            const prevButton = document.createElement('button');
            prevButton.id = 'lightbox_prev_button';
            prevButton.title = 'Jump to the previous image in the document [←]';
            prevButton.classList.add('icon_button', 'lightbox_button');
            prevButton.innerHTML = document.querySelector(
                '.icon_caret_left'
            ).innerHTML;

            // attach click listeners to button
            prevButton.addEventListener('click', function() {
                getPrevImg(img).click();
            });

            return prevButton;
        }

        // make button to jump to next image in document
        function makeNextButton(img) {
            const nextButton = document.createElement('button');
            nextButton.id = 'lightbox_next_button';
            nextButton.title = 'Jump to the next image in the document [→]';
            nextButton.classList.add('icon_button', 'lightbox_button');
            nextButton.innerHTML = document.querySelector(
                '.icon_caret_right'
            ).innerHTML;

            // attach click listeners to button
            nextButton.addEventListener('click', function() {
                getNextImg(img).click();
            });

            return nextButton;
        }

        // get previous image in document
        function getPrevImg(img) {
            const imgs = document.querySelectorAll('.lightbox_document_img');

            // find index of provided img
            let index;
            for (index = 0; index < imgs.length; index++) {
                if (imgs[index] === img)
                    break;
            }


            // wrap index to other side if < 1
            if (index - 1 >= 0)
                index--;
            else
                index = imgs.length - 1;
            return imgs[index];
        }

        // get next image in document
        function getNextImg(img) {
            const imgs = document.querySelectorAll('.lightbox_document_img');

            // find index of provided img
            let index;
            for (index = 0; index < imgs.length; index++) {
                if (imgs[index] === img)
                    break;
            }


            // wrap index to other side if > total
            if (index + 1 <= imgs.length - 1)
                index++;
            else
                index = 0;
            return imgs[index];
        }

        // close lightbox
        function closeLightbox() {
            focusBody();

            const lightbox = document.getElementById('lightbox_overlay');
            if (lightbox)
                lightbox.remove();
        }

        // make all elements behind lightbox non-focusable
        function blurBody(overlay) {
            const all = document.querySelectorAll('*');
            for (const element of all)
                element.tabIndex = -1;
            document.body.classList.add('body_no_scroll');
        }

        // make all elements focusable again
        function focusBody() {
            const all = document.querySelectorAll('*');
            for (const element of all)
                element.removeAttribute('tabIndex');
            document.body.classList.remove('body_no_scroll');
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>

<!-- caret left icon -->

<template class="icon_caret_left">
    <!-- modified from: https://fontawesome.com/icons/caret-left -->
    <svg width="16" height="16" viewBox="0 0 192 512">
        <path
            fill="currentColor"
            d="M192 127.338v257.324c0 17.818-21.543 26.741-34.142 14.142L29.196 270.142c-7.81-7.81-7.81-20.474 0-28.284l128.662-128.662c12.599-12.6 34.142-3.676 34.142 14.142z"
        ></path>
    </svg>
</template>

<!-- caret right icon -->

<template class="icon_caret_right">
    <!-- modified from: https://fontawesome.com/icons/caret-right -->
    <svg width="16" height="16" viewBox="0 0 192 512">
        <path
            fill="currentColor"
            d="M0 384.662V127.338c0-17.818 21.543-26.741 34.142-14.142l128.662 128.662c7.81 7.81 7.81 20.474 0 28.284L34.142 398.804C21.543 411.404 0 402.48 0 384.662z"
        ></path>
    </svg>
</template>
<!-- attributes plugin -->

<script>
    (function() {
        // /////////////////////////
        // DESCRIPTION
        // /////////////////////////

        // This Manubot plugin allows arbitrary HTML attributes to be attached
        // to (almost) any element. Place an HTML comment inside or next to the
        // desired element in the format <!-- $attribute="value" -->

        // /////////////////////////
        // OPTIONS
        // /////////////////////////

        // plugin name prefix for url parameters
        const pluginName = 'attributes';

        // default plugin options
        const options = {
            // whether plugin is on or not
            enabled: 'true'
        };

        // change options above, or override with url parameter, eg:
        // 'manuscript.html?pluginName-enabled=false'

        // /////////////////////////
        // SCRIPT
        // /////////////////////////

        // start script
        function start() {
            // get list of comments in document
            const comments = findComments();

            for(const comment of comments)
                if (comment.parentElement)
                    addAttributes(
                        comment.parentElement,
                        comment.nodeValue.trim()
                    );
        }

        // add html attributes to specified element based on string of 
        // html attributes and values
        function addAttributes(element, text) {
            // regex's for finding attribute/value pairs in the format of
            // attribute="value" or attribute='value
            const regex2 = /\$([a-zA-Z\-]+)?=\"(.+?)\"/;
            const regex1 = /\$([a-zA-Z\-]+)?=\'(.+?)\'/;

            // loop through attribute/value pairs
            let match;
            while(match = text.match(regex2) || text.match(regex1)) {
                // get attribute and value from regex capture groups
                let attribute = match[1];
                let value = match[2];

                // remove from string
                text = text.substring(match.index + match[0].length);

                if (!attribute || !value)
                    break;

                // set attribute of parent element
                try {
                    element.setAttribute(attribute, value);
                } catch(error) {
                    console.log(error);
                }

                // special case for colspan
                if (attribute === 'colspan')
                    removeTableCells(element, value);
            }
        }

        // get list of comment elements in document
        function findComments() {
            const comments = [];

            // iterate over comment nodes in document
            function acceptNode(node) {
                return NodeFilter.FILTER_ACCEPT;
            }
            const iterator = document.createNodeIterator(
                document.body,
                NodeFilter.SHOW_COMMENT,
                acceptNode
            );
            let node;
            while(node = iterator.nextNode())
                comments.push(node);

            return comments;
        }

        // remove certain number of cells after specified cell
        function removeTableCells(cell, number) {
            number = parseInt(number);
            if (!number)
                return;

            // remove elements
            for(; number > 1; number--) {
                if (cell.nextElementSibling)
                    cell.nextElementSibling.remove();
            }
        }

        // load options from url parameters
        function loadOptions() {
            const url = window.location.search;
            const params = new URLSearchParams(url);
            for (const optionName of Object.keys(options)) {
                const paramName = pluginName + '-' + optionName;
                const param = params.get(paramName);
                if (param !== '' && param !== null)
                    options[optionName] = param;
            }
        }
        loadOptions();

        // start script when document is finished loading
        if (options.enabled === 'true')
            window.addEventListener('load', start);
    })();
</script>
<!-- math plugin configuration -->

<script type="text/x-mathjax-config">
    MathJax.Hub.Config({
        "CommonHTML": { linebreaks: { automatic: true } },
        "HTML-CSS": { linebreaks: { automatic: true } },
        "SVG": { linebreaks: { automatic: true } },
        "fast-preview": { disabled: true }
    });
</script>

<!-- math plugin -->

<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML'>
    // /////////////////////////
    // DESCRIPTION
    // /////////////////////////

    // This third-party plugin 'MathJax' allows the proper rendering of
    // math/equations written in LaTeX.

    // https://www.mathjax.org/
</script>
<!-- annotations plugin configuration -->

<script>
    window.hypothesisConfig = function() {
        return {
            branding: {
                accentColor: '#2196f3',
                appBackgroundColor: '#f8f8f8',
                ctaBackgroundColor: '#f8f8f8',
                ctaTextColor: '#000000',
                selectionFontFamily: 'Open Sans, Helvetica, sans serif',
                annotationFontFamily: 'Open Sans, Helvetica, sans serif'
            }
        };
    };
</script>

<!-- annotations plugin -->

<script src='https://hypothes.is/embed.js'>
    // /////////////////////////
    // DESCRIPTION
    // /////////////////////////

    // This third-party plugin 'Hypothesis' allows public annotation of the
    // manuscript.

    // https://web.hypothes.is/
</script>
<!-- analytics plugin -->

<!-- copy and paste code from Google Analytics or similar service here -->
</body>
</html>
back to top