@misc{Lang2016,
author = {Lang, Duncan Temple and {The CRAN Team}},
title = {{RCurl: General Network (HTTP/FTP/...) Client Interface for R}},
year = {2016}
}
@article{Finkel2017,
abstract = {Finkel, Eastwick, and Reis (2015; FER2015) argued that psychological science is better served by responding to apprehensions about replicability rates with contextualized solutions than with one-size-fits-all solutions. Here, we extend FER2015's analysis to suggest that much of the discussion of best research practices since 2011 has focused on a single feature of high-quality science-replicability-with insufficient sensitivity to the implications of recommended practices for other features, like discovery, internal validity, external validity, construct validity, consequentiality, and cumulativeness. Thus, although recommendations for bolstering replicability have been innovative, compelling, and abundant, it is difficult to evaluate their impact on our science as a whole, especially because many research practices that are beneficial for some features of scientific quality are harmful for others. For example, FER2015 argued that bigger samples are generally better, but also noted that very large samples ("those larger than required for effect sizes to stabilize"; p. 291) could have the downside of commandeering resources that would have been better invested in other studies. In their critique of FER2015, LeBel, Campbell, and Loving (2016) concluded, based on simulated data, that ever-larger samples are better for the efficiency of scientific discovery (i.e., that there are no tradeoffs). As demonstrated here, however, this conclusion holds only when the replicator's resources are considered in isolation. If we widen the assumptions to include the original researcher's resources as well, which is necessary if the goal is to consider resource investment for the field as a whole, the conclusion changes radically-and strongly supports a tradeoff-based analysis. In general, as psychologists seek to strengthen our science, we must complement our much-needed work on increasing replicability with careful attention to the other features of a high-quality science. (PsycINFO Database Record},
author = {Finkel, Eli J. and Eastwick, Paul W. and Reis, Harry T.},
doi = {10.1037/pspi0000075},
issn = {1939-1315},
journal = {Journal of Personality and Social Psychology},
month = {aug},
number = {2},
pages = {244--253},
pmid = {28714730},
title = {{Replicability and other features of a high-quality science: Toward a balanced and empirical approach.}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/28714730 http://doi.apa.org/getdoi.cfm?doi=10.1037/pspi0000075},
volume = {113},
year = {2017}
}
@article{Schuemie2014,
author = {Schuemie, M. J. and Ryan, P. B. and Suchard, M. A. and Shahn, Z. and Madigan, D.},
doi = {10.1093/biostatistics/kxt037},
issn = {1465-4644},
journal = {Biostatistics},
month = {jan},
number = {1},
pages = {36--39},
pmid = {24068252},
title = {{Discussion: An estimate of the science-wise false discovery rate and application to the top medical literature}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068252 http://www.pubmedcentral.nih.gov/articlerender.fcgi?artid=PMC3862211 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt037},
volume = {15},
year = {2014}
}
@article{Benjamini2014,
author = {Benjamini, Y. and Hechtlinger, Y.},
doi = {10.1093/biostatistics/kxt032},
issn = {1465-4644},
journal = {Biostatistics},
month = {jan},
number = {1},
pages = {13--16},
pmid = {24068247},
title = {{Discussion: An estimate of the science-wise false discovery rate and applications to top medical journals by Jager and Leek}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068247 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt032},
volume = {15},
year = {2014}
}
@article{Cox2014,
author = {Cox, D. R.},
doi = {10.1093/biostatistics/kxt033},
issn = {1465-4644},
journal = {Biostatistics},
month = {jan},
number = {1},
pages = {16--18},
pmid = {24068248},
title = {{Discussion: Comment on a paper by Jager and Leek}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068248 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt033},
volume = {15},
year = {2014}
}
@article{Gelman2014,
author = {Gelman, A. and O'Rourke, K.},
doi = {10.1093/biostatistics/kxt034},
issn = {1465-4644},
journal = {Biostatistics},
month = {jan},
number = {1},
pages = {18--23},
pmid = {24068249},
title = {{Discussion: Difficulties in making inferences about scientific truth from distributions of published p-values}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068249 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt034},
volume = {15},
year = {2014}
}
@article{Goodman2014,
author = {Goodman, S. N.},
doi = {10.1093/biostatistics/kxt035},
issn = {1465-4644},
journal = {Biostatistics},
month = {jan},
number = {1},
pages = {23--27},
pmid = {24068250},
title = {{Discussion: An estimate of the science-wise false discovery rate and application to the top medical literature}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068250 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt035},
volume = {15},
year = {2014}
}
@article{Ioannidis2014,
abstract = {Jager and Leek have tried to estimate a false-discovery rate (FDR) in abstracts of articles published in five medical journals during 2000-2010. Their approach is flawed in sampling, calculations, and conclusions. It uses a tiny portion of select papers in highly select journals. Randomized controlled trials and systematic reviews (designs with the lowest anticipated false-positive rates) are 52{\%} of the analyzed papers, while these designs account for only 4{\%} in PubMed in the same period. The FDR calculations consider the entire published literature as equivalent to a single genomic experiment where all performed analyses are reported without selection or distortion. However, the data used are the P-values reported in the abstracts of published papers; these P-values are a highly distorted, highly select sample. Besides selective reporting biases, all other biases, in particular confounding in observational studies, are also ignored, while these are often the main drivers for high false-positive rates in the biomedical literature. A reproducibility check of the raw data shows that much of the data Jager and Leek used are either wrong or make no sense: most of the usable data were missed by their script, 94{\%} of the abstracts that reported ≥2 P-values had high correlation/overlap between reported outcomes, and only a minority of P-values corresponded to relevant primary outcomes. The Jager and Leek paper exemplifies the dreadful combination of using automated scripts with wrong methods and unreliable data. Sadly, this combination is common in the medical literature.},
author = {Ioannidis, J. P. A.},
doi = {10.1093/biostatistics/kxt036},
issn = {1465-4644},
journal = {Biostatistics},
keywords = {Bias,False discovery rate,P-value,Science,Selection bias},
month = {jan},
number = {1},
pages = {28--36},
pmid = {24068251},
title = {{Discussion: Why "An estimate of the science-wise false discovery rate and application to the top medical literature" is false}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068251 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt036},
volume = {15},
year = {2014}
}
@misc{Storey2015,
author = {Storey, John D. and Bass, Andrew J. and Dabney, Alan and Robinson, David},
title = {{qvalue: Q-value estimation for false discovery rate control}},
year = {2015}
}
@article{Storey2002,
author = {Storey, John D.},
doi = {10.1111/1467-9868.00346},
issn = {13697412},
journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
keywords = {False discovery rate,Multiple comparisons,Positive false discovery rate,Sequential p‐value methods,Simultaneous inference,p‐values,q‐values},
month = {aug},
number = {3},
pages = {479--498},
publisher = {Blackwell Publishers},
title = {{A direct approach to false discovery rates}},
url = {http://doi.wiley.com/10.1111/1467-9868.00346},
volume = {64},
year = {2002}
}
@article{Benjamin2017,
abstract = {We propose to change the default P-value threshold for statistical significance from 0.05 to 0.005 for claims of new discoveries.},
author = {Benjamin, Daniel J. and Berger, James O. and Johannesson, Magnus and Nosek, Brian A. and Wagenmakers, E.-J. and Berk, Richard and Bollen, Kenneth A. and Brembs, Bj{\"{o}}rn and Brown, Lawrence and Camerer, Colin and Cesarini, David and Chambers, Christopher D. and Clyde, Merlise and Cook, Thomas D. and {De Boeck}, Paul and Dienes, Zoltan and Dreber, Anna and Easwaran, Kenny and Efferson, Charles and Fehr, Ernst and Fidler, Fiona and Field, Andy P. and Forster, Malcolm and George, Edward I. and Gonzalez, Richard and Goodman, Steven and Green, Edwin and Green, Donald P. and Greenwald, Anthony G. and Hadfield, Jarrod D. and Hedges, Larry V. and Held, Leonhard and {Hua Ho}, Teck and Hoijtink, Herbert and Hruschka, Daniel J. and Imai, Kosuke and Imbens, Guido and Ioannidis, John P. A. and Jeon, Minjeong and Jones, James Holland and Kirchler, Michael and Laibson, David and List, John and Little, Roderick and Lupia, Arthur and Machery, Edouard and Maxwell, Scott E. and McCarthy, Michael and Moore, Don A. and Morgan, Stephen L. and Munaf{\'{o}}, Marcus and Nakagawa, Shinichi and Nyhan, Brendan and Parker, Timothy H. and Pericchi, Luis and Perugini, Marco and Rouder, Jeff and Rousseau, Judith and Savalei, Victoria and Sch{\"{o}}nbrodt, Felix D. and Sellke, Thomas and Sinclair, Betsy and Tingley, Dustin and {Van Zandt}, Trisha and Vazire, Simine and Watts, Duncan J. and Winship, Christopher and Wolpert, Robert L. and Xie, Yu and Young, Cristobal and Zinman, Jonathan and Johnson, Valen E.},
doi = {10.1038/s41562-017-0189-z},
issn = {2397-3374},
journal = {Nature Human Behaviour},
keywords = {Behavioral Sciences,Experimental Psychology,Life Sciences,Microeconomics,Neurosciences,Personality and Social Psychology,general},
month = {sep},
pages = {1},
publisher = {Nature Publishing Group},
title = {{Redefine statistical significance}},
url = {http://www.nature.com/articles/s41562-017-0189-z},
year = {2017}
}
@article{Jager2014,
abstract = {The accuracy of published medical research is critical for scientists, physicians and patients who rely on these results. However, the fundamental belief in the medical literature was called into serious question by a paper suggesting that most published medical research is false. Here we adapt estimation methods from the genomics community to the problem of estimating the rate of false discoveries in the medical literature using reported {\$}P{\$}-values as the data. We then collect {\$}P{\$}-values from the abstracts of all 77 430 papers published in The Lancet, The Journal of the American Medical Association, The New England Journal of Medicine, The British Medical Journal, and The American Journal of Epidemiology between 2000 and 2010. Among these papers, we found 5322 reported {\$}P{\$}-values. We estimate that the overall rate of false discoveries among reported results is 14{\%} (s.d. 1{\%}), contrary to previous claims. We also found that there is no a significant increase in the estimated rate of reported false discovery results over time (0.5{\%} more false positives (FP) per year, {\$}P = 0.18{\$}) or with respect to journal submissions (0.5{\%} more FP per 100 submissions, {\$}P = 0.12{\$}). Statistical analysis must allow for false discoveries in order to make claims on the basis of noisy data. But our analysis suggests that the medical literature remains a reliable record of scientific progress.},
author = {Jager, L. R. and Leek, J. T.},
doi = {10.1093/biostatistics/kxt007},
issn = {1465-4644},
journal = {Biostatistics},
keywords = {False discovery rate,Genomics,Meta-analysis,Multiple testing,Science-wise false discovery rate,Two-group model},
month = {jan},
number = {1},
pages = {1--12},
pmid = {24068246},
title = {{An estimate of the science-wise false discovery rate and application to the top medical literature}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/24068246 https://academic.oup.com/biostatistics/article-lookup/doi/10.1093/biostatistics/kxt007},
volume = {15},
year = {2014}
}
@article{Camerer2016,
abstract = {The replicability of some scientific findings has recently been called into question. To contribute data about replicability in economics, we replicated 18 studies published in the American Economic Review and the Quarterly Journal of Economics between 2011 and 2014. All of these replications followed predefined analysis plans that were made publicly available beforehand, and they all have a statistical power of at least 90{\%} to detect the original effect size at the 5{\%} significance level. We found a significant effect in the same direction as in the original study for 11 replications (61{\%}); on average, the replicated effect size is 66{\%} of the original. The replicability rate varies between 67{\%} and 78{\%} for four additional replicability indicators, including a prediction market measure of peer beliefs.},
author = {Camerer, Colin F and Dreber, Anna and Forsell, Eskil and Ho, Teck-Hua and Huber, J{\"{u}}rgen and Johannesson, Magnus and Kirchler, Michael and Almenberg, Johan and Altmejd, Adam and Chan, Taizan and Heikensten, Emma and Holzmeister, Felix and Imai, Taisuke and Isaksson, Siri and Nave, Gideon and Pfeiffer, Thomas and Razen, Michael and Wu, Hang},
doi = {10.1126/science.aaf0918},
issn = {1095-9203},
journal = {Science},
month = {mar},
number = {6280},
pages = {1433--6},
pmid = {26940865},
publisher = {American Association for the Advancement of Science},
title = {{Evaluating replicability of laboratory experiments in economics.}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/26940865},
volume = {351},
year = {2016}
}
@misc{Team2016,
author = {{R Core Team}},
title = {{R: A Language and Environment for Statistical Computing}},
year = {2016}
}
@misc{Wickham2009,
author = {Wickham, Hadley},
title = {{ggplot2: Elegant Graphics for Data Analysis}},
year = {2009}
}
@article{Ioannidis2005,
abstract = {There is increasing concern that most current published research findings are false. The probability that a research claim is true may depend on study power and bias, the number of other studies on the same question, and, importantly, the ratio of true to no relationships among the relationships probed in each scientific field. In this framework, a research finding is less likely to be true when the studies conducted in a field are smaller; when effect sizes are smaller; when there is a greater number and lesser preselection of tested relationships; where there is greater flexibility in designs, definitions, outcomes, and analytical modes; when there is greater financial and other interest and prejudice; and when more teams are involved in a scientific field in chase of statistical significance. Simulations show that for most study designs and settings, it is more likely for a research claim to be false than true. Moreover, for many current scientific fields, claimed research findings may often be simply accurate measures of the prevailing bias. In this essay, I discuss the implications of these problems for the conduct and interpretation of research.},
author = {Ioannidis, John P. A.},
doi = {10.1371/journal.pmed.0020124},
issn = {1549-1676},
journal = {PLoS Medicine},
month = {aug},
number = {8},
pages = {e124},
pmid = {16060722},
title = {{Why Most Published Research Findings Are False}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/16060722 http://www.pubmedcentral.nih.gov/articlerender.fcgi?artid=PMC1182327 http://dx.plos.org/10.1371/journal.pmed.0020124},
volume = {2},
year = {2005}
}
@article{OpenScienceCollaboration2015,
abstract = {Reproducibility is a defining feature of science, but the extent to which it characterizes current research is unknown. We conducted replications of 100 experimental and correlational studies published in three psychology journals using high-powered designs and original materials when available. Replication effects were half the magnitude of original effects, representing a substantial decline. Ninety-seven percent of original studies had statistically significant results. Thirty-six percent of replications had statistically significant results; 47{\%} of original effect sizes were in the 95{\%} confidence interval of the replication effect size; 39{\%} of effects were subjectively rated to have replicated the original result; and if no bias in original results is assumed, combining original and replication results left 68{\%} with statistically significant effects. Correlational tests suggest that replication success was better predicted by the strength of original evidence than by characteristics of the original and replication teams.},
author = {{Open Science Collaboration}},
doi = {10.1126/science.aac4716},
issn = {0036-8075},
journal = {Science},
month = {aug},
number = {6251},
pages = {aac4716--aac4716},
pmid = {26315443},
title = {{Estimating the reproducibility of psychological science}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/26315443 http://www.sciencemag.org/cgi/doi/10.1126/science.aac4716},
volume = {349},
year = {2015}
}