1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099 | <!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta name="generator" content="pandoc" />
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="author" content="Christian Thiele" />
<meta name="date" content="2021-06-29" />
<title>An introduction to cutpointr</title>
<script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
// be compatible with the behavior of Pandoc < 2.8).
document.addEventListener('DOMContentLoaded', function(e) {
var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
var i, h, a;
for (i = 0; i < hs.length; i++) {
h = hs[i];
if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6
a = h.attributes;
while (a.length > 0) h.removeAttribute(a[0].name);
}
});
</script>
<script>// Hide empty <a> tag within highlighted CodeBlock for screen reader accessibility (see https://github.com/jgm/pandoc/issues/6352#issuecomment-626106786) -->
// v0.0.1
// Written by JooYoung Seo (jooyoung@psu.edu) and Atsushi Yasumoto on June 1st, 2020.
document.addEventListener('DOMContentLoaded', function() {
const codeList = document.getElementsByClassName("sourceCode");
for (var i = 0; i < codeList.length; i++) {
var linkList = codeList[i].getElementsByTagName('a');
for (var j = 0; j < linkList.length; j++) {
if (linkList[j].innerHTML === "") {
linkList[j].setAttribute('aria-hidden', 'true');
}
}
}
});
</script>
<style type="text/css">
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
span.underline{text-decoration: underline;}
div.column{display: inline-block; vertical-align: top; width: 50%;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
</style>
<style type="text/css">
code {
white-space: pre;
}
.sourceCode {
overflow: visible;
}
</style>
<style type="text/css" data-origin="pandoc">
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
</style>
<script>
// apply pandoc div.sourceCode style to pre.sourceCode instead
(function() {
var sheets = document.styleSheets;
for (var i = 0; i < sheets.length; i++) {
if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue;
try { var rules = sheets[i].cssRules; } catch (e) { continue; }
for (var j = 0; j < rules.length; j++) {
var rule = rules[j];
// check if there is a div.sourceCode rule
if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") continue;
var style = rule.style.cssText;
// check if color or background-color is set
if (rule.style.color === '' && rule.style.backgroundColor === '') continue;
// replace div.sourceCode by a pre.sourceCode rule
sheets[i].deleteRule(j);
sheets[i].insertRule('pre.sourceCode{' + style + '}', j);
}
}
})();
</script>
<style type="text/css">body {
background-color: #fff;
margin: 1em auto;
max-width: 700px;
overflow: visible;
padding-left: 2em;
padding-right: 2em;
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 14px;
line-height: 1.35;
}
#TOC {
clear: both;
margin: 0 0 10px 10px;
padding: 4px;
width: 400px;
border: 1px solid #CCCCCC;
border-radius: 5px;
background-color: #f6f6f6;
font-size: 13px;
line-height: 1.3;
}
#TOC .toctitle {
font-weight: bold;
font-size: 15px;
margin-left: 5px;
}
#TOC ul {
padding-left: 40px;
margin-left: -1.5em;
margin-top: 5px;
margin-bottom: 5px;
}
#TOC ul ul {
margin-left: -2em;
}
#TOC li {
line-height: 16px;
}
table {
margin: 1em auto;
border-width: 1px;
border-color: #DDDDDD;
border-style: outset;
border-collapse: collapse;
}
table th {
border-width: 2px;
padding: 5px;
border-style: inset;
}
table td {
border-width: 1px;
border-style: inset;
line-height: 18px;
padding: 5px 5px;
}
table, table th, table td {
border-left-style: none;
border-right-style: none;
}
table thead, table tr.even {
background-color: #f7f7f7;
}
p {
margin: 0.5em 0;
}
blockquote {
background-color: #f6f6f6;
padding: 0.25em 0.75em;
}
hr {
border-style: solid;
border: none;
border-top: 1px solid #777;
margin: 28px 0;
}
dl {
margin-left: 0;
}
dl dd {
margin-bottom: 13px;
margin-left: 13px;
}
dl dt {
font-weight: bold;
}
ul {
margin-top: 0;
}
ul li {
list-style: circle outside;
}
ul ul {
margin-bottom: 0;
}
pre, code {
background-color: #f7f7f7;
border-radius: 3px;
color: #333;
white-space: pre-wrap;
}
pre {
border-radius: 3px;
margin: 5px 0px 10px 0px;
padding: 10px;
}
pre:not([class]) {
background-color: #f7f7f7;
}
code {
font-family: Consolas, Monaco, 'Courier New', monospace;
font-size: 85%;
}
p > code, li > code {
padding: 2px 0px;
}
div.figure {
text-align: center;
}
img {
background-color: #FFFFFF;
padding: 2px;
border: 1px solid #DDDDDD;
border-radius: 3px;
border: 1px solid #CCCCCC;
margin: 0 5px;
}
h1 {
margin-top: 0;
font-size: 35px;
line-height: 40px;
}
h2 {
border-bottom: 4px solid #f7f7f7;
padding-top: 10px;
padding-bottom: 2px;
font-size: 145%;
}
h3 {
border-bottom: 2px solid #f7f7f7;
padding-top: 10px;
font-size: 120%;
}
h4 {
border-bottom: 1px solid #f7f7f7;
margin-left: 8px;
font-size: 105%;
}
h5, h6 {
border-bottom: 1px solid #ccc;
font-size: 105%;
}
a {
color: #0033dd;
text-decoration: none;
}
a:hover {
color: #6666ff; }
a:visited {
color: #800080; }
a:visited:hover {
color: #BB00BB; }
a[href^="http:"] {
text-decoration: underline; }
a[href^="https:"] {
text-decoration: underline; }
code > span.kw { color: #555; font-weight: bold; }
code > span.dt { color: #902000; }
code > span.dv { color: #40a070; }
code > span.bn { color: #d14; }
code > span.fl { color: #d14; }
code > span.ch { color: #d14; }
code > span.st { color: #d14; }
code > span.co { color: #888888; font-style: italic; }
code > span.ot { color: #007020; }
code > span.al { color: #ff0000; font-weight: bold; }
code > span.fu { color: #900; font-weight: bold; }
code > span.er { color: #a61717; background-color: #e3d2d2; }
</style>
</head>
<body>
<h1 class="title toc-ignore">An introduction to cutpointr</h1>
<h4 class="author">Christian Thiele</h4>
<h4 class="date">2021-06-29</h4>
<p><strong>cutpointr</strong> is an R package for tidy calculation of “optimal” cutpoints. It supports several methods for calculating cutpoints and includes several metrics that can be maximized or minimized by selecting a cutpoint. Some of these methods are designed to be more robust than the simple empirical optimization of a metric. Additionally, <strong>cutpointr</strong> can automatically bootstrap the variability of the optimal cutpoints and return out-of-bag estimates of various performance metrics.</p>
<div id="installation" class="section level2">
<h2>Installation</h2>
<p>You can install <strong>cutpointr</strong> from CRAN using the menu in RStudio or simply:</p>
<div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb1-1"><a href="#cb1-1"></a><span class="kw">install.packages</span>(<span class="st">"cutpointr"</span>)</span></code></pre></div>
</div>
<div id="example" class="section level2">
<h2>Example</h2>
<p>For example, the optimal cutpoint for the included data set is 2 when maximizing the sum of sensitivity and specificity.</p>
<div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb2-1"><a href="#cb2-1"></a><span class="kw">library</span>(cutpointr)</span>
<span id="cb2-2"><a href="#cb2-2"></a><span class="kw">data</span>(suicide)</span>
<span id="cb2-3"><a href="#cb2-3"></a><span class="kw">head</span>(suicide)</span></code></pre></div>
<pre><code>## age gender dsi suicide
## 1 29 female 1 no
## 2 26 male 0 no
## 3 26 female 0 no
## 4 27 female 0 no
## 5 28 female 0 no
## 6 53 male 2 no</code></pre>
<div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb4-1"><a href="#cb4-1"></a>cp <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, </span>
<span id="cb4-2"><a href="#cb4-2"></a> <span class="dt">method =</span> maximize_metric, <span class="dt">metric =</span> sum_sens_spec)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb7-1"><a href="#cb7-1"></a><span class="kw">summary</span>(cp)</span></code></pre></div>
<pre><code>## Method: maximize_metric
## Predictor: dsi
## Outcome: suicide
## Direction: >=
##
## AUC n n_pos n_neg
## 0.9238 532 36 496
##
## optimal_cutpoint sum_sens_spec acc sensitivity specificity tp fn fp tn
## 2 1.7518 0.8647 0.8889 0.8629 32 4 68 428
##
## Predictor summary:
## Data Min. 5% 1st Qu. Median Mean 3rd Qu. 95% Max. SD NAs
## Overall 0 0.00 0 0 0.9210526 1 5.00 11 1.852714 0
## no 0 0.00 0 0 0.6330645 0 4.00 10 1.412225 0
## yes 0 0.75 4 5 4.8888889 6 9.25 11 2.549821 0</code></pre>
<div class="sourceCode" id="cb9"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb9-1"><a href="#cb9-1"></a><span class="kw">plot</span>(cp)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABjFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYZGT8ZGWIZP4EZYp8aGhozMzM6AAA6ADo6AGY6OgA6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNs/GRk/GT8/GWI/P4E/Yp8/gb1NTU1NTW5NTY5NbqtNjshZWVliGRliGT9iGWJiP4FiYmJin9lmAABmADpmOgBmOmZmZmZmkJBmkLZmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SBPz+BP2KBYhmBvZ+BvdmOTU2OTW6OTY6ObquOyP+QOgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+fYhmfYj+f2dmrbk2rbm6rbo6rjk2ryKur5P+2ZgC2Zjq2kDq2kGa2tpC2ttu225C229u22/+2/9u2//+9gT+92dnIjk3I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrbkGbbtmbbtpDb25Db29vb2//b/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T////1+by3AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAc5ElEQVR4nO2djZ/btnnHqYtPe3Flp2sy7S3npN7W9eK1a3PuMndbss7KliW5LOkW37ZudS65vFl12jWXRCf3ZJn/ePHghQBJgAIBkaLE3+9jWxTJB3gAfA08oEgiSSEoQsmmHYC2WwAIihIAgqIEgKAoASAoSgAIihIAgqIEgKAohQG0nOyfl/cNa6fisJj/9WktBxaHQ+t+qHl1EqDpHgDaFm0pQHkBoM0pHCD292dPJ4Mf0PeHo+TKzzkOj24lyTfvU5uOZ2zncWrsq7bIji4nSZKMeT6CJM7Hw6cTbsqomyZ7P+egqH3sBMrsvgJIJQk1rxiAEq4j1s70eeUWa+b5iDYHx6xNvzUSW3pftUV21ABoPmLnpjNmMVWmy8lTo2T/CwIl27c4fGqkEmH7sySh5hUF0JAaccjab/Bm+oh9ZfsHL6Xp5yPax44u/zu3r9pCH9VDmOhQTvZOl5O9+8QTP22sHFD7WGZ/qDKj/SpJqHnFAETNTMPLfET9BbWk2CIAFoc8GGFtr/dVW+ijRgw05Z0ZR+FXP/uXEUeEDspYR+4TmdER2q+TjKkYyE9RMdC5CIRpiFFbQoNj2eoMAL2v2kIfNdqeYOAni2FJ9jGpOVRxgMYyM9qvk1xHBUHV6jhAtJP1YjRIfevf/vfXhyZAeh8A2pzWAZAYM8TQdCTP0KOK3ldtYQWIQfHW4VBF0yIGUgDpfYJWPYSpJKHmtQ6AWEj8Eo+BKSR+k82iJxQD6bhW7au2MAHSncd89D0xgrEAm83OdbYEitrHMhunZhAtk1xbLUFOrQMgMQ7xSbkcPsZqZk2NmO2rtjABUtN4+p6IoFmcZgKk98lrBuxEfjRLEmpeawEo/fxWcuV9vjW/lSRPvaSu7T3Dr+apfdUWxtHFLZrRC/F5vbg2eOXNE91R7Z/rfeKq5R9kFxJVklDzaurXeP3rArTTAkBQlAAQFCUABEUJdyRCUQJAUJQAEBQlAARFCQBBUQJAUJQAEBQlAARFaU0APbqfzq/Zb+By7X9UfmzCPNVyONPMuFODbdvM2L5Sxm4nXT5WH4LSdQFUVcv122zl4QJADrNiEmGUAKBqAaDwQ1AaDBDdzEV3ml57nW7oorvbx3zQeINvjZLsPvhx1gKZCT0qJk8U5uYxceSIp8jNslTeEKkuDpPB6xIauU1J8tOE2fz6vyZ7/0dJidTzOR7rzGSKaVaO9IR+wJsOi/5KH2Qm+hMKv6FsKO9tpju8hqKWWZON+ON+9A/d5XXE726WAIk7WY9Ucx6LhhHmWXIqCR3XZKmMVKp0t6MASG2reGc+OhKpShZHdNfbMJ/jtWPtu0iRUlKnzvhNjUcFf3MlkZmkuPNaKAwgPm7M5A3zum3EV7nvN3RPoY5lFRF5gMSpWXIqCR0BZ6mYp8pmV9t0+PpplnqWhJmUdrLoOzeT25wYkZThb+aDyuQ67rbOFAgQv3FUtg2rdQWQbCr5z4w/WyNbQsUqhebk5kZy5mGZl06F/TPlp4oWVNt07ISPN2mWqmpv6ZwJkOm7AkieSsOXHMFy/gofVCbZJ9QkQIvDwbGBQiBAuVQqAOIBkQyHIgCaX39fjGCmv8oHlYnxCQUCRM/dZMPA9VMbQLyhZoPVQ9j10yy5MkC5VIwBiB/TIx8/V5NcGrpMgHRmGiC1vZx83xgM5afyQWWS++y71hNEy2gzDxD9tx1lACkTinyXk4EINyxBdNZ/yY7ATIXvHxpB9FAF0ZwlFcQoYGRknMvRCKJNgOSp2UMgqemT8kFloj6jK38XtIZpPNX4STIsxUAsUhi8JUcQbcLnwD988ViYSHNzGp9Zi5Y0U6F/HNN4GaVIR2RSr1tyPC5cT5A9kDw196Rsdpr0IctEfUKxFxIj/xt28H8xZlg1BYDymuJ51noCQKb4xUWojnA7BxQlAARFCQBBUQJAUJQAEBQlAARFKQygB42r6w5AUgAozAFICgCFOQBJAaAwByApABTmACQFgMIcgKQAUJgDkBQACnMAkgJAYQ5AUgAozAFICgCFOQBJNQ3QJ3/6D1evfvfBg0///OrvvrO+9vNM5e3vMgc+lHl/fLWOC0H10kM1DtAfPffg49/78MHb4mNd7eeZysfP0R+W9we//+DTv3yHPtbkACTVOEB/8g79pdb79C9e8bdbjwOffufDf3+F8mX518oeAPmqJYDo47Mftw7QZz/+x+/QCHb16u+8wjpD+ndNDkBSu90DPfjgr57jY5dUjVE0qF56qJYA2lAM9OCTP2bQirwpewC0drUF0GZmYQ8++/t3+AyQxq63MQtrQDt+HeiTP2vIAUhqtwH6oEbUXM8BSGq3AWrOAUgKAIU5AEkBoDAHICkAFOYAJBUG0NdS2YbYTEp7i2dUbuZ2d9wBSAoAAaAoASAAFCUABICiBIAAUJQAEACKEgACQFECQAAoSgAIAEUJAAGgKAEgABQlAASAogSAAFCUABAAihIAAkBRAkAAKErhAN0U2kz7ORJyO2DxFgCtQwAIAEUJAAGgKAEgABQlANQUQNNE6Jv3+dfl/zydJE/9jVxRU3x7KTqTzQsANQ1Qwtcof3RLfLnCcaKFyEnPRueycQGgxgASi9sv/ysZ80Xon2HoPHotoVV9F4fJs/+fpr+8lRytSqXzAkANA8RoGVN3JJcT5xvqm7FI9MNRcuWYQKM9y8mQ/k6Tvdd4IovDoejD5HDYJQGghgH6zQmjZTmRNDEW9s/1t1+pk2c0oA2OcwA9NUr2fzk64keP5Kg36Ng66VsLUO3SSG/Dkg2SGQPxLkToZO/U+Ca1nDAyHnLSNEBi6KPvzIadwULuz0dFy41rSwFyJNSpHkjgM3j2PF0J0HwkRzgTIN5NTRlZdL48Q42L3REAanQIezh6hqKc1QDJaNoEiMdHBA6NYDPZnXVuDANAzcZAs0QQYY+BfvGMiIrdANEHYw4A9RWg9ITPuOyzsMWhJCkbwqiDEh2WBIiNYW9Rh5Uh1jUBoIYBEpC4rgNloQ/rWeasc2K8/YBO1gDNR9+jq0XsjDeZ/QQxUN8AYt0NRTwL65XoLBYSI9SR3LhySwPEaJJjIdfYktNGBYCaBogRQKNP/rewR6+N8r+FPRwlA/71F6PkmS90DKQAZNDd6uSvZwCoKYB6IgAEgKIEgABQlAAQAIoSAAJAUQJAAChKYQCRZJOsJ7G6AkBdEXogABQlAASAogSAAFCUAFBDANkzyMt5ZAtMVDkBEAAKMlHlBEAAKMhElRMAAaAgE1VOAASAgkxUOQEQAAoyUeUEQAAoyESVEwABoCATVU4AFAHQ5d9+xD8fv3rwwlfZR97DraAhwESVc3cBku41CNDFwfMcoCfv3k2//Lb6AEAAyMMBprMb74ke6PFPPqLOSH4AIADk4QCXxOXy5a/Sx/90T36wHd9gCqrX7ko+GGs5EJQcAOKSAF28wMmRHwUPt6I7sR1JciqZAKA4B0yASj3QlgJkRcZlAoDiHDAB2t4YyIUMYqCv2wToybt3xCzsTvdnYR69jF8uACjOgQwg+tvp60BuZKJyAUBxDnhVUTmDvBoCqIBMM7mocgKgXQCocmACQBXN01uASsi03c+pcgKgbQFoRfgLgL4GQIUDNmQaCpvqm6hyAqDuAVQ1yQZAFZsAiETgtE1DgIkqZ8cB+vKAdFdtPl+8zrtzAIl+BwDZXAntgS7E1bmzu5bm2TGAVkY6AKhq095+8vfJJz+9V3DAktB2A6SjHgBkcyUQIPnz0uNX1VhWcbeNWRrhXv7bzZJFY6rdTl4/ZgKgqk0rQOoGicsf3TN6IUdC29sDrfFXqnZMVDk7D9CF/n3SiIMcCW0rQMUpOwCyuRIG0NkdI9/dBKh8xQcA2VwJAigbtagnevIfOzaNp6uFtguGAMjmShBAIgSim26+PDi4cc90wJLQlgFU/3ozAKrarNF+joQAUGsmqpwACAAFmahyAqBNx0DWA26T2gc2B9CF+O3JdcMvAIpXey/WbkzuIvDA9dvuB78BkJeH1T2Q64DbpPaBjQ5h+mmn8kNPAMjLw34DxPoc54PfsknM0zu11MEWAFT/uaztAujy9o177ge/0QMBoBUApUbXU37wGwB5edhvgNKzu4iBwmTPIKekMzQEmKhyugGSg5bzwW8ABIAqAVI/PuE6EACyHPAAaEXtACAvD92t0dRT6+2YqHICIAAUZKLKCYAAUJCJKicAAkBBJqqcAGhTAFU9gAqAbK4AoJwAEADy8dDVGpXv9AFANlcAkKHqR+ABkM0VAKSVOI9UH+iOiSonAAJAQSaqnABoEwB17XVjASaqnABoAwCVV54oCADZXAFASgCIUgBAXh4CIFft9B6g7AYX9R4+60v4LK2x+j1SAMjmyo4BlH/QiW6/s76EDwC52q/vAOVu8qXbxe0v4QNArvbrO0C5xwyoK/J6CR9pB55H1QJAoQCZDzrxT/tL+Mr/nRPnkdUHumMCgOIcyPdA+j18pXeoASBX+/UdIDMG0u/hA0BeohT6DpDxoJMYuOwv4QNAAMghcR2IL3kpRjLrS/hKreHzOmgAZHNl1wDy8xAAuWoHAHl5CIBctQOAvDwEQK7aAUBeHhZbw3wjR2doCDBR5dxSgBzagsVWAJCqHfRAXh4CIFftACAvDwGQq3YAkJeHhdbIvZSsMzQEmKhyAiAAFGSiygmAAFCQiSonAAJAQSaqnACoVYDy72XtDA0BJqqcAAgABZmocgIgfwcWh4PjWlVUzgAAackmMXft/FIHJ0myf17PQwDkqp3+9UCkxWGSjH2rqJRo4dX0naEhwESVEwDVdYAQ2jv1qqJSogDIqJ1+AjRNkiEbylYPZNZEAZBROz0EaDlJkiPamK3ugqyJAiCjdvoH0OLQY+iq0k49USgFgOoA5Nv9pPYeqNABdac7CTBR5QRAACjIRJUTAPk6ME2Uhr5VVEwUAJm10zeAsh7Iv4oKiRb56Q4NASaqnAAozAGvKiokCoBytdMzgFj3Q5cQSaExEADK1U7PAAqpokKiAChXOwDIy0OdaImf7tAQYKLKCYD8HVgcev2MmvcQALlqp38A8Zm870SsnCgAytdODwFK+R1BfneVlRMFQPna6SdATCdhs7AyP92hIcBElRMA1e6B/H5RLSUKgAq100OAvMevFACtrp3+ART3UwYAKtRO/wCqXUW5RAFQoXZ6BlDsTxkWfrpDQ4CJKicAaqkHAkDF2ukfQFE3lAGgYu0AIC8PAZCrdvoGUOQdiQCoWDt9AyhyGg+AirXTP4AKypa8VGtd6h1lgGz8dIeGABMAVNOB0jTeWPJSLNGTXwOzkCgA6jtAJenlnuQqc7k1MAEQAFohveCcXOtS77AsebmLD6VyAaAaDohhTM7i9ZKXcq1Lcw1M9EBdAUivxW4NUdsF6GSYTvdOp2Ian1t0l+Kg/I5CogBoQwBlS0jaQ9RWAWId0HIyVBcS8yEPc7QqBrLy0x0aAky2BCC9CrK9edoGiO6rn6lZmFryUq51aayB2VOA/C/VUwqtAKTXYq8MUW2lEe7lv90sWdTRcjKeDY5pIFO+ySUv5VqXFdeBAFBelEIrAOm12O0harsx0HyUDH3eTtZPgOr81kMptDcL43GQPURtF6A6sjpWUGdoCDApA1Tntx5KoWWAOhAD1ZLVsYI6Q0OAiSpnRRB9eZuHH3JkLw3wrQCk12K3h6itD2FhdyTa+ekODQEmVoByFUTjBQtB5PS5PItu7ToQi0/54uwbvw5Ec3hv5RLtCUD5CrogXLKrG+URpL0hrKJ5WgUo/HaOngBUriDWC8ngtTyLtkyMd3ypg+UEAOktqXwPVKggijvk9Lk8i+5fD+R5M2t/ASpW0ONX72TT5/Isun8AhT/W0xOAChV0eduYPiMGqqlcoj0BKC/Bj5o+l2fRAAgAVQJEN1LQhaCNXgfqGEDTJDmaBvyU0ROA/Md4SqGHAJ3s/1rc0QGAKnqgxYur32BCKfQPIH47x1HIg4W9Aiid+S2GBYC8POwjQBjCHA5MaQjzfFdrLtF+AeS5HF8PAUpnFCL6ves3l2hPAJJBtMdb3CiFPgJUQ7lEewJQvdoBQF4eUqIOfrpDQ4CJKicA8nZgyrpm71W/c9rZpwrTYtmmnmN8HwGi++npGlBAEN2bHmhK8y+fCqIUegbQcjKmO+6OaC7mcynaTLQvAHXwqYzuAMQrh3qhgOtAAAgAicrhnQ8A4lsWgDCEVQ5hR/KeX78Hw8xEewMQgugKB1jvw0Ogmd+KT2ai/QHIV5RC3wCilTLGrBPyXC3DTBQA2WqndwDVryKVaG8Aoqmq1+0ulAIA8vKwVwDx9074EEQpACAvD/sEEKbxAMjbRJUTAAGgIBNVTlwHAkBBJqqc+VmY7w1TlAIA8vKwXwDVqR0A5OUhAHLVzpYAdDOntTngVUUqUQBkqx0A5OUhAHLVDgDy8hAAuWoHAHl5CIBctQOAvDwEQK7aAUBeHgIgV+30HiD9vhL5Plu9MAwAAkAr60C/t1a+z1YvDAOAANBqgPQ72+T7bPXCMACo8wA5ZFlsJQ9QWLJW5d8aybb0wjCF5WB2+bnCbQXIkVCbPVDuvbX0BkC9MIzpIUvJ1QF1pzsJMAFAcQ7keyD+PluuLA4yUgJA1vbrO0DGe2vl+0hJAMhLlELfAdLvrZX86IVhAJCPLGFpF5c6aAwgvWKhep+tXLgw7yEAcrVf33sgTw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPLyEAC5agcAeXkIgFy1A4C8PARArtoBQF4eAiBX7QAgLw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPLyEAC5agcAeXkIgFy1A4C8PARArtoBQF4eAiBX7QAgLw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPIVnkwtqZMAMWcS4U8FQPITPVCsiSonAAJAQSaqnAAIAAWZqHICIAAUZKLKCYAAUJCJKicAAkBBJqqcAAgABZmocgIgABRkosoJgABQkIkqJwACQEEmqpwACAAFmahyAiAAFGSiygmAAFCQiSonAAJAQSaqnAAIAAWZqHJ2HSC5jknqWMoEAG3KRJWz4wCpdUzS1L6UyTYA5OSnOzQEmKhydhwguY4J+8e+lAkA2pSJKmclQPxF7HJZLL06VpsAkcR6As6lTDIJgMyvN8uLrajPxqWL0l+ALijmkMti6dWxVO20BRC9EZ7JvpQJeqBNmXgAdHbjPdYDySUhjJUh2gVIr2OSWlaiAECbMvEASAxhclEavTaNGkEsg0EDjx8Y65ikHQNID+quUV6o5wDJZbFyq2O11wNpfuxLmWwQID2oO0d5oZ4DVOqBWgVIrmNCjliXMtkgQHpQd47yQj0HaNMxkF02i5YB0v+lnKN8H7QSILksll4diwSAzCUvnaN8OYO8OtOdBJh4A7T560AdBaiiBwJAq0Qp9B0g7xhoC2gIMAFAcQ6YS146R/lyBnl1hoYAEwAU50BqLHm54jrQFtAQYAKA4hzwqqJyBnl1hoYAE1VOAASAgkxUOQEQAAoyUeUEQAAoyESVEwABoCATVU4A1BBAWgG/amyRyTYDVGACAG3CBAABoCgTANQ4QLstAASAogSAAFCUABAAihIAAkBRAkDNAFT9yMYKE/lCgNzbAFaYyHNr5CJvN/fIJc1uc7KUBQA1AtCKRzaqTdQLAXJvA6g2kefWyYVEd+GuzEU+YeooCwBqBKAVtytWm8gXAuTfBlBtIs+tk0sqSF2di3zC1FEWANQIQCtumK42IbEt/TaA1Sby3Jq5UDeyOpc0G8JsZQFAjQC04pGNapNU3CWbfxtAtYk8t14u/HN1LmkGkK0sAKiLPZB+IUBlhFJI9OxuvVwuskh4VRyEHqhtgKJiIPOFAJVNW0j07G69GOgse22FJ0CIgVqchVU+slFtIvnJvw2g2kSeWycXOXCtziXNALKVBQA1eR3I9chGtYlxheZGdXCic5Hn1shFjUKrc5EvSbCXBQA1A1BvBIAAUJQAEACKEgACQFECQAAoSgAIAEUJAAGgKAGgbgO0ePG4tG+aJMmgvDvT/NoxO2fwxrVjc8+j+404CIC6DdBJmZTp3mmazpKjKrPFYfHw/FoFchECQF0GaHFY7mqWE87Gyf55hWEZFwBUcqAHAM2G5XZfTsZya37t9SQhjpaTJKFuiT6HtP+NEdsg09ye8QnbTKfDtboYD5DRJO0BxKXBMQByHL1ZOHlbZOk4ZgwFcWy0d7qcDFP6m073z+mTxi5mQ2bsb2HPjFEmO7C1aUsB+lp7sMs9kGPkoSia+pXRET+BuCBO1LkaoMIeztL107U6GL68heX/dANrZTjUb4BS4mX/nB9jUBBPTGMOUmoCVNhDw9eaRzD0QNsCEFEyzh/IAJLx9GqA5tffX/MIBoC2BaDiHmKCD2HXT2eD49y5ziGMBUDfX/MIBoC2DiBxaYjCYx1Esy6IUaQC6nwQLfeIK0PTZM0jGADaPoD4aEbDGZ/GExA0WRdUyUm7AsjYk55kcfdaBYC6DVCVAq4NrnsOBoB6BtB0vPqceto9gPJ+OQCq/rqrAM1Hlb9/BAkAbS9AnRAAAkBRAkAAKEoACABFCQABoCgBIAAUJQAEgKIEgABQlAAQAIoSAAJAUQJAAChKTQGkjgKgHRcAAkBRAkAAKEoACABFCQABoCgBIAAUJQAEgKIEgABQlAAQAIoSAAJAUQJAAChKAAgARQkAAaAorQ2gm/n2A0A9EQACQFFqGiCzSQDQDgoAAaAoASAAFCUABICi1BJARssAoJ0SAAJAUfIGKLcecHsA6WxLDgCgLsgXoCfv3jVWJG8NIJ1t2QEA1AX5AvT4Jx+p5efTtQOkjpYB0tmWHQBAXZAvQJcvfyVXqv8GU2Ri/tLZbsgBaIV8q/ziBdV+JEdPsv4nU3W2G3IAWqH6PRCptfaz9UCtOgCtUHgM1Er7VcZAbTgArZD/LOxOcRLURvvpbDfkALRC4deB2mk/kS11PRtyAKpW+JXojbbfxh2ApAAQAIoSAAJAUQJAAChKAAgARQkAAaAoASAAFCUABICiBIAAUJQAEACKUtwdNPb7chx366zh5O450HsBIAAUJQAEgKIEgABQlHAXMRQlAARFCQBBUQJAUJQAEBSlCIByNylnujg4eP6j8slP3j24ca+09/K27WT+8AU7dHC34w5AMQDlH1ZXorov703Ts7v80cC8Hr/KkijtvaA2pUfALn9UbvEuOQClMQDlH9QyZdlLJ1tOzD0sKHV24z1mf0EMnFX2ABt3AEpjALJWPpelA7h8+T8tI4gjCdX+9tS74wCUxgCUf1hd6/K2Nda4y1srLz6CWJpVtB89SthpB6C0mR7I9n/a8V/99sHf/dTRfo9fXdF8G3cASpuJgSyhw+N/djZ2OTiRk6BV8cfGHYDSuFnYHUuw4RpXzqwjCDvRFq+w9vNovo07AKUNXAf68sB2vYVOtlxwuTiwJcFn4gek6jbcuAMQrkRDkQJAUJQAEBQlAARFCQBBUQJAUJS2A6D5teNNuwDZBYCgKAEgKErdBGg5GbN/p3un81GSJGMCiDNE/ywnSbJ3umkPIaluApRO98+JosXhkeBIA7ScDMVhqBPqKECEC/v7m3OxbQA0o96HgwV1QR0FiMYw3s3M2BA2MAGaJlzjTXsICXUUoHS2/wVjaHE4OC70QBi9uqWuArR48Y3rpwwj6oRyPRD7tmnfIENdBSg9SYacnXQ+4gAtDsdsYBuwIJpBBYo6o84CNEsoTj5hEdBbh0c8pmYz+h++KKbx4Kcz6ixA0HYIAEFRAkBQlAAQFCUABEUJAEFRAkBQlAAQFCUABEUJAEFR+i1V8T4NlX3iKgAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<p>When considering the optimality of a cutpoint, we can only make a judgement based on the sample at hand. Thus, the estimated cutpoint may not be optimal within the population or on unseen data, which is why we sometimes put the “optimal” in quotation marks.</p>
<p><code>cutpointr</code> makes assumptions about the direction of the dependency between <code>class</code> and <code>x</code>, if <code>direction</code> and / or <code>pos_class</code> or <code>neg_class</code> are not specified. The same result as above can be achieved by manually defining <code>direction</code> and the positive / negative classes which is slightly faster, since the classes and direction don’t have to be determined:</p>
<div class="sourceCode" id="cb10"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb10-1"><a href="#cb10-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, <span class="dt">direction =</span> <span class="st">">="</span>, <span class="dt">pos_class =</span> <span class="st">"yes"</span>,</span>
<span id="cb10-2"><a href="#cb10-2"></a> <span class="dt">neg_class =</span> <span class="st">"no"</span>, <span class="dt">method =</span> maximize_metric, <span class="dt">metric =</span> youden)</span></code></pre></div>
<p><code>opt_cut</code> is a data frame that returns the input data and the ROC curve (and optionally the bootstrap results) in a nested tibble. Methods for summarizing and plotting the data and results are included (e.g. <code>summary</code>, <code>plot</code>, <code>plot_roc</code>, <code>plot_metric</code>)</p>
<p>To inspect the optimization, the function of metric values per cutpoint can be plotted using <code>plot_metric</code>, if an optimization function was used that returns a metric column in the <code>roc_curve</code> column. For example, the <code>maximize_metric</code> and <code>minimize_metric</code> functions do so:</p>
<div class="sourceCode" id="cb11"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb11-1"><a href="#cb11-1"></a><span class="kw">plot_metric</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABLFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubo5ubqtuq8huq+SOTU2OTW6OTY6Obk2ObquOyP+QOgCQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rjk2r5OSr5P+2ZgC2kDq2tpC2ttu225C229u22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+rSkCjAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAYRUlEQVR4nO2dC3vcxnWGh1K0XspNE5dMZNq9pPaqrirZDdk2bhwyqZg0cWWyTmuL0mpd3vb//4dicFtgFwPMzJnLGeD7nifxane+M+cMXg4GWAAr1hBEkIidAJS2ABBEEgCCSAJAEEkACCIJAEEkASCIJAAEkWQB0PmPvypf/fmvD5ofrP7mVfnq7uTR68E4Wo3WN0cz7cw2CfT2pGqmnxRUywYgsXeav1jNRQugyweRAdok0NuTqpl+UlAtG4D23i+4uXw4VwCkpXAAGTQDQIayAejBr/JBvjv5RQ7QDx8L8eOvs38KIQ7uTmaX4sGf8s3w7Vz86LQ0FRstp+Hb94XY+/tiWxXbKzNt4mT//NeswaelMbMsszjZ+0uxyP9dQVuGr0MUCWSfL6t+ZUTpLHv6w/uy36KZcVJQt6wA+uO+3D6rx3+UAGU7skx7pxVAD+fi0fdyEyyL9wvTai63vmTgUuRabG+rKk6xhUVOy1pu3ofz4v1iLlpWEZd18y2AflK2Xy/njSZlVLFoAGSQFKSQFUD/dSI3wPlslQF0d7L3abacns+KP+hs8xxUf8fZuH9brZKKjXL+4NXdyYOv5YaZbW2rTZzV/IPXeYNcN0fir17f/V7Mcnf2f+Uupg7f2Nwygaz9rGifpZJFvBRFL9m/Zq+zf80auzCDpCCFrAB6JbfizdFCArQq1kFyoxQAya0jN8Fqa4FUzyHr//3Dv83F9rbaxFnNH/7iT7Xv5qhokYWVU0W9B6vD7wCUE3Yu4+S9ncvJUfYkE8sz2KyB9JMyH6aJyA4guR9Zyk10UOxJimm+AKgYfAnQoukq2i6qvcLOtmrEOZf//Wm58CiJqTb1sr1PXHcANCvbly2WFUCtZsZJQd2yA0hu1fPZ2gQg+dZ5sYv5yb//53dHfdvqz9nKVZRbuQFQNfflcgKQSVJQp+wAkgvpbPMUu7Cak22AWruwbJP+5mhWbfhquVGsa+RG3+Ltf/6lXEUXQBQ7oKX4pD4G24Q/b+2bNru87V1YB0CGSUG7sgRoufe32f+Xi+ivsqNduYEvG8dE5SJ6dVSfVlnN/67YWWRr2ezguNyk5yI/sJ414iyzVfP67vflX302OWSd5IvorM1f1HPBJnwdotzPVYvu7UX0BqDNhKKdFGmQxyxLgOSxTjkLlNP8gdxW8jC+Aqj8oP4TzrZIsb4u2pfbKm/0o49njThli5K88rC82ISXYrMp6/CbEJf5YfzDqv3WYfwGIFGfQddOClLIEqDsj3RR7UZW2Yrl4afZi5uPxez7DUDyTN/epxtffghdnt37Kj92lo3+ey5++n1+zq6Ok59I/KCcuYoTg39ZLKlXzQPqOnwdQibwXX7isViCr9onEiuAZLNqXtROCupWWt/GL8XgmsTkqw+IrqQA+uHj4S+qAFBYJQRQtu7SWIwAoLBKCKBspfvBcCsAFFYJAQRxFACCSAJAEEkACCIJAEEkASCIJAAEkQSAIJLMAVrtW1xdZWKSbX/AjRCJKNAMZAiQFaRQDAEgiCSrXdhq/9fzzaVi+QXp1YXpB+viw4NV3mK1/2V+aZgkQl61VV4Ptnr8KyGvvSneKAPk1NTRD+q4EGfZATTPmKiuLC62+3xxc7Qo78qRH4qixWouOZnJRsXFgMX1GPmFYdUbVYANQPkMVL3ttl7IsSwBWmz2SqvHBUj/97r5Ydmi/u/+6bK4Uj0HIn+7eqMKsA3QY1yInIAsd2GnjWXNuSguC80vIN5rfFi1lJDsn142Li/O367fKANsAbSJCzEWCaDLEoibIyHvFds7bdO1BVDjasICoM0beYBtgMq3HRQJ+ZODGUgqY2QpeVhuz0D5LuyxhGPZuDsv/7j5RgnZFkD1Lg9iKzpA+VKmBGQ13wGosYiWN8wU0OQfV29UAeQ9qPJur/zloo7rtl7IsRzMQMvy3t9szbL3m81EUrb8Ml/JVIfx1bPNcm/1RhVAHrd/8lEe/VzM6rchzvJ8IhEzyNgFgCCSABBEEi7ngEgCQBBJAAgiCQBBJAEgiCQABJFkDtBbhZQfaCq2P3oCaRUAgNglkFYBAIhdAmkVAIDYJZBWAQCIXQJpFQCA2CWQVgEAiF0CaRUAgNglkFYBAIhdAmkVAIDYJZBWAQCIXQJpFQCA2CWQVgEAiF0CaRUAgNglkFYBAIhdAmkVAIDYJZBWAWkAJIQg+ckJpOQHQDvKH/9C8JMTSMoPgHYEgPgmAICcB4jtB0BbytiR/OgSlNb4e/ADoJY2U4/mJJTW+HvwA6CG2tBoIZTW+HvwA6Bau8BoEJTW+HvwA6BSnfPN8CSU1vh78AOgXEpShghKa/w9+AHQ2/6JZmASSmv8PfgB0OB+qvfztMbfg3/yAOkca/U0SWv8PfinDpDe2R41ZWmNvwf/tAESQtevIiit8ffgnzJAcl7R9ismobTG34N/ugAVRBj4OxFKa/w9+KcKUEWDkb+DoLTG34N/mgBtJhMz/+4klNb4e/BPEaAmBqb+bYLSGn8P/ukB1J5FjP1bk1Ba4+/BPzWAtndCFuW3QqQ1/h780wEov87QzRqGsAvcUWwA0iogIkCKK53DLsI7FBuAtAoYC0CbSSit8ffgB0CWKoOlNf4e/KkARFfGj+uAbuNB+opyFNb5ZRbp70cY3QXdrdgzyFRmIHr3Pr4MNboHsVuxAQBAuvIGUKXhtu4TiO8HQBS1wBGd6mzqLIH4fgBEktbE0ztJxQYAAOnKzwVh1NMI0QEAQLqKDBDWQC78UwZIkUJsAACQrgCQF/9kAFIsdsOOv4drYmP7ARBNAAgAkWTo380iNgAASFM8ANpNIzYAAEhTAMiPHwDRZOx3fVtHbD8AogkATQQgXw9HMPfTbwth5QdANAEgAESShd/tnYmx/QCIJgAEgEiy8dPurXaQgEs/AKLJyu/y3ujYfgBEEwACQCTZ+R3eXB/bPxGAvD1lFQABIJIs/Zt8YgMAgLQEgHz5ARBNtn5nz4eJ7QdANFn7XT0fJrYfANEEgAAQSfZ+mwdVO03AjX8aAKnvPwZARD8Aoongd/OIs9h+AEQTAAJAJFH8pr/24jwBF34ARBPJL2In4MAPgGgCQACIJJrf5BfvvCQAgABQZP8kAPL4q8tEv4idAADS6RYA+fMDIJqofu2fjfaVwOgAun1x+ORN/ur+7PBnLwGQ5wTGBtD92fH66sP85TfH63clS+MFqC+3IAmMDaDbLy7W159dlK/Gvwt7u078pxK4AXT97M369vOX+avflbuw9zIpDTpi/bM6rJPjKvWgyZ1WCdDT4xynQhR++/7GGfwB06YgBgUE9A8D1JyBqlcAyGMCYwOosQb65UQAohHEoYBw/mGA7s+eN47CJrELA0AGjQcBKs8DyUkoe/Xz+kCM0j13gEgEsSggmF8DIIUo3bMHiEIQjwJC+QFQdwAApNsYAHUHsCeISQGB/FEA6t04TMYfAGk2BkCKANYEcSkgjB8AqQIAIL3GAEgVwJYgNgUE8QMgdQBLgvgUEMIPgNQBAJBOYwCkDmBHEKMCAvgBUE8AAKTROAJA/duF0/hbEcSpAP9+ANQXAAANNwZAfQFsCGJVgHc/AOoPYEEQrwJ8+wFQfwAANNQYAPUHMCeIWQGe/QBoIAAAGmgcHqCBTcJt/I0J4laAXz8AGgoAgPobA6ChAKYEsSvAqx8ADQYAQL2NAdBgAEOC+BXg0w+ANAKYEcSwAI9+AKQRAAD1NAZAGgGMCOJYgD9/eICGNgbH8QdA6sYASCeAEPoMsSzAmx8A6QQQwoAgjgX48wMgnQAASN0YAGkEAEDqxgBIJwAAUjYGQLoBdAliW4AXf3CABjcD3/HXJIhvAT78AMgggB5BjAvw4AdAJgG0COJcgHs/ADIKoEMQ6wKc+wGQWQANgngX4NoPgAwDDBPEvADHfgBkGiD5Atz67QGyVPo/hZN+BT4UagYawx4gwQua/PkBkEWApO6M9OwHQDYBUnlCVgA/ALIKkMCj9gP5AZBdANY/+hrSD4AsA6gLSaQAR34AZBtAWUkqBbjxBwZoTN8EqGpJpgAnfgBkH0BRTDoFuPADIEKA7stcEyrAgR8AkQJ01ZNUAWQ/AKIF6CgorQKofgBEDLBbUWIFEP0AiBpgp6TUCqD5ARA5wHZNyRVA8ocFaJyXFG9VlV4BFD8AchCgXVaCBRD8AMhFgFZdKRZg7wdATgI0C0uyAGs/AHIToFFZmgXY+gGQowCb0hItwNIPgFwFqGtLtQA7f1CAxn1reVVdsgVY+QGQuwDCTQIASKmRA1TWl3ABFn4A5DKAcJEAAFJq9ADlFSZdgLEfALkNIFIvwNQPgBwHMHmeq4/+HQQAQDEDGD1S2kP/DgKwBWgSz6gEQACI5AZAAIhmxxpovV7N87+jB68AkI1/8gDdncwwA1H8U9+F3RwtABDFP3WA7k4AEMk/dYDWy/7VjyVAE/qtEhJBHAowaNwB0M2R8LGIBkBh+o8PkJ6Mu58QQCSCWBSg3xgA+Uhg6gBdCrG4fPQaANn6Jw7Q+aPvjhb12aDbF4dP3pSf3J8dAyAdP4EgHgVoN+4A6OZoIU8FlcdikpmrD8uPrg4BkJYfANUA3X5xsb7+7CL/5Pof/xkAafknDdD6Uu7Cbo4OCmievVnffv4yn4x++x/FLuy9TN0rI7Um9iM3Eyt3q96lPA1U8LN+96QG6Oo5YQ00sV/Ntp+CmBSg27gToKY2M1D2CgBp+60J4lKAZuNBgDZroKtDqecASMs/VYCq7zHqrzLuz55vjsIwA2n7pwqQ1KVEp76oozgPVExCAEjfb0sQmwL0GncAVKIz8J28afcAKFD/IwVIezhHA5AtQXwK0Grcsws76OMHAA36pwtQcR5o4LJEw+4BUKj+OQCkI8PuJwiQJUGMCtBpDID8JTBVgLxc0jpFgOwI4lSARmPlDHTz0SlmIKJ/0gCtl/2XJBp2D4BC9c8HIJe7MP2hHBNAVgSxKmC4sRqgc5czEAAK1n98gMpF9J7LNdBEAbIhiFcBg43VM9CAzLoHQMH6B0A08Rr/iQJ02bikFQCR/OYEMStgqHEXQB6+TAVAwfqPD5CHyzkMhhEAEfsHQDRxG39jgrgVMNC4AyAPuzAAFK5/BgC5X0RPGCBjgtgV0N+4EyAdGXUPgML1D4BoYjf+0wPo5mjgWwwAZOQ3JIhfAb2NO2egcyEGHi8FgPT9EwSo+ELV3VGYyRCODyBDghgW0NdYAVCBUN+ZIJPuAVDA/nkAlB3Iz/ovCTLpHgAF7J8BQHcn5V1hfSejTbqfOEBmBHEsoKdxB0A3R66fVA+AAvYfHyA9mXQ/dYCCDgAAoonl+AMgAETyAyASQEGXADzHP+CZVABEE8/xB0AAiOQHQACI5g93UTgAoonp+AMgAETzB3u6BACiiev4AyBrgMJ+l8h1/AGQtab2wzUKjXsYMAN5TyDUk9ZTmYH0uwdAhQL94h4Aoonv+AMgAETyAyAARPPrDQTjAroaA6BwCQAgK4AC3xbFefy1hoJzAR2NAVDABAAQACL5ARAAovl1xoJ1AbuNAVDIBAAQAKL5NQaDdwE7jQFQ0AQAkDFAoZ+vxHv8ARAAovmHh4N5AduNAVDYBAAQACL5ARAAovkHx4N7AVuNAVDgBAAQAKL5hwaEfQHtxgAodAIAyAig4D8VwX78ARAAovkHhoR/Aa3GACh4AgAIANH8/WOSQAHNxgAofAIACACR/AAIANH8vYOSQgGNxp4BCv+bsymMPwACQDR/37AkUcCmMQCKkQAAAkAkPwACQDR/z7ikUUDdGABFSQAAASCaXz0wiRRQNQZAcRIAQFrdm/MDgBIpoGoMgCIloByaVAooGwOgSAkAIJ3uAZBSAEinewCklmpskimgaAyAYiUwGYBuXxw+eZO/un56eHgMgFz5FYOTTgF540GA7s+O11cf5iR9/nJ9/Q8vDQCy4AcApVNA3ngQoNsvLtbXn11kr95JjL6ppiCd7gFQnyYC0PWzN/ncU+KUv3ovk9LQ0Lh/oIasMQ2PupZ3TxoA3Z89r97X4RczUK+E6BqghAp4azoD3b6o+QFAZL8QnQSlU0DeeBCgzRooOwo73ryv0z0A6tNEAJJ7reIorMUPACL7JwJQeR4om4SuDqVwFObMPxGAFNLo3oafKQEktTtGaRUAgGInsDMJpVUAAIqfwNYwpVUAAGKQQHuc0ioAAHFIoDVQaRUAgFgk0ByptAoAQDwSaAxVWgX4BMiKn4kC1BistAoAQFwSqA/n0yoAAPFJQLjpHwDRlNb4tySc9A+AaEpr/NsSLvoHQDSlNf5bEg76B0A0pTX+2xKpFQCAmCUgEivAI0B2/EwdoLcirQIAELsEOq+0D5kAAIobgO4nEgSASBoBQESCABBJYwCIRhAAImkUAJEIAkAkjQMgCkEjAch2BGJvv+gJlH57ggAQSWMByJ4gAETSaACyPiEEgEgaD0BJfBkEgNgl0PTzv6ocALFLoOVn/3QBAMQugbaf+2NKARC7BLb8zH+uxhtAaRyEckxg28/7d68BELsEdvymh/MAiKTxAWQ6mACIpDECpHiWWcAE1I0BELcEVABpjygAIgkAASCSxguQLkIAiKRRAlQ+1F4PolEAlMjVLBwTGPAPMwSASBo7QG8HGUoEoH6N6YdoOEruzGLn0BJmIDYJaPtVE1EiM1B/9wAoiN/LbyUAoLgBAvt3j8wAEElTA0iqzdAYAErlriaOCdj5GwwBIJImCtDbemdm8r0rPQEAxC4Bkt/oOw8XCQAgdgkQ/QDI3urEHz0BFwCREAJAcQPE9pt870pPAACxS8BdAZYMAaC4AWL7WwFsGGIAUDKPR+KYgOsCjHdmAChugNj+rgBGDAGguAFi+xUB9BkCQHEDxParA2juzABQ3ACx/f0BNBgCQHEDxPYPBhhiCADFDRDbrxOg93vX+ACl85hsjgmEKkD5nQcAihsgtl8/AADy4Y+eQGCAdhkCQHEDxPYbBCjh2WIIAMUNENtvFaA5EQGguAFi+60DVBABoLgBYvtJASRDAChugNh+agCzb+99AJTQ7+1xTIBDAQbfvAIgbgkwKUD3m1cAxC0BRgVoQASA2CXArIDuc0WbxgCIWwL8ClCcsS4aAyBuCbAsAAClkwDLAkICFPLWfg/+6AnwLCDgGggAxfWHTQAAsUsgrQIAELsE0ioAALFLIK0CABC7BNIqAACxSyCtAgAQuwTSKkADoNsXh0/ebL3qAYjIz8TG34OfG0D3Z8frqw/br3oACvuMUA/+6AmkVcAwQLdfXKyvP7tovVIDFPgRjx780RNIq4BhgK6fvVnffv6y9eq9TN2tBbufkYGCSL3R3z2psNm8kuoGEjNQdH8KM5AaIKyBovu5AWS2Boo/fmmNvwc/N4Duz57XR2HPh4/Coo9fWuPvwc8NoPLsj5x6tM4DxR6/tMbfg58dQAqxSN+DP3oCaRUAgNglkFYBAIhdAmkVAIDYJZBWAQCIXQJpFQCA2CWQVgEAiF0CaRUAgNglkFYBAIhdAmkVAIDYJZBWAQCIXQJpFWAPkEqKK82CKXb/E00AACEBkgAQEiAJACEBknAhPEQSAIJIAkAQSQAIIgkAQSQ5Aqh12X0EvTs8/PnFcDN/uj87/NnL4Wb+dP00zgi4Aaj9+IXwkreOxOx/vf7mOL+BN5puX2RbIEYCbgBq33oYR1H7lwMQVe27hwPKDUDR0m8o6gx0/ex3cXdhiQPUfvxCDF0/jbv9nh7n2zCa8l1YjCEYzwwUtf/4A5Atov/pt8kCxGENJNex0XT7y9gArSMtxFwdhT2PugaJvwv9JvYuLKs+yhYYyXmgq8PIp2GyAYh7IurdYZwtgDPREEkACCIJAEEkASCIJAAEkQSAIJIAUJ8uhRB7p403fvh6p81q/7Tv47ELAKl1d/LotYRoUb/ThKVDAx+PUgBIrXPJT0ZQ8R8pALQjAKTUzVE99eRkrPZ/PRfiYLX/pRCSqbsTIWb5Z8Uni5X8OGbGMQSAlGrMJyVAp/K/q/mDV3cns3X1P/n2PAPq8sErzEBQQ6vHr+qXLYAW+evlg+zjZU5N9RYAghpSzUDydbZ7W8rdWPFe8+OpCQApVa2Bbj46BUBKASC1ysOvanHT2oU9frWUJ4jKXRgAgjq0OQ90c3SQ/WvvVE5KXYvoEqDGcdtkBID6dF6diZZH6J98dJq9McsP47PD99ZhfAGQ/Dh2yqEFgEw1xf1UjwCQqQBQSwDIVACoJQAEkQSAIJIAEEQSAIJIAkAQSQAIIgkAQST9P+N+ZGhepQ35AAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<p>Predictions for new data can be made using <code>predict</code>:</p>
<div class="sourceCode" id="cb12"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb12-1"><a href="#cb12-1"></a><span class="kw">predict</span>(opt_cut, <span class="dt">newdata =</span> <span class="kw">data.frame</span>(<span class="dt">dsi =</span> <span class="dv">0</span><span class="op">:</span><span class="dv">5</span>))</span></code></pre></div>
<pre><code>## [1] "no" "no" "yes" "yes" "yes" "yes"</code></pre>
</div>
<div id="features" class="section level2">
<h2>Features</h2>
<ul>
<li>Calculation of optimal cutpoints in binary classification tasks</li>
<li>Tidy output, integrates well with functions from the tidyverse</li>
<li>Functions for plotting ROC curves, metric distributions and more</li>
<li>Bootstrapping for simulating the cutpoint variability and for obtaining out-of-bag estimates of various metrics (as a form of internal validation) with optional parallelisation</li>
<li>Multiple methods for calculating cutpoints</li>
<li>Multiple metrics can be chosen for maximization / minimization</li>
<li>Tidyeval</li>
</ul>
</div>
<div id="calculating-cutpoints" class="section level1">
<h1>Calculating cutpoints</h1>
<div id="method-functions-for-cutpoint-estimation" class="section level2">
<h2>Method functions for cutpoint estimation</h2>
<p>The included methods for calculating cutpoints are:</p>
<ul>
<li><code>maximize_metric</code>: Maximize the metric function</li>
<li><code>minimize_metric</code>: Minimize the metric function</li>
<li><code>maximize_loess_metric</code>: Maximize the metric function after LOESS smoothing</li>
<li><code>minimize_loess_metric</code>: Minimize the metric function after LOESS smoothing</li>
<li><code>maximize_spline_metric</code>: Maximize the metric function after spline smoothing</li>
<li><code>minimize_spline_metric</code>: Minimize the metric function after spline smoothing</li>
<li><code>maximize_gam_metric</code>: Maximize the metric function after smoothing via Generalized Additive Models</li>
<li><code>minimize_gam_metric</code>: Minimize the metric function after smoothing via Generalized Additive Models</li>
<li><code>maximize_boot_metric</code>: Bootstrap the optimal cutpoint when maximizing a metric</li>
<li><code>minimize_boot_metric</code>: Bootstrap the optimal cutpoint when minimizing a metric</li>
<li><code>oc_manual</code>: Specify the cutoff value manually</li>
<li><code>oc_mean</code>: Use the sample mean as the “optimal” cutpoint</li>
<li><code>oc_median</code>: Use the sample median as the “optimal” cutpoint</li>
<li><code>oc_youden_kernel</code>: Maximize the Youden-Index after kernel smoothing the distributions of the two classes</li>
<li><code>oc_youden_normal</code>: Maximize the Youden-Index parametrically assuming normally distributed data in both classes</li>
</ul>
</div>
<div id="metric-functions" class="section level2">
<h2>Metric functions</h2>
<p>The included metrics to be used with the minimization and maximization methods are:</p>
<ul>
<li><code>accuracy</code>: Fraction correctly classified</li>
<li><code>abs_d_sens_spec</code>: The absolute difference of sensitivity and specificity</li>
<li><code>abs_d_ppv_npv</code>: The absolute difference between positive predictive value (PPV) and negative predictive value (NPV)</li>
<li><code>roc01</code>: Distance to the point (0,1) on ROC space</li>
<li><code>cohens_kappa</code>: Cohen’s Kappa</li>
<li><code>sum_sens_spec</code>: sensitivity + specificity</li>
<li><code>sum_ppv_npv</code>: The sum of positive predictive value (PPV) and negative predictive value (NPV)</li>
<li><code>prod_sens_spec</code>: sensitivity * specificity</li>
<li><code>prod_ppv_npv</code>: The product of positive predictive value (PPV) and negative predictive value (NPV)</li>
<li><code>youden</code>: Youden- or J-Index = sensitivity + specificity - 1</li>
<li><code>odds_ratio</code>: (Diagnostic) odds ratio</li>
<li><code>risk_ratio</code>: risk ratio (relative risk)</li>
<li><code>p_chisquared</code>: The p-value of a chi-squared test on the confusion matrix</li>
<li><code>cost_misclassification</code>: The sum of the misclassification cost of false positives and false negatives. Additional arguments: cost_fp, cost_fn</li>
<li><code>total_utility</code>: The total utility of true / false positives / negatives. Additional arguments: utility_tp, utility_tn, cost_fp, cost_fn</li>
<li><code>F1_score</code>: The F1-score (2 * TP) / (2 * TP + FP + FN)</li>
<li><code>metric_constrain</code>: Maximize a selected metric given a minimal value of another selected metric</li>
<li><code>sens_constrain</code>: Maximize sensitivity given a minimal value of specificity</li>
<li><code>spec_constrain</code>: Maximize specificity given a minimal value of sensitivity</li>
<li><code>acc_constrain</code>: Maximize accuracy given a minimal value of sensitivity</li>
</ul>
<p>Furthermore, the following functions are included which can be used as metric functions but are more useful for plotting purposes, for example in <code>plot_cutpointr</code>, or for defining new metric functions: <code>tp</code>, <code>fp</code>, <code>tn</code>, <code>fn</code>, <code>tpr</code>, <code>fpr</code>, <code>tnr</code>, <code>fnr</code>, <code>false_omission_rate</code>, <code>false_discovery_rate</code>, <code>ppv</code>, <code>npv</code>, <code>precision</code>, <code>recall</code>, <code>sensitivity</code>, and <code>specificity</code>.</p>
<p>The inputs to the arguments <code>method</code> and <code>metric</code> are functions so that user-defined functions can easily be supplied instead of the built-in ones.</p>
</div>
<div id="separate-subgroups-and-bootstrapping" class="section level2">
<h2>Separate subgroups and bootstrapping</h2>
<p>Cutpoints can be separately estimated on subgroups that are defined by a third variable, <code>gender</code> in this case. Additionally, if <code>boot_runs</code> is larger zero, <code>cutpointr</code> will carry out the usual cutpoint calculation on the full sample, just as before, and additionally on <code>boot_runs</code> bootstrap samples. This offers a way of gauging the out-of-sample performance of the cutpoint estimation method. If a subgroup is given, the bootstrapping is carried out separately for every subgroup which is also reflected in the plots and output.</p>
<div class="sourceCode" id="cb14"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb14-1"><a href="#cb14-1"></a><span class="kw">set.seed</span>(<span class="dv">12</span>)</span>
<span id="cb14-2"><a href="#cb14-2"></a>opt_cut_b <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, <span class="dt">boot_runs =</span> <span class="dv">1000</span>)</span></code></pre></div>
<div class="sourceCode" id="cb15"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb15-1"><a href="#cb15-1"></a>opt_cut_b</span></code></pre></div>
<pre><code>## # A tibble: 1 x 16
## direction optimal_cutpoint method sum_sens_spec acc sensitivity
## <chr> <dbl> <chr> <dbl> <dbl> <dbl>
## 1 >= 2 maximize_metric 1.75179 0.864662 0.888889
## specificity AUC pos_class neg_class prevalence outcome predictor
## <dbl> <dbl> <fct> <fct> <dbl> <chr> <chr>
## 1 0.862903 0.923779 yes no 0.0676692 suicide dsi
## data roc_curve boot
## <list> <list> <list>
## 1 <tibble [532 x 2]> <roc_cutpointr [13 x 10]> <tibble [1,000 x 23]></code></pre>
<p>The returned object has the additional column <code>boot</code> which is a nested tibble that includes the cutpoints per bootstrap sample along with the metric calculated using the function in <code>metric</code> and various default metrics. The metrics are suffixed by <code>_b</code> to indicate in-bag results or <code>_oob</code> to indicate out-of-bag results:</p>
<div class="sourceCode" id="cb17"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb17-1"><a href="#cb17-1"></a>opt_cut<span class="op">$</span>boot</span></code></pre></div>
<pre><code>## [1] NA</code></pre>
<p>The summary and plots include additional elements that summarize or display the bootstrap results:</p>
<div class="sourceCode" id="cb19"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb19-1"><a href="#cb19-1"></a><span class="kw">summary</span>(opt_cut)</span></code></pre></div>
<pre><code>## Method: maximize_metric
## Predictor: dsi
## Outcome: suicide
## Direction: >=
##
## AUC n n_pos n_neg
## 0.9238 532 36 496
##
## optimal_cutpoint youden acc sensitivity specificity tp fn fp tn
## 2 0.7518 0.8647 0.8889 0.8629 32 4 68 428
##
## Predictor summary:
## Data Min. 5% 1st Qu. Median Mean 3rd Qu. 95% Max. SD NAs
## Overall 0 0.00 0 0 0.9210526 1 5.00 11 1.852714 0
## no 0 0.00 0 0 0.6330645 0 4.00 10 1.412225 0
## yes 0 0.75 4 5 4.8888889 6 9.25 11 2.549821 0</code></pre>
<div class="sourceCode" id="cb21"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb21-1"><a href="#cb21-1"></a><span class="kw">plot</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABjFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYZGT8ZGWIZP4EZYp8aGhozMzM6AAA6ADo6AGY6OgA6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNs/GRk/GT8/GWI/P4E/Yp8/gb1NTU1NTW5NTY5NbqtNjshZWVliGRliGT9iGWJiP4FiYmJin9lmAABmADpmOgBmOmZmZmZmkJBmkLZmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SBPz+BP2KBYhmBvZ+BvdmOTU2OTW6OTY6ObquOyP+QOgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+fYhmfYj+f2dmrbk2rbm6rbo6rjk2ryKur5P+2ZgC2Zjq2kDq2kGa2tpC2ttu225C229u22/+2/9u2//+9gT+92dnIjk3I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrbkGbbtmbbtpDb25Db29vb2//b/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T////1+by3AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAc5ElEQVR4nO2djZ/btnnHqYtPe3Flp2sy7S3npN7W9eK1a3PuMndbss7KliW5LOkW37ZudS65vFl12jWXRCf3ZJn/ePHghQBJgAIBkaLE3+9jWxTJB3gAfA08oEgiSSEoQsmmHYC2WwAIihIAgqIEgKAoASAoSgAIihIAgqIEgKAohQG0nOyfl/cNa6fisJj/9WktBxaHQ+t+qHl1EqDpHgDaFm0pQHkBoM0pHCD292dPJ4Mf0PeHo+TKzzkOj24lyTfvU5uOZ2zncWrsq7bIji4nSZKMeT6CJM7Hw6cTbsqomyZ7P+egqH3sBMrsvgJIJQk1rxiAEq4j1s70eeUWa+b5iDYHx6xNvzUSW3pftUV21ABoPmLnpjNmMVWmy8lTo2T/CwIl27c4fGqkEmH7sySh5hUF0JAaccjab/Bm+oh9ZfsHL6Xp5yPax44u/zu3r9pCH9VDmOhQTvZOl5O9+8QTP22sHFD7WGZ/qDKj/SpJqHnFAETNTMPLfET9BbWk2CIAFoc8GGFtr/dVW+ijRgw05Z0ZR+FXP/uXEUeEDspYR+4TmdER2q+TjKkYyE9RMdC5CIRpiFFbQoNj2eoMAL2v2kIfNdqeYOAni2FJ9jGpOVRxgMYyM9qvk1xHBUHV6jhAtJP1YjRIfevf/vfXhyZAeh8A2pzWAZAYM8TQdCTP0KOK3ldtYQWIQfHW4VBF0yIGUgDpfYJWPYSpJKHmtQ6AWEj8Eo+BKSR+k82iJxQD6bhW7au2MAHSncd89D0xgrEAm83OdbYEitrHMhunZhAtk1xbLUFOrQMgMQ7xSbkcPsZqZk2NmO2rtjABUtN4+p6IoFmcZgKk98lrBuxEfjRLEmpeawEo/fxWcuV9vjW/lSRPvaSu7T3Dr+apfdUWxtHFLZrRC/F5vbg2eOXNE91R7Z/rfeKq5R9kFxJVklDzaurXeP3rArTTAkBQlAAQFCUABEUJdyRCUQJAUJQAEBQlAARFCQBBUQJAUJQAEBQlAARFaU0APbqfzq/Zb+By7X9UfmzCPNVyONPMuFODbdvM2L5Sxm4nXT5WH4LSdQFUVcv122zl4QJADrNiEmGUAKBqAaDwQ1AaDBDdzEV3ml57nW7oorvbx3zQeINvjZLsPvhx1gKZCT0qJk8U5uYxceSIp8jNslTeEKkuDpPB6xIauU1J8tOE2fz6vyZ7/0dJidTzOR7rzGSKaVaO9IR+wJsOi/5KH2Qm+hMKv6FsKO9tpju8hqKWWZON+ON+9A/d5XXE726WAIk7WY9Ucx6LhhHmWXIqCR3XZKmMVKp0t6MASG2reGc+OhKpShZHdNfbMJ/jtWPtu0iRUlKnzvhNjUcFf3MlkZmkuPNaKAwgPm7M5A3zum3EV7nvN3RPoY5lFRF5gMSpWXIqCR0BZ6mYp8pmV9t0+PpplnqWhJmUdrLoOzeT25wYkZThb+aDyuQ67rbOFAgQv3FUtg2rdQWQbCr5z4w/WyNbQsUqhebk5kZy5mGZl06F/TPlp4oWVNt07ISPN2mWqmpv6ZwJkOm7AkieSsOXHMFy/gofVCbZJ9QkQIvDwbGBQiBAuVQqAOIBkQyHIgCaX39fjGCmv8oHlYnxCQUCRM/dZMPA9VMbQLyhZoPVQ9j10yy5MkC5VIwBiB/TIx8/V5NcGrpMgHRmGiC1vZx83xgM5afyQWWS++y71hNEy2gzDxD9tx1lACkTinyXk4EINyxBdNZ/yY7ATIXvHxpB9FAF0ZwlFcQoYGRknMvRCKJNgOSp2UMgqemT8kFloj6jK38XtIZpPNX4STIsxUAsUhi8JUcQbcLnwD988ViYSHNzGp9Zi5Y0U6F/HNN4GaVIR2RSr1tyPC5cT5A9kDw196Rsdpr0IctEfUKxFxIj/xt28H8xZlg1BYDymuJ51noCQKb4xUWojnA7BxQlAARFCQBBUQJAUJQAEBQlAARFKQygB42r6w5AUgAozAFICgCFOQBJAaAwByApABTmACQFgMIcgKQAUJgDkBQACnMAkgJAYQ5AUgAozAFICgCFOQBJNQ3QJ3/6D1evfvfBg0///OrvvrO+9vNM5e3vMgc+lHl/fLWOC0H10kM1DtAfPffg49/78MHb4mNd7eeZysfP0R+W9we//+DTv3yHPtbkACTVOEB/8g79pdb79C9e8bdbjwOffufDf3+F8mX518oeAPmqJYDo47Mftw7QZz/+x+/QCHb16u+8wjpD+ndNDkBSu90DPfjgr57jY5dUjVE0qF56qJYA2lAM9OCTP2bQirwpewC0drUF0GZmYQ8++/t3+AyQxq63MQtrQDt+HeiTP2vIAUhqtwH6oEbUXM8BSGq3AWrOAUgKAIU5AEkBoDAHICkAFOYAJBUG0NdS2YbYTEp7i2dUbuZ2d9wBSAoAAaAoASAAFCUABICiBIAAUJQAEACKEgACQFECQAAoSgAIAEUJAAGgKAEgABQlAASAogSAAFCUABAAihIAAkBRAkAAKErhAN0U2kz7ORJyO2DxFgCtQwAIAEUJAAGgKAEgABQlANQUQNNE6Jv3+dfl/zydJE/9jVxRU3x7KTqTzQsANQ1Qwtcof3RLfLnCcaKFyEnPRueycQGgxgASi9sv/ysZ80Xon2HoPHotoVV9F4fJs/+fpr+8lRytSqXzAkANA8RoGVN3JJcT5xvqm7FI9MNRcuWYQKM9y8mQ/k6Tvdd4IovDoejD5HDYJQGghgH6zQmjZTmRNDEW9s/1t1+pk2c0oA2OcwA9NUr2fzk64keP5Kg36Ng66VsLUO3SSG/Dkg2SGQPxLkToZO/U+Ca1nDAyHnLSNEBi6KPvzIadwULuz0dFy41rSwFyJNSpHkjgM3j2PF0J0HwkRzgTIN5NTRlZdL48Q42L3REAanQIezh6hqKc1QDJaNoEiMdHBA6NYDPZnXVuDANAzcZAs0QQYY+BfvGMiIrdANEHYw4A9RWg9ITPuOyzsMWhJCkbwqiDEh2WBIiNYW9Rh5Uh1jUBoIYBEpC4rgNloQ/rWeasc2K8/YBO1gDNR9+jq0XsjDeZ/QQxUN8AYt0NRTwL65XoLBYSI9SR3LhySwPEaJJjIdfYktNGBYCaBogRQKNP/rewR6+N8r+FPRwlA/71F6PkmS90DKQAZNDd6uSvZwCoKYB6IgAEgKIEgABQlAAQAIoSAAJAUQJAAChKYQCRZJOsJ7G6AkBdEXogABQlAASAogSAAFCUAFBDANkzyMt5ZAtMVDkBEAAKMlHlBEAAKMhElRMAAaAgE1VOAASAgkxUOQEQAAoyUeUEQAAoyESVEwABoCATVU4AFAHQ5d9+xD8fv3rwwlfZR97DraAhwESVc3cBku41CNDFwfMcoCfv3k2//Lb6AEAAyMMBprMb74ke6PFPPqLOSH4AIADk4QCXxOXy5a/Sx/90T36wHd9gCqrX7ko+GGs5EJQcAOKSAF28wMmRHwUPt6I7sR1JciqZAKA4B0yASj3QlgJkRcZlAoDiHDAB2t4YyIUMYqCv2wToybt3xCzsTvdnYR69jF8uACjOgQwg+tvp60BuZKJyAUBxDnhVUTmDvBoCqIBMM7mocgKgXQCocmACQBXN01uASsi03c+pcgKgbQFoRfgLgL4GQIUDNmQaCpvqm6hyAqDuAVQ1yQZAFZsAiETgtE1DgIkqZ8cB+vKAdFdtPl+8zrtzAIl+BwDZXAntgS7E1bmzu5bm2TGAVkY6AKhq095+8vfJJz+9V3DAktB2A6SjHgBkcyUQIPnz0uNX1VhWcbeNWRrhXv7bzZJFY6rdTl4/ZgKgqk0rQOoGicsf3TN6IUdC29sDrfFXqnZMVDk7D9CF/n3SiIMcCW0rQMUpOwCyuRIG0NkdI9/dBKh8xQcA2VwJAigbtagnevIfOzaNp6uFtguGAMjmShBAIgSim26+PDi4cc90wJLQlgFU/3ozAKrarNF+joQAUGsmqpwACAAFmahyAqBNx0DWA26T2gc2B9CF+O3JdcMvAIpXey/WbkzuIvDA9dvuB78BkJeH1T2Q64DbpPaBjQ5h+mmn8kNPAMjLw34DxPoc54PfsknM0zu11MEWAFT/uaztAujy9o177ge/0QMBoBUApUbXU37wGwB5edhvgNKzu4iBwmTPIKekMzQEmKhyugGSg5bzwW8ABIAqAVI/PuE6EACyHPAAaEXtACAvD92t0dRT6+2YqHICIAAUZKLKCYAAUJCJKicAAkBBJqqcAGhTAFU9gAqAbK4AoJwAEADy8dDVGpXv9AFANlcAkKHqR+ABkM0VAKSVOI9UH+iOiSonAAJAQSaqnABoEwB17XVjASaqnABoAwCVV54oCADZXAFASgCIUgBAXh4CIFft9B6g7AYX9R4+60v4LK2x+j1SAMjmyo4BlH/QiW6/s76EDwC52q/vAOVu8qXbxe0v4QNArvbrO0C5xwyoK/J6CR9pB55H1QJAoQCZDzrxT/tL+Mr/nRPnkdUHumMCgOIcyPdA+j18pXeoASBX+/UdIDMG0u/hA0BeohT6DpDxoJMYuOwv4QNAAMghcR2IL3kpRjLrS/hKreHzOmgAZHNl1wDy8xAAuWoHAHl5CIBctQOAvDwEQK7aAUBeHhZbw3wjR2doCDBR5dxSgBzagsVWAJCqHfRAXh4CIFftACAvDwGQq3YAkJeHhdbIvZSsMzQEmKhyAiAAFGSiygmAAFCQiSonAAJAQSaqnACoVYDy72XtDA0BJqqcAAgABZmocgIgfwcWh4PjWlVUzgAAackmMXft/FIHJ0myf17PQwDkqp3+9UCkxWGSjH2rqJRo4dX0naEhwESVEwDVdYAQ2jv1qqJSogDIqJ1+AjRNkiEbylYPZNZEAZBROz0EaDlJkiPamK3ugqyJAiCjdvoH0OLQY+iq0k49USgFgOoA5Nv9pPYeqNABdac7CTBR5QRAACjIRJUTAPk6ME2Uhr5VVEwUAJm10zeAsh7Iv4oKiRb56Q4NASaqnAAozAGvKiokCoBytdMzgFj3Q5cQSaExEADK1U7PAAqpokKiAChXOwDIy0OdaImf7tAQYKLKCYD8HVgcev2MmvcQALlqp38A8Zm870SsnCgAytdODwFK+R1BfneVlRMFQPna6SdATCdhs7AyP92hIcBElRMA1e6B/H5RLSUKgAq100OAvMevFACtrp3+ART3UwYAKtRO/wCqXUW5RAFQoXZ6BlDsTxkWfrpDQ4CJKicAaqkHAkDF2ukfQFE3lAGgYu0AIC8PAZCrdvoGUOQdiQCoWDt9AyhyGg+AirXTP4AKypa8VGtd6h1lgGz8dIeGABMAVNOB0jTeWPJSLNGTXwOzkCgA6jtAJenlnuQqc7k1MAEQAFohveCcXOtS77AsebmLD6VyAaAaDohhTM7i9ZKXcq1Lcw1M9EBdAUivxW4NUdsF6GSYTvdOp2Ian1t0l+Kg/I5CogBoQwBlS0jaQ9RWAWId0HIyVBcS8yEPc7QqBrLy0x0aAky2BCC9CrK9edoGiO6rn6lZmFryUq51aayB2VOA/C/VUwqtAKTXYq8MUW2lEe7lv90sWdTRcjKeDY5pIFO+ySUv5VqXFdeBAFBelEIrAOm12O0harsx0HyUDH3eTtZPgOr81kMptDcL43GQPURtF6A6sjpWUGdoCDApA1Tntx5KoWWAOhAD1ZLVsYI6Q0OAiSpnRRB9eZuHH3JkLw3wrQCk12K3h6itD2FhdyTa+ekODQEmVoByFUTjBQtB5PS5PItu7ToQi0/54uwbvw5Ec3hv5RLtCUD5CrogXLKrG+URpL0hrKJ5WgUo/HaOngBUriDWC8ngtTyLtkyMd3ypg+UEAOktqXwPVKggijvk9Lk8i+5fD+R5M2t/ASpW0ONX72TT5/Isun8AhT/W0xOAChV0eduYPiMGqqlcoj0BKC/Bj5o+l2fRAAgAVQJEN1LQhaCNXgfqGEDTJDmaBvyU0ROA/Md4SqGHAJ3s/1rc0QGAKnqgxYur32BCKfQPIH47x1HIg4W9Aiid+S2GBYC8POwjQBjCHA5MaQjzfFdrLtF+AeS5HF8PAUpnFCL6ves3l2hPAJJBtMdb3CiFPgJUQ7lEewJQvdoBQF4eUqIOfrpDQ4CJKicA8nZgyrpm71W/c9rZpwrTYtmmnmN8HwGi++npGlBAEN2bHmhK8y+fCqIUegbQcjKmO+6OaC7mcynaTLQvAHXwqYzuAMQrh3qhgOtAAAgAicrhnQ8A4lsWgDCEVQ5hR/KeX78Hw8xEewMQgugKB1jvw0Ogmd+KT2ai/QHIV5RC3wCilTLGrBPyXC3DTBQA2WqndwDVryKVaG8Aoqmq1+0ulAIA8vKwVwDx9074EEQpACAvD/sEEKbxAMjbRJUTAAGgIBNVTlwHAkBBJqqc+VmY7w1TlAIA8vKwXwDVqR0A5OUhAHLVzpYAdDOntTngVUUqUQBkqx0A5OUhAHLVDgDy8hAAuWoHAHl5CIBctQOAvDwEQK7aAUBeHgIgV+30HiD9vhL5Plu9MAwAAkAr60C/t1a+z1YvDAOAANBqgPQ72+T7bPXCMACo8wA5ZFlsJQ9QWLJW5d8aybb0wjCF5WB2+bnCbQXIkVCbPVDuvbX0BkC9MIzpIUvJ1QF1pzsJMAFAcQ7keyD+PluuLA4yUgJA1vbrO0DGe2vl+0hJAMhLlELfAdLvrZX86IVhAJCPLGFpF5c6aAwgvWKhep+tXLgw7yEAcrVf33sgTw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPLyEAC5agcAeXkIgFy1A4C8PARArtoBQF4eAiBX7QAgLw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPLyEAC5agcAeXkIgFy1A4C8PARArtoBQF4eAiBX7QAgLw8BkKt2AJCXhwDIVTsAyMtDAOSqHQDk5SEActUOAPIVnkwtqZMAMWcS4U8FQPITPVCsiSonAAJAQSaqnAAIAAWZqHICIAAUZKLKCYAAUJCJKicAAkBBJqqcAAgABZmocgIgABRkosoJgABQkIkqJwACQEEmqpwACAAFmahyAiAAFGSiygmAAFCQiSonAAJAQSaqnAAIAAWZqHJ2HSC5jknqWMoEAG3KRJWz4wCpdUzS1L6UyTYA5OSnOzQEmKhydhwguY4J+8e+lAkA2pSJKmclQPxF7HJZLL06VpsAkcR6As6lTDIJgMyvN8uLrajPxqWL0l+ALijmkMti6dWxVO20BRC9EZ7JvpQJeqBNmXgAdHbjPdYDySUhjJUh2gVIr2OSWlaiAECbMvEASAxhclEavTaNGkEsg0EDjx8Y65ikHQNID+quUV6o5wDJZbFyq2O11wNpfuxLmWwQID2oO0d5oZ4DVOqBWgVIrmNCjliXMtkgQHpQd47yQj0HaNMxkF02i5YB0v+lnKN8H7QSILksll4diwSAzCUvnaN8OYO8OtOdBJh4A7T560AdBaiiBwJAq0Qp9B0g7xhoC2gIMAFAcQ6YS146R/lyBnl1hoYAEwAU50BqLHm54jrQFtAQYAKA4hzwqqJyBnl1hoYAE1VOAASAgkxUOQEQAAoyUeUEQAAoyESVEwABoCATVU4A1BBAWgG/amyRyTYDVGACAG3CBAABoCgTANQ4QLstAASAogSAAFCUABAAihIAAkBRAkDNAFT9yMYKE/lCgNzbAFaYyHNr5CJvN/fIJc1uc7KUBQA1AtCKRzaqTdQLAXJvA6g2kefWyYVEd+GuzEU+YeooCwBqBKAVtytWm8gXAuTfBlBtIs+tk0sqSF2di3zC1FEWANQIQCtumK42IbEt/TaA1Sby3Jq5UDeyOpc0G8JsZQFAjQC04pGNapNU3CWbfxtAtYk8t14u/HN1LmkGkK0sAKiLPZB+IUBlhFJI9OxuvVwuskh4VRyEHqhtgKJiIPOFAJVNW0j07G69GOgse22FJ0CIgVqchVU+slFtIvnJvw2g2kSeWycXOXCtziXNALKVBQA1eR3I9chGtYlxheZGdXCic5Hn1shFjUKrc5EvSbCXBQA1A1BvBIAAUJQAEACKEgACQFECQAAoSgAIAEUJAAGgKAGgbgO0ePG4tG+aJMmgvDvT/NoxO2fwxrVjc8+j+404CIC6DdBJmZTp3mmazpKjKrPFYfHw/FoFchECQF0GaHFY7mqWE87Gyf55hWEZFwBUcqAHAM2G5XZfTsZya37t9SQhjpaTJKFuiT6HtP+NEdsg09ye8QnbTKfDtboYD5DRJO0BxKXBMQByHL1ZOHlbZOk4ZgwFcWy0d7qcDFP6m073z+mTxi5mQ2bsb2HPjFEmO7C1aUsB+lp7sMs9kGPkoSia+pXRET+BuCBO1LkaoMIeztL107U6GL68heX/dANrZTjUb4BS4mX/nB9jUBBPTGMOUmoCVNhDw9eaRzD0QNsCEFEyzh/IAJLx9GqA5tffX/MIBoC2BaDiHmKCD2HXT2eD49y5ziGMBUDfX/MIBoC2DiBxaYjCYx1Esy6IUaQC6nwQLfeIK0PTZM0jGADaPoD4aEbDGZ/GExA0WRdUyUm7AsjYk55kcfdaBYC6DVCVAq4NrnsOBoB6BtB0vPqceto9gPJ+OQCq/rqrAM1Hlb9/BAkAbS9AnRAAAkBRAkAAKEoACABFCQABoCgBIAAUJQAEgKIEgABQlAAQAIoSAAJAUQJAAChKTQGkjgKgHRcAAkBRAkAAKEoACABFCQABoCgBIAAUJQAEgKIEgABQlAAQAIoSAAJAUQJAAChKAAgARQkAAaAorQ2gm/n2A0A9EQACQFFqGiCzSQDQDgoAAaAoASAAFCUABICi1BJARssAoJ0SAAJAUfIGKLcecHsA6WxLDgCgLsgXoCfv3jVWJG8NIJ1t2QEA1AX5AvT4Jx+p5efTtQOkjpYB0tmWHQBAXZAvQJcvfyVXqv8GU2Ri/tLZbsgBaIV8q/ziBdV+JEdPsv4nU3W2G3IAWqH6PRCptfaz9UCtOgCtUHgM1Er7VcZAbTgArZD/LOxOcRLURvvpbDfkALRC4deB2mk/kS11PRtyAKpW+JXojbbfxh2ApAAQAIoSAAJAUQJAAChKAAgARQkAAaAoASAAFCUABICiBIAAUJQAEACKUtwdNPb7chx366zh5O450HsBIAAUJQAEgKIEgABQlHAXMRQlAARFCQBBUQJAUJQAEBSlCIByNylnujg4eP6j8slP3j24ca+09/K27WT+8AU7dHC34w5AMQDlH1ZXorov703Ts7v80cC8Hr/KkijtvaA2pUfALn9UbvEuOQClMQDlH9QyZdlLJ1tOzD0sKHV24z1mf0EMnFX2ABt3AEpjALJWPpelA7h8+T8tI4gjCdX+9tS74wCUxgCUf1hd6/K2Nda4y1srLz6CWJpVtB89SthpB6C0mR7I9n/a8V/99sHf/dTRfo9fXdF8G3cASpuJgSyhw+N/djZ2OTiRk6BV8cfGHYDSuFnYHUuw4RpXzqwjCDvRFq+w9vNovo07AKUNXAf68sB2vYVOtlxwuTiwJcFn4gek6jbcuAMQrkRDkQJAUJQAEBQlAARFCQBBUQJAUJS2A6D5teNNuwDZBYCgKAEgKErdBGg5GbN/p3un81GSJGMCiDNE/ywnSbJ3umkPIaluApRO98+JosXhkeBIA7ScDMVhqBPqKECEC/v7m3OxbQA0o96HgwV1QR0FiMYw3s3M2BA2MAGaJlzjTXsICXUUoHS2/wVjaHE4OC70QBi9uqWuArR48Y3rpwwj6oRyPRD7tmnfIENdBSg9SYacnXQ+4gAtDsdsYBuwIJpBBYo6o84CNEsoTj5hEdBbh0c8pmYz+h++KKbx4Kcz6ixA0HYIAEFRAkBQlAAQFCUABEUJAEFRAkBQlAAQFCUABEUJAEFR+i1V8T4NlX3iKgAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<div id="parallelized-bootstrapping" class="section level3">
<h3>Parallelized bootstrapping</h3>
<p>Using <code>foreach</code> and <code>doRNG</code> the bootstrapping can be parallelized easily. The <strong>doRNG</strong> package is being used to make the bootstrap sampling reproducible.</p>
<div class="sourceCode" id="cb22"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb22-1"><a href="#cb22-1"></a><span class="kw">library</span>(doParallel)</span>
<span id="cb22-2"><a href="#cb22-2"></a>cl <-<span class="st"> </span><span class="kw">makeCluster</span>(<span class="dv">2</span>) <span class="co"># 2 cores</span></span>
<span id="cb22-3"><a href="#cb22-3"></a><span class="kw">registerDoParallel</span>(cl)</span>
<span id="cb22-4"><a href="#cb22-4"></a><span class="kw">registerDoRNG</span>(<span class="dv">12</span>) <span class="co"># Reproducible parallel loops using doRNG</span></span>
<span id="cb22-5"><a href="#cb22-5"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">pos_class =</span> <span class="st">"yes"</span>,</span>
<span id="cb22-6"><a href="#cb22-6"></a> <span class="dt">direction =</span> <span class="st">">="</span>, <span class="dt">boot_runs =</span> <span class="dv">1000</span>, <span class="dt">allowParallel =</span> <span class="ot">TRUE</span>)</span>
<span id="cb22-7"><a href="#cb22-7"></a><span class="kw">stopCluster</span>(cl)</span>
<span id="cb22-8"><a href="#cb22-8"></a>opt_cut</span></code></pre></div>
</div>
</div>
</div>
<div id="more-robust-cutpoint-estimation-methods" class="section level1">
<h1>More robust cutpoint estimation methods</h1>
<div id="bootstrapped-cutpoints" class="section level2">
<h2>Bootstrapped cutpoints</h2>
<p>It has been shown that bagging can substantially improve performance of a wide range of types of models in regression as well as in classification tasks. This method is available for cutpoint estimation via the <code>maximize_boot_metric</code> and <code>minimize_boot_metric</code> functions. If one of these functions is used as <code>method</code>, <code>boot_cut</code> bootstrap samples are drawn, the cutpoint optimization is carried out in each one and a summary (e.g. the mean) of the resulting optimal cutpoints on the bootstrap samples is returned as the optimal cutpoint in <code>cutpointr</code>. Note that if bootstrap validation is run, i.e. if <code>boot_runs</code> is larger zero, an outer bootstrap will be executed. In the bootstrap validation routine <code>boot_runs</code> bootstrap samples are generated and each one is again bootstrapped <code>boot_cut</code> times. This may lead to long run times, so activating the built-in parallelization may be advisable.</p>
<p>The advantages of bootstrapping the optimal cutpoint are that the procedure doesn’t possess parameters that have to be tuned, unlike the LOESS smoothing, that it doesn’t rely on assumptions, unlike the Normal method, and that it is applicable to any metric that can be used with <code>minimize_metric</code> or <code>maximize_metric</code>, unlike the Kernel method. Furthermore, like Random Forests cannot be overfit by increasing the number of trees, the bootstrapped cutpoints cannot be overfit by running an excessive amount of <code>boot_cut</code> repetitions.</p>
<div class="sourceCode" id="cb23"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb23-1"><a href="#cb23-1"></a><span class="kw">set.seed</span>(<span class="dv">100</span>)</span>
<span id="cb23-2"><a href="#cb23-2"></a><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, </span>
<span id="cb23-3"><a href="#cb23-3"></a> <span class="dt">method =</span> maximize_boot_metric,</span>
<span id="cb23-4"><a href="#cb23-4"></a> <span class="dt">boot_cut =</span> <span class="dv">200</span>, <span class="dt">summary_func =</span> mean,</span>
<span id="cb23-5"><a href="#cb23-5"></a> <span class="dt">metric =</span> accuracy, <span class="dt">silent =</span> <span class="ot">TRUE</span>)</span></code></pre></div>
<pre><code>## # A tibble: 2 x 18
## subgroup direction optimal_cutpoint method accuracy acc
## <chr> <chr> <dbl> <chr> <dbl> <dbl>
## 1 female >= 5.73246 maximize_boot_metric 0.956633 0.956633
## 2 male >= 8.41026 maximize_boot_metric 0.95 0.95
## sensitivity specificity AUC pos_class neg_class prevalence outcome
## <dbl> <dbl> <dbl> <fct> <fct> <dbl> <chr>
## 1 0.444444 0.994521 0.944647 yes no 0.0688776 suicide
## 2 0.222222 1 0.861747 yes no 0.0642857 suicide
## predictor grouping data roc_curve boot
## <chr> <chr> <list> <list> <lgl>
## 1 dsi gender <tibble [392 x 2]> <roc_cutpointr [11 x 9]> NA
## 2 dsi gender <tibble [140 x 2]> <roc_cutpointr [11 x 9]> NA</code></pre>
</div>
<div id="loess-smoothing-for-selecting-a-cutpoint" class="section level2">
<h2>LOESS smoothing for selecting a cutpoint</h2>
<p>When using <code>maximize_metric</code> and <code>minimize_metric</code> the optimal cutpoint is selected by searching the maximum or minimum of the metric function. For example, we may want to minimize the misclassification cost. Since false negatives (a suicide attempt was not anticipated) can be regarded as much more severe than false positives we can set the cost of a false negative <code>cost_fn</code> for example to ten times the cost of a false positive.</p>
<div class="sourceCode" id="cb25"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb25-1"><a href="#cb25-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">method =</span> minimize_metric,</span>
<span id="cb25-2"><a href="#cb25-2"></a> <span class="dt">metric =</span> misclassification_cost, <span class="dt">cost_fp =</span> <span class="dv">1</span>, <span class="dt">cost_fn =</span> <span class="dv">10</span>)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<div class="sourceCode" id="cb28"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb28-1"><a href="#cb28-1"></a><span class="kw">plot_metric</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABNVBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SOTU2OTW6OTY6ObquOyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rjk2ryKur5P+2ZgC2kDq2tpC2ttu225C229u22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+MAhEmAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAcqklEQVR4nO2dC3vcxnWGQcpeekU3TVzSlum4l9Rd1a5kl0xbq3aotNqkiUuJtdtYErurdXkR/v9PKGYALG4D4AzmsmeA73seaUksPsycg5dzwTWKIchA0a4rAIUtAAQZCQBBRgJAkJEAEGQkAAQZCQBBRgJAkJEGArT82bfZTz/85VH5i81fXWQ/3Z29+7J3O6SV4puTGblmRQU6S2pbjV4pSGooQNHeufxhM48qAF3u7xigogKdJbWtRq8UJDUUoL33U24u781bACLJH0AaqwEgDQ0FaP9rmeS7s19JgH76NIp+9l3yaxRFR3dns8to/49yN3w/j945z0zpTpM0fP9+FO39bbqv0v2VmIrtJL/+c7LC32XGxLJOtpMsX0cL+XsObbb57SbSCiTfr/NyxRaFMyvp9++LctPVtCsFNTUYoD/cF/tnc/gHAVDSkSXaO88BujeP3v1R7IJ1ujw1beaLOGXgMpJa1PdVvp10D0eSlljs3nvzdHnaFq3zLa63q9cA+nm2fryel1bJthotSgBpVApSaDBA/3kmdsBytkkAujsTjcUP81n6B53snqP87zjJ+/f5KCndKcv9i7uz/e/EjpnV9lWxnc38g5dyBambk+gvXt79LppJd/Jf1sVsN1/a3aICyfqzdP2kKskWL6O0lOS32cvkt1mpC9OoFKTQYIAuxF68OVkIgDbpOEjslBQgsXfELtjUBkjbNiT+n9//yzyq76tiO5v5vV/9ceu7OUnXSDYrmoptD7bdfAMgSdhSbEeWthSNoyhJVEzWoBgD0Ss1LFUj13CARD+yFrvoKO1J0mY+BShNvgBoUXal6y7yXqGxr0rbWYrPX2QDj4yYfFevq31irABolq2frbHOAaqspl0pqKnhAIm9upzFOgCJRcu0i/n5v/7Hn0669tUPycg1yvZyCaC87ZOyApBOpaCGhgMkBtLJ7km7sEX+RR2gSheW7NLfiH2brp8PN9JxjdjpNd7++5+yUXQKRNoBraPPtnOwYvPLSt9UdHn1LkwBkGaloKoMAFrv/XXyfzaI/jaZ7YodfFmaE2WD6M3J9rDKZv43aWeRjGWTyXG2S5eRnFjPSttZJ6Pm+O532V990jgkhchBdLLOn23bgmLz201k/Vw+6K4PoguAigaFXKlhqRq5DAASc52sFcia+SOxr8Q0Pgco+2KR25I9ko6v0/WzfSVXeufTWWk72RoZedm0PN2Fl1GxK7ebLzZxKafx9/L1a9P4AqBoewSdXClIIQOAkj/SRd6NbJIRyz1x3O/m02j2YwGQONK3PR4Yx+kUOju6962cO4uV/mse/eJHecxuux15IPGDrOVKDwz+eTqk3pQn1NvNbzchKvAneeAxHYJvqgcSc4DEanm7SK4U1FR4Z+PXRYPWJp1TH5CZggPop0/7T1QBIH8KDKBk3EUYjAAgfwoMoGSk+0H/WgDInwIDCOImAAQZCQBBRgJAkJEAEGQkAAQZCQBBRgJAkJGGAbS5P+DqKh2TWPcn3AgRgDy2QJoADYIU8i0ABBlpcBe2uf/NvLhUTF6Qvsg+j+L0y6ONXGNz/4m8NEwQIa7ayq4H2xx+HYlrb9IF2QYkNdutH223C3HVcIDmCRP5lcXpfp8vbk4W2V054ssoXWMzF5zMxErpxYDp9RjywrB8Qb6BAiDZAuWLrYQKuZABQIuiV9ocpiD938vyl9ka28/75+v0SvWF9IjF+YJ8A3WADnEhMnMZdGHnpWHNMkovC5UXEO+VvszXFJDcP78sXV4sF28XZBuoAVRsF2IqY4AuMyBuTiJxr9jeeZWuGkClqwlTgIoFcgN1gLLFhkFC7mSpBRJKGFkLHtb1Fkh2YYcCjnXp7jz5dXlBBlkNoG2XB7GUHYDkUCYDZDNvAFQaRIsbZlJo5Nf5gnwD4h5UcbeX/HGx3a69eCHLstQCrbN7f5Mxy95vioYkW/OJHMnk0/j82WbSmy/INyDm7Z99Ire+jGbbxRBXeTiQiBZkzAJAkJEAEGQkXM4BGQkAQUYCQJCRABBkJAAEGQkAQUYaBtBVU6plffLkYVw1Z+FYxqRdACg4DwDy7WFcNQBkkglWGQ/HA4B8exhXDQCZZIJVxsPxACDfHsZVA0AmmWCV8XA8AMi3h3HVAJBJJlhlPBwPAPLtYVw1AGSSCVYZD8cDgHx7GFcNAJlkglXGw/EAIN8exlUDQCaZYJXxcDzjBGi1WjlKhbmHMQwAKNNqNYQgThkPxwOA9FJh7mEMAwDKBID8eUYJEMZA/jzjBOhqAD+sMh6OBwDppcLcwxgGAFTUGF0Yp3AsY9IuABScBwDppcLcwxgGAFTUGABxCscyJu0CQMF5AJBeKsw9jGEAQEWNARCncCxj0i4AFJxnrAANOJTIKePheACQXirMPYxhAEClsACQH88YAFJqZW9TUDBCCxScZwwtkDIsAOTHA4D0UmHuYQwDACqFBYD8eACQXirMPYxhAEDlsLQJ4pTxcDwASC8V5h7GMACgclgAyIsHAOmlwtzDGAYAVA4LAHnxACC9VJh7GMMAgMphASAvHgCklwpzD2MYAFAlLF2COGU8HA8A0kuFuYcxDACoEhYA8uEBQHqpMPcwhgEAVcICQD48AEgvFeYexjAAoEpYAMiHBwDppcLcwxgGAFQNS5MgThkPxwOA9FJh7mEMAwCqhgWAPHgAkF4qzD2MYQBA1bAAkAcPANJLhbmHMQwAqBoWAPLgAUB6qTD3MIYBAFXDAkAePGMGSJMgThkPxwOA9FJh7mEMAwCqhQWA3HsAkF4qzD2MYQBAtbAAkHsPANJLhbmHMQwAqBYWAHLvAUB6qTD3MIYBANXD0iKIU8bD8QAgvVSYexjDAIDqYQEg5x4ApJcKcw9jGABQPSwA5NwDgPRSYe5hDAMAqocFgJx7AJBeKsw9jGEAQPWwAJBzz7gB0iKIU8bD8QAgvVSYexjDMGqA3hwff/Qijm8fHz94vf1I1REWAHLtCQWg689fxK8+jt8+PS19ZOoICwC59oQCkFAC0e1XL0of2fKOsACQa09IACVtzvUXr+PbL59lH8my9xJ1WPDq3WmpC6Drhx8+i988kORkH9k3HX8XaIFce0JqgYqmp2iBhDrCAkCuPSEBFD8/1R0D6RDEKePheEIBKOu03j59lM7CHtFmYQDItScUgOJXx8fJGEj3OBAAcu0JBqAOdYUFgBx7AJBeKsw9jGEAQM2wAJBjDwDSS4W5hzEMAKgZFgBy7Bk7QBoEccp4OB4ApJcKcw9jGACQIiwA5NYDgPRSYe5hDAMAUoQFgNx6AJBeKsw9jGEAQIqwAJBbDwDSS4W5hzEMAEgRFgBy6xk9QHSCOGU8HA8A0kuFuYcxDABIFRYAcuoBQHqpMPcwhgEAqcICQE49AEgvFeYexjAAIFVYAMipBwDppcLcwxgGAKQMi0oQp4yH4wFAeqkw9zCGod+zWtWzCYD0UmHuYQPDAM9q1SAIAOmlwtzDBYYhnqAAujlZiI/1/kWfqycTAMiOZ5VpQDGueGkIAHH1ZOCEMwa6jHLNel09mQBApp4mNprFOESmKkULRFBPWADIxNPstAYU4wAVtTCI5uXpgYdcjGVM2lUBKGmCbk6i/iFQH0BUggBQ9e+OQA+1GGfA1FUBaDmLL/cvLme9rr6wAJC2hwYPuRhXvDRUGwPdnc0szMIAEFWtUy3TYhwiU1UNoJuTIwDkz6M+0GOjGIfIVFUG6O7saL13LjqyPvWFBYBo0qeHWowzYOqqjIE282gWL9992evqCwsAkTSg+aEW4wyYupxM4wEQRQKdAfwAIM1UmHtYApSR4yocy5i0qwqQPJ1x1O/qDYtG0HQB2jY84wLoUsy/xESsT71hAaAulfqtUQFk7Ww8AOpQddgDgNRhAaAWDbq4Z4DFFS8NoQvz6WnOucYFEAbRTj2qKfvIAKKqNywA1JD6iA8AUocFgGpqO2A4LoDuzo5icT7egvDu3bJW401H/XqgmERQ/98FqQmaRgvUebpiVC2QxWk8AMrVc7ILALWEBYCkes+Vjgogi8eBAJAQ4VT7uACK17aOAwEgEj6jA4iq/rAmDxDxQp+RAnTzy+5hUH9Y0wbI8i0WAywuWFEKANn3WL/FYoDFBStKuQKIRNAoAdK7SBUAtYU1UYB0r3EGQG1hTRIgR7dYDLC4YEUpAGTPM+AOCwDUGtbkAFqtOIXjghWlAJAVj5y3cwrHBStKuTqQOCmAht/jNTKANnP5iDsbJ1MnBJDJPV7jAoh+LRklLAJBnDI+1GN2j9e4ALL2jESpsQO0Su9td17OMIsDVNSqtkALoosS1sgBUjxZg1M4DlBRqzIGIlxKlooS1hQA8lDOQAttP27un+sCU1e1C4vsDaIBkJ1yBlpoe98yQHRRwho5QEMfIW/uAUB6qTD3OALIUznDLC07Th6oWaTgJP9t7j+JIvFEuqTz2fvm8GJz+HXS/9ydybcVNFYiAGTt1uar0QNk5zZlzwClTMwXWzbm774UB2/EdfA3J/sXm/ksPZoj/hUrCaZmFIAsXlQPgCyVM9DSAtDhRQkkwcZCfsrZ06UAaJFNpZL/6iv1A2Tztp4rCkGcMq7pUQXHKZyWHbdMX6WzZeNQthiLS9FDJb/I5Wv5i+i6zovP1mOEAGiYJ1CA5Gin1LjYBchuFwaAbJQz1NKx8xIWKmzkXVjea4lnhTe7sMOWVsXdIHrUANl6VItngCQoCROikbg722sOogUz+SC6WIk8iCaLFBYAMi9nsKVlx4k7R/fO0+n8Z5+kM3SBhpjGP8kAirNpfGMlAGTNow6NUzgDduq65VhP99HGLUDpu8IsnsoAQOblDLdospO0Sq29FBEgLZHCAkDG5Qy3aO7Qy45X5dIBsjyN7yeIU8Z1PPYeV8cFoOECQAM8AKiQk9d+ZxorQG1xcQrHITJVubukFQCZlmNicYCKWg4H0WMFqDUsTuFYxqRdzm7ruQJAhuUYWdT77X/VsgXQ3dnR3dmC0pHRwgJARuUYWXYCkEBnedR6RBIASbVHxSmcnQF0aeW98ZkcPOMWADEGSDypPqHn0lYLNEqA7D51fmQAiXdlLOXZWgDU9gUA6gCILGJYAGh4OYYW9X4DQDsHqCskTuGo95uPLoz0iA5iWABocDmmliEA3Z21X41Ku6TV4uuepMYHUGdEnMLpA2i1WjUAaoWECpDts/EAaHA5xpYegOSd/TWAxGWt5/Kmjfju10+i6Ggtr48XpycWEqD0OzpA1w+Pj0/j+Pbx8YPX249pA2T93V/eAVo1VG2BlkfiUKC8MXU+E0tuPjmXN/xsv+sAqHpbz+2Xz+Lrv3/29ulp/OrjOPvQA6iPIE4ZJ3nCB6ijBcpxufnlxd2vz2PxT/way2euFt91AVR53dMbgcvz09uvXsTXn7/IPgCQl3LMLT0AtY2B5GXxe+cVgJbi9HrxXSdAdSWt0PUXr0sfybL3EnWaShrZu2aDD6d7FiYgSVuYEkBiXJN2YepnP3cC9Pbpo/jNA0lO9pF9Qf27GFkLFFA4AwHKTqaXABILs5uglSfau27ruX38KBlK11ug6QIU0pxgKEByKlZugcSlzu98uii+o7dA1w9PBUUYA+UaO0CDVG2BFg1+ZDcmZ2GPJj8LA0BaAL06Fjo1OQ40LoCCuj7OP0Cl23qsHYkOad7b7wFA3QDZv63nalQAhXWn9k4AooscFgCy72EKkP2ncwiNB6DAnhnqHyAtkcMCQPY9AEgvFeYeAJTLNUBpN0Z44wo5rNEAFNr783YD0HImruhoXvMBgAAQBaCkARJXEtm7InE8AJFe6M0pnF0BJK4mswlQONeAdnsAEAWgu7Mj8bRFeWk9AKoIAFEAEtdPz+Jl/53NkwOIxA+rcOwA1HWfhgogsuhhASDrHisAHRwcACCHlqqHxg+rcHoAOjgoE5R9q3EzTwtA9o8DASD7HiOADhoqAUS/macFIPvHgUYBEJEfVuG0AdTdApFv5lED5OA4EACy73E2BtK4macVIOvHgYJ5GkGHZ4wAqWZhGjfzqAFycRxoBABR+WEVjiFAfTfzqAFycRwIAFn3+ACo72aeFoDI0ggreIDI/LAKZwhAgwSA+jwAqFMun1QvBYBse/gCZPtJ9VKhA0Tnh1U4OwHI9pPqpQCQbQ9vgKw+qV4qkOfitng0+GEVzk4Asv6keikAZNljAJADOX1SvVTQAOnwwyocZ8DU5XwaD4BsewCQXirMPQDIoUq3Nrt4OsdV2ABp8cMqHLfUlIQWqMsDgHoFgDo8evywCscyJu1y+rIVKQBk2cMYINsvW0kVxjsiVR4A1C+nL1tJFSxAmvywCscVLw0BoHYPACKo/WUrXdIKK1SAdPlhFY4zYOpqf9lKl7TCAkB2PZwBokorrEAB0uaHVTiWMWkXAGrzACCSXN/afAWAbHsYA+Tg1mah1l3BKeN1jz4/rMJxxUtDrm9tFgoQoNUKANHk/NbmqxABkq8U9VDOyABycmvzFQCy7OELkJNbm68AkGUPY4DI0gsrTIB8lDNZgPQU3MuOg6vwDuX4bT1SobVAq2HFsArHLTUloQtraDWwGFbhWMakXT4AaiWIU8bD8TAGyMXTOYSCAmg1tBhW4bgjpqbqcaAZ0aUZVkgArQYXwyocB6io5filu6kCAiivKgAiqtoCLYguzbDCAWhbUwBEVGUMRDgLlkozrGAAKioKgIjCILqkUj0BEFG1k6lEl2ZYgQBUriYAIsrLILqNIE4Zr1USABHlZRAdBEDVOgIgoqpjoP73i6XSDSsAgGpVBEBEVbswNydTQwCoXkMARJSXc2H8AWpUEAARBYCC8wAgvVSYe/otzfoBIKJsAXRwcNARDnOAFNUDQERZAki+g7MjHjVBTDKuqhwAIgoAqesGgIgCQDutGgDK1c0PY4B2OzwDQEWNwwRox+d5AVAprBC7sF3fMAKASmEFCNDOH10EgEphhQfQ7p+BDoDKYXUQxBIgBm/jBEDlsLSbIE4ZD8czXoC0m6DdZrzz+RsAiKjpAtT9/BYARJTds/HtBLEDqOf5PwCIqEkCRHh+FAAiaooAUR5gB4CIsnxBWStBAMiaBwDppcLcA4AcyvYlrVoE7RAgB8UAILrawwoAINLzVwEQUQDIWjEAiK6OsNoIYgMQ7QHQAIioyQFEfIA4ACLK/n1hLQQxAYj6AHoARNTEACK/wAAAETUtgOgvwABARDm4tVlNEACy5ZkoQDu8f3hr0XgDDwAiysXDFchNkOeM67zBCQARNSGAtN4ABoCIcvJ4FyVBuwZI7w1yAIioyQCk+QZCAERUJ0DXn7+I49vHxw9ebz9S9YQFgFx6wgHozfFHL+K3T0/jVx/nH5n6wlIRtFuAdF+BCoCI6gDo+Yf/nrRAt1+9EC1R9pF91RcWO4C0X6ELgIjq7cKuv3gd3375LPtIFr6XqHezB4plO3yVLd6i60y9AL15IMnJPrIvev8uaE2Qrz9Zvq/wnmILJNQbFiuABvADgKjqBWjQGEhJEACy4wkMoLdPH6WzsEcaszBWAK0YwzABgIYcBxJqErQjgFacYRg5QK0ihMUFoBVrGABQa1hMAFr5KcanZyIAUQhyn/GVn2K8egCQXiqMPADItRy+radOUPOGYucZX/kpxq9nqgApHmngOuMrP8V49kwGoBpB/gHKS2MMAwDqCssFQIQHa2xXNSiGs2eqACkIGnBUmfJolmxVg2JYe6YDUO/5DAAEgDrD6jsnPxAgEkSlNRjDAIC6w+ohaMB5rZydPojMOGXtmTpAV6uho9tV7bxEB0SGPSVrz+QBGjq/Vh4VVPdnpmN11p7mMX/F2yItY9Iux++N7yZI66Bgh6cOkefjlZ49jdPWqvfVWsakXbsBSP8cQ2+3t2ofHTGGAQD1ZaKVIL3rLGhz8pY5GmMYzAE6OJgmQFpXeq00Zm6TAiglZ8xjoO43QBGzpzWlmgxAqnantJIvuQaokyBS9nRPf+zgqhG/HmHphEeu5Eu7BKjOhnodYjkdYgzDoHB66bkaE0BdBMW9t2w1EZs6QBR6rqYDUA9Bdh7IwBYGbY+kZ0pHotOo26O86uzGlF9NFqDtoBkA1Tyk81mEcnqKCdTTnKkDoLpHSUpbyzQxgBTHCScHUDtBuUfBSmvHNi2AVIeZAVDT02hu2gdGEwEob3kAkMxGW5TFj9Wz6R1zs7EDVO+zmtN2AKT0rJQ/6pVjz7IDT8tZ0YHFWMakXWwA2mLTc3h6fACRydEpxjIm7fICUBtBVU96GtTB+7jZAqRJjk4xljFpFyOAaPfsjAKgotFxFY5lTNrlB6AWgqYHUKO7AkC0sCYJUG0ypeyuABAtLBJApBvfwwGodCCnfaADgIhhKTPoac/uBKAD2uQKABHDYgUQYepjNiWXBZCmWACIGpYqlTsCyHrLUMamuli7bgQBIL1UmHuUAPXsXPqMqtgSpx7ZMibt8gaQiqCdZLzaxbSB1FpMS2szsGoAiB4WC4Cal2eVkLiqr6dYp6vdAkBkDQlr9wD1dVuN8S8JG6OqASCNsHbb5mucdNLDplaOe890ARp2ZYu5J9ahR2oIPgBIQ0PCUuwUTxkfwMIQfgAQXUPC2hFASaGMTqba8IwBoCEqRqSqN/I6K9NbWRPVTsZA+RDV+UU3upYhxXj3jKEFspIJRyPV5qEexjAAIJNMFGcfs99tnDxSzMAZwwCATDJReErjI7KnRbub7AEguuxkQn0Flp6n4QdA6Uq+xA6gvnZIXU7JtLPjlQCILjuZUF1nUfzUglGjnP52izEMAMgkE8QBcYeHNo9jDAMAMskE0aOeqGkcBGAMAwAyyYSW56AmR8UE4AFABp5BBx8ZwwCATDIBgACQUSaGXZrB9jodAESXnUywyng4HgDk28O4agDIJBOsMh6OBwD59jCuGgAyyQSrjIfjAUC+PYyrBoBMMsEq4+F4AJBvD+OqASCTTLDKeDgeAOTbw7hqAMgkE6wyHo4HAPn2MK4aADLJBKuMh+MBQL49jKs2VYAUes/WhniUM7Jw3AkA7bQYALTVyDI+snDcCQDttBgABE1cAAgyEgCCjASAICMBIMhIdgC6fXz84LWVLXXrzfHxRy/cF/P26fGHz9wXE18/9BKOU1kB6O3T0/jVxza21K3rz194Kef5afzGwx/E7eMkbV7+8BzKCkC3X72QO9eHPJQjwvGh6y9ex7df+mjqHMoKQD4z4aEFuv7i37x0YQAol2jv/WTi+qGHPXv98FTuXNeSXZiXwZZDBdcCeSjHWzjJIPoffguA/I6BxAjXsW7/0ePfg6fxljNZmoU98jI78tVVPvfUhSWh+EibU4V1HOjVsZfRbRKOl+Mzb479pM2lcCQaMhIAgowEgCAjASDISAAIMhIAgow0MYAuoyjaOy8t+Om7xjqb++ddX0MVTQqgu7N3XwqIFtslZVgU6vkamhhAS8FPQlD6IQSATDUlgG5OFvmPkozN/W/mUXS0uf8kigRTd2dRNJPfpd8sNuLr3dU3CE0JoFJ7kgF0Lj438/2Lu7NZnP8Ti+cJUJf7F2iBejUpgA4vtj9WAFrIn9f7yddrSU2+CAD1alIAtbRA4ueke1uLbixdVv4a6tSUAMrHQDefnAMgW5oSQPn0Kx/cVLqww4u1OECUdWEAiKpJAVQcB7o5OUp+2zsXjZJqEJ0BVJq3QWpNCqA4XuZHosUM/bNPzpMFMzmNn8XVaXwKkPh6p/Xlr4kBpBL6KRMBIABkJAAEgIwEgCAjASDISAAIMhIAgowEgCAjASDISAAIMtL/A1PJ7KfaXqAkAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<p>As this “optimal” cutpoint may depend on minor differences between the possible cutoffs, smoothing of the function of metric values by cutpoint value might be desirable, especially in small samples. The <code>minimize_loess_metric</code> and <code>maximize_loess_metric</code> functions can be used to smooth the function so that the optimal cutpoint is selected based on the smoothed metric values. Options to modify the smoothing, which is implemented using <code>loess.as</code> from the <strong>fANCOVA</strong> package, include:</p>
<ul>
<li><code>criterion</code>: the criterion for automatic smoothing parameter selection: “aicc” denotes bias-corrected AIC criterion, “gcv” denotes generalized cross-validation.</li>
<li><code>degree</code>: the degree of the local polynomials to be used. It can be 0, 1 or 2.</li>
<li><code>family</code>: if “gaussian” fitting is by least-squares, and if “symmetric” a re-descending M estimator is used with Tukey’s biweight function.</li>
<li><code>user.span</code>: the user-defined parameter which controls the degree of smoothing.</li>
</ul>
<p>Using parameters for the LOESS smoothing of <code>criterion = "aicc"</code>, <code>degree = 2</code>, <code>family = "symmetric"</code>, and <code>user.span = 0.7</code> we get the following smoothed versions of the above metrics:</p>
<div class="sourceCode" id="cb29"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb29-1"><a href="#cb29-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, </span>
<span id="cb29-2"><a href="#cb29-2"></a> <span class="dt">method =</span> minimize_loess_metric,</span>
<span id="cb29-3"><a href="#cb29-3"></a> <span class="dt">criterion =</span> <span class="st">"aicc"</span>, <span class="dt">family =</span> <span class="st">"symmetric"</span>, </span>
<span id="cb29-4"><a href="#cb29-4"></a> <span class="dt">degree =</span> <span class="dv">2</span>, <span class="dt">user.span =</span> <span class="fl">0.7</span>,</span>
<span id="cb29-5"><a href="#cb29-5"></a> <span class="dt">metric =</span> misclassification_cost, <span class="dt">cost_fp =</span> <span class="dv">1</span>, <span class="dt">cost_fn =</span> <span class="dv">10</span>)</span></code></pre></div>
<div class="sourceCode" id="cb30"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb30-1"><a href="#cb30-1"></a><span class="kw">plot_metric</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABOFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SOTU2OTW6OTY6ObquOyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rjk2ryKur5P+2ZgC2Zma2kDq2tpC2ttu225C229u22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+iRaHgAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAgAElEQVR4nO2dC5vctnWGuXI045HSNkl3E3kd95K449qV7WrTNq6TVVtvkybpWtukjSVtORp3L+L//wclAF5AEiAPrnNAnu95pJkleQieg3dwIwhmBYnkoOzQF0BKWwQQyUkEEMlJBBDJSQQQyUkEEMlJBBDJSQQQyUmWAF386S+qb3/4i2N5x/4vL6tv92cPX06eB3RQcXu6Al9ZewGjKekOg18UicsWoOzonH/Zb7IOQFcPDgxQewGjKekOg18UicsWoKPvCm6u3tloAAIpHkAGhxFABrIF6MHPeZDvz37CAfr2gyz709+Vf2ZZdnx/trrKHvyWZ8PvN9l3zisjkWmcht9/N8uOfirySuRXadSep/zzH8sD/qYyLE125XnK7btsy/+uoa1O35xCXEC5f1eny87ILKuUfv1dlq44zPiiSENZA/SbRyx/9o9/wwAqK7JSR+c1QO9ssoffsCzYie3CaL9huc8YuMq4tv28qs8jcjjjtBQse9/ZiO2iLNrVZ9w1h/cA+l51fLHbSIdUZ822EkAGF0VSyBqg/zxjGXCx2pcA3Z+xwuIPm5X4QZfZc1z/jsu4/75uJYlMuXhweX/24HcsY1a9vGrPs9/84CU/gOv2NPvzl/e/ylbcuvyvqmKa00vZzS6gPH4lji8vpTzjVSZSKf9avSz/WklVmMFFkRSyBuiS5eLt6ZYBtBftIJYpAiCWOywL9r0GUlOGFP/z63/aZP28as+z37zzk982dren4ojytKyoaGqw5vQDgDhhF+w8PLULVjiylNiF8Sto20Dwi7IL1cxlDxCrR3Ysi45FTSKKeQGQCD4DaCtbiWO3da0wyCvpPBfs8/tVw6Mips7qXbdOLBQArarjqyN2NUCdw4wvijSUPUAsVy9WhQlAbNOFqGK+98//8cfTsbz6Q9lyzapclgCqyz4uLwCZXBRpIHuAWEO6zB5RhTWc9AHqVGFllv7ydFVnfN3cEO0aluk93v77H6pWtABCVEC77MOmD9ae/qJTN7VVXr8KUwBkeFGkrhwA2h39Vfl/1Yj+RdnbZRl8JfWJqkb0/rQZVtlv/lpUFmVbtuwcV1l6kfGO9Uo6z65sNRf3v6p+9WXhUCbCG9HlMX/SlAXt6ZtTVPVc3ejuN6JbgNoCBXxR9lGesRwAYn2dqhSoivljllesG18DVO1ofsJljoj2tTi+yit+0Hc+WEnnqY6oyKu65SILr7I2K5vTt6e44t34d+rje934FqCsGUEHXxRJIQeAyh/ptq5G9mWL5R027nf7Qbb6pgWIjfQ144FFIbrQ1ejeL3jfmR30X5vs+9/wMbvmPHwg8QdVySUGBv9MNKn3coe6OX1zCnYBf+QDj6IJvu8OJNYAscNeml4Uaaj07sbvssk2icmtD5KbkgPo2w+mb1QRQPGUGEBluwvQGCGA4ikxgMqW7g+mjyKA4ikxgEjYRACRnEQAkZxEAJGcRACRnEQAkZxEAJGcRACRnGQH0P6RxewqEyN27Le+HoQoT6RMe6ecoWHlm/psY6mW+2xTQqWIJZAhQN7Cy06EEiCrZLCJAFIdbiMCyEA8f7/ctFPF+IT0emL6cSF2Hu/5EftHX/CpYSxkbNZWFej9459nbO6N2FCdgIe1Oftxc9423W7ijZmUHk9ETF5kn3u+h11DvU9MVs2OvuBXcs+fT+JPglTXfl4nU/SurnX1uLutPltv30iqVTKNI1Jg0pI9QJuSiXpmsYj4Znt7uq3ygu3MxBH7DeNkxQ4SkwHFfAw+MazeUJ+gJeS8KTikScnN7irxxkxOTzzM1f7jJ9q0+1h6bJr+7Wk1xbHcVVLUXHsLUP/q2NHSYdI2cbb+vrFUKyd7R7hk5WHkANC2LYf3jwVI//dS3lkd0Xw+Ot+JnyGPOt9cb6hP0Afo8WU/3e4pazM5PX7O8r/6UwAk7SvT459XbWVS/muuvQWof3UdF6Vt9dn6+yZSlRyRA5OWHKqwc6kivxCVhphAfCTtrI9k2fDo/EqaXsw3NxuqE/QAas/bpNtNvGcmMoP9kKXPtihrL4D/2Kt8ZnUY/7u59v7R3cvYNU+u1dvas3X3jaXaRVUOTFpyBuiq8rus3lkZfnTepasHkFRIC4DaDfwEfYCqzR2bLr0ds2mAqvTkrCx2D785Y3VLc+39o+XLqA+Tt9Vn6+8bS1WbUmLyVAIxsTKa/5D7JRAvph+zcO6kp/P4bnlDBVkPoE7JrgCoYybA4YsqiEdnmypM3ldUlU7db7p9/8syU6Vr7x8tX0Z9WGdbXWn29o2l2qssk31s0Q9APChVHPabAUBSI5o9MCOCxXfXG+oTsGYme9qLf902563TlXa3TYd+iddvRLMTtftEemzOYt2cZRXlSmR8de11Mv2r465Wh8nb6rP1942l2gVIDkxa8lQCVXU/q/yPftkrEXhndlU0/eJ6bTNuW2+oT8A6wB++fy7aN6t6s+hqF53d/ATV/k56vW4856Oz7+i806GuH/SQrr1KZnB1RXtYZ1t9tt6+sVS7AMmBSUsRBhI79ZztOX7q4UJIAZQIQLsE+yfLUCIA/XuKg7SLEE3nIDmJACI5iQAiOYkAIjmJACI5iQAiOckOoOuhVNumFMkG8aUFc8czJnoRQMnZEECxbRBfGgHkEglUEU/HhgCKbYP40gggl0iging6NgRQbBvEl0YAuUQCVcTTsSGAYtsgvjQCyCUSqCKejg0BFNsG8aURQC6RQBXxdGwIoNg2iC+NAHKJBKqIp2NDAMW2QXxpBJBLJFBFPB0bAii2DeJLI4BcIoEq4unYzBOgPM8DhcLdBjEMBFClPLchCFPE07EhgMxC4W6DGAYCqBIBFM9mlgBRGyiezTwBui4IIEzueMZELwIoORsCyCwU7jaIYSCA2ismgDC54xkTvTwCRL2wODYEkFko3G0Qw0AAtVdMAGFyxzMmenm8mUoAxbEhgMxC4W6DGAYCSHbLmCBMEU/HhgAyC4W7DWIYlgqQUnnu71ykVOSzDUQlUBSbOZRASrcIoDg28wXIuBGEKeLp2MwWIPNWNKaIp2NDAJmFwt0GMQwEUMctAiiGzYwBMiUIU8TTsZkvQMataEwRT8eGADILhbsNYhgIINktAiiKDQFkFgp3G8QwEEA9t8wIwhTxdGwIILNQuNsghoEA6rlFAIW3IYDMQuFugxgGAqjjlmkrGlPE07GZNUBmBGGKeDo2BJBZKNxtEMNAAHXcIoBi2MwYoGsCKILNvAEyIghTxNOxIYDMQuFugxgGAmjgFgEU2mbOABm2ojFFPB0bAsgsFO42iGEggLpu5ddGBGGKeDo2BJBZKNxtEMNAAA3dIoAC2xBAZqFwt0EMAwE0dIsACmwzd4AMCMIU8XRsZg2QWSsaU8TTsSGAzELhboMYBgKo5xYBFN5m9gDBCcIU8XRsZg0QFwEU1IYAMguFuw1iGAgglVsEUFAbAsgsFO42iGEggPpuGbWiMUU8HRsCyCwU7jaIYSCA+m4RQMFtCCCzULjbIIaBAFK6RQCFtFkAQFCCMEU8HRsCyCwU7jaIYSCA1G4RQAFtZg6QSSsaU8TTsVkCQECCMEU8HZtkAHpzcvKjF0Vx9+nJk9fNh9CIWwRQaJtUALr5+EXx6r3i7fNn0kelEbcIoNA2qQDEVEJ09/kL6aPaPukWARTOJiWAyjLn5pPXxd1nX1Uf5bZ3S02elt6/uxSNAXTz0Q+/Kt484eRUH9We6d8FqAjC9JNNxyalEqgtetoSiGnaLQIomE1KABVfPzNuAxm0ojFFPB2bVACqKq23z5+KXthTWC+MAAptkwpAxauTk7INZDoORACFtkkGoBGNuZV3PtxD4W6DGAYCSOsWARTKhgAyC4W7DWIYCCCtWwRQKJuFAAQhCFPE07HBC9Dt6ZZ97B5cTlmNugVvRWOKeDo2BJBZKNxtEMMwJ4CuslqrSatRtwigsDZYAWpKIIBG3SKAwtrgBQgukFvTBGGKeDo2iAEqi6Db02y6CUQAHdIGMUAXq+LqweWVYxuoFgEUxgYvQGUBdH+2cu6F1SKAwtigBuj29NhXN54ACmSDF6D7s+Pd0TmryKY06lY++OIUCncbxDDMC6Biv8lWxcXDl5NWo24RQGFtEAME1rhb4DoMU8TTsZk/QLUIoCA2mAHitzOOp62Abk0RhCni6dggBuiK9b9YR2xKQLcIoBA2eAHydDe+FQEUwmb+AIG7YZgino4NXoB8VWEEUFAbxAB5akTnim/2oXC3QQzD3ACCasItaBGEKeJYbfK8H8QFANR6P+ElJBTuNmhgsLDJ8wFBiAG6Pzsu2P34SUEjQQA52uRpAcRvo0IIgkaCALK2ySVZJBOKl4GCjgNNEUQAqYfPZGpSagN5AwjaDSOA+jaKAscumWDA9BXkVgYBBFWnpGF/TOKDGqBi5+dmKgEE1KCd421FgUC4DBX6XRmjESGAJHT8vmnWMyZ6qQG6/fF4MwjuFgE0pgYgOD4EkGko3G3QAqTqpHtKJgQrShFAh7PJVZ10T8mEYEUpAuhgNrllOksAKFd+tQuFuw1OgHLbdAggs1C422AEqK66CCClWwTQhJqoEEBqt2ADQYsFqA3KTAGakoFbBFBPeScm8wJov+FL3Hm8G389StACAcq7/MwLINBcMi4TtwggSXl/0HlWAHlaI7EnAqgjm9lhFiYhWFGqWwJ5AwjWDVsUQOq3GM0KIMhUMqFJt3LNd4tQuNtgACjXvAVrVgDdnmbARvSkcs33pUrEIJ9hKEJ140F12CJKoPEhsVmVQHAZubVogPKJIdWZAeR5fSChJQM0WRDPCyDf6wNV0hI0f4Aa6eb9zAqgAM+FcS0ToFz7h2M6ywAI1I+fMUA5jJ95AeSzCls4QF2X/Y6lIgbIYyN64QB15HlCAmaAoAK4BSFolgD1nR2fNk8AQdxaEEADXPwvUIIUIPGusADzga6XBNDA0wDLRCIFyEiGbi0HoL5CLNaPF6BQ40DaOM4JoLLuGtj4WmrDwiQYMH0FAwgyDjIfgDgqoyHwkw7YJBgwfQV47bfQsgBS2YAeWp4RQJ6ntC4GIN0jOrCH3mcFEFwQtwADQTMASG7m6G9luKdjauIZE73CP9bD5W0uAzKAdE9Y+F0tysIkHDE99d6Zen+2hVRkxm7NFSCNDXzNllkBxNC5OC520y9NNXZrjgDpJ8j7Xm7MwgSW+/tH55bcNOoDdOXvvfGyZgjQ0KVCu8clHTsTWO57BoitVF/Sc+WnBAJ0w2YJkNmaY/MCiL0r4yI7mj4pxC3AlJiUAVI4VGi2O6VjaaIjhnWTtgKc8r/9oy+yjJUXt6fZ0ZePL/ePf152oe7P+Fjg4KBpgMCCuDVzgDQ2pksexgVIMLHZNmxsHr5kKyKwSYS3pw8u95uVWCKB/WsPYkzpRpcD3kydrsPmB1CYJVctTDQAPb6UQGJsbPknb/ZeMYC21a2s8r/+QQCAvL/uqdUiAAq15KqFiSbjLsSNqoaNx3wO85Y3e8s/+HbeC+dV13n7qR3cCfu6J0meHsvEDJA5PgdoRJetHalw8QtQuOkc17MDSOFOjsmdkcwrs7nDRl2F1bXWjvWiBlXYYw0UBJCVjYofVO6o841nbckEazTfnx0NG9GMmboR3R4EbkT7fTJ1uhuGKeJmNsOpqzkudzQZx17HdHQuuvMfvi966AwN1o3/ogKoqLrxg4OmAfL1uiehGQOknvqMyZ3pPBzCpRnrGR9tjNWN14+8GQoFQH1ZrzqPBCDW7NHWUgcDaLoIwhRxFxv7VeeRAMSno+pqKSBAIR/r4ZovQLVnmNwxBMheUR7r4ZorQO3oISZ3PGOiV7cECrHMb635AKS7R4PJnUC4DBUPoJEJNCZCBpDjqvMzAkh6rMdXG2iyG4Yp4nAb7dMCmNwJCo2skCvVzxug4ftOMbmjzrf/VcsXQHDB3JopQEKK9+VickedbyEBCtGN7w0EDWY7YIq4qQ0B1APISDZuIY+4qQ1yd9T5RgDhAQh5garOt9AAiWoM8KoMG7dmAlDe+wyVjqPJYQC6WLEZHVdBZiRi/8kCbUb4QeXOQQAqCyB2S9bfhLKp9QIxRRxoQwCNA8RmkxFAepsxflC5YwPQ/Zl+LhhoSuv92TGbGHLhrQqbWnISU8SNbPA/aDsFUNmeGACkhQQKEJvDuCoupp9sJoAipWNvMgEQ79H0AGLTWs9FJ+r+Z19k2fGOz05lE1u3HCBlB2ukG3/z0cnJs6K4+/TkyevmwwiggWYCUALrZekAygfqlkAXx2x9Df5g6mbFtty+f84f+Gn2gQG6++yr4uZvv3r7/Fnx6r2i+nAEqBd6TBE3sElh1WIdQCMlUI3L7Y8v7392XrB/7M9S5ZZ23xhAnXGgNwyXr5/dff6iuPn4RfWxcIBy6f+Q6bibTACkawPxu1lH5x2ALtjNrXbfGECDcaCyFLr55LX0UW57t5T68gBK+62zufR/khrvhTFIRAkjAcSmaIgqbFj6MI2OA719/rR484STU31UO6C/i+FvFe0EGohN3vwXNh0PJpYAVSvUSQCxjdVD0MrV68bGge4+fVo2pfsl0HIBGq/AcLljCxDvisklEJto+J0Ptu2+EYB640A3Hz1jFLm0geYFENfICgqY3LEByEr6cSDBD6/GeC/sqUUvTPkIuWEoIOl4N5m8lRE6HVeTwwDU0asTpmdex4GuCSB3m1QAGpNDJNIGaHQJIEzuqPMtQhUW7snUWkgfpJq0ya+nlpDC5M5BAAq4Ur0kAsjRBi9AAVaqH29FY4r4lM00P6jcORhAnleqV4W83YYp4lM2+fQaiJjcOQhAfleq55oPQNcE0DRAXleq55oTQJOLsGJy5zAAgeUWCZTroUzaTC/ii8kddb4RQIdtRMdIx5OJOt9CAlQ/1xx8HChJgHICaAogIzlGAuOacKM2IH5QuaPOtwQBUgY+PYBAbzHA5I4630wBGntOQwVQgJetzAIgWAGEyp0pgNbrdQCAArxsRR15hOvijtrAXqOCyZ0JgNZrmaBqr8HDPGqAQrwrYxYAwfhB5Y4OoPVAEkDwh3niAaQWAeRgE7AEAj/MowbI88tWxoTv3QAjNkB+ULkzAZCyDWTwMI8GIL8vWxlTSgCB30OIyZ0pgFS9MIOHeXQAQeUciaQAipSOVxNHgKYe5okIkCb8OTgUwHT8mkg28BehYnLHEaCph3k0AIVY4i51gAxepIvJHRuArBR8ibvkAYqUjmeTgwDkfYk7Jl0G5NBQANPxa9LYGPCDyp1DAeR3ibsxpQGQCT+o3DkIQN6XuBsTAWRrgxcg70vcjSrHFXGlDfAuqnM6/k0OAxBYPiKRAECAJzG8pBPAxIUJI4UHaKQVjSniKhuzAgiVO54x0Sv8S3e1eZCjirjCxrAAQuVOWGokHa4EQg+QETwO6QQx8YyJXgSQ1oYAgij+6hytckwRH9gY80MAgaZDc/mJBGKAhm8WCpPOvAAK/NrvgfACVL3bzBAiTO6EYEWpbgkUFyCLWiIqQKaXt3iAIHfBhIzc0ucDATQzgMI0ohME6NqGHwKIPVcIk5FbIytzx2mpWpRAsWCYF0CBGtEEkF8bvABFb0RjBSjebbp5AQR5FFrITyRijdaZmkScqjQvgMLcTB0NBWKAaCQapMPNB7rGCpCY708j0TCpAZp6HNHMLW1eRLpjaWZSPfNIAMFEAPU19VYwX+lY2xBAsg06gOIuHEIAQdwaAyjG/QICKKBiADQaCmQARV59b6kAeRSudyAn/17m6Dp4CRRh0o2BSV7/RyUQUASQrIYfAgiqKAOJOkawAST14AkgoIKvD8Q0CpAhQeEBsk2GAAqyPhBTIgDJF0IAARV8fSAmAsinDWqA4q0PJNmEfn4YaNK5DAIIqIOtDyTZ4AAo71wJAQTU4dYHusYHEDWizXXg+UBCgRfhAZnk1AayUhyANIQgA8gxGQIo2DhQAgD1L4EAAirKONAUQIEXYwaYDB4jJICAijIOlABAVAJZCsM40OEBGi7mQgABhWEc6NqEoBARVzwHTwABhWEc6PrwAPlIhgCCa3jB6/XaJRQHBUiVOAEElCeA+Ds4R/xRA1JMHTBu48uEAHJQF6CrLNte2VRhKQOkTJoAAqo7DvTwj6InP6XB9SYM0OSleUrHmw1egHg3fmvXjZ9oA03nEpQg3xHXLMdKAAHlC6DrwqIVjQAg3XK+BBBQ3ffGsyrM/r3xxgQhAGhykNxXOv5sEAPk+N54J4CgBPkugfwlQwDBpXHLlKBDAzSyEisBBJTf+UBaggAt1fgAja1FTwABNRgHcqrCXAACEuQv4qPvMiCAgOo2oln/y6kRrSMIJ0AjKRJAQCnWibafzmFaBB0WoNEUCSCgPANk1o7unQdEkN82kM9kCCDXKozLhCACaGYAOTaiudIBaDQ5Aggo/4/1GBDUPw+EIC8R7z6F6ikZAgiuUbeUBIHmTMQCqF1HymcyCweofs+B46sOdO1oGECAN5V6iDiAHwIIKt8l0PpaTRAEoOpNpRORmTpg2mS6AiOAwAoBEJigAwGkuxzHZAgguEbc0hVBgFBEAyhMMgQQXFNuAQlStIGiVGGAdAggoAKtzgEjKFLdIpvw9g8B5E/oAApgI5mIDhgB5E+h1gcaEAR++Mp//7o1qTrw1Abyp2ADiX2C4E/veR/h6wIU7KY/AQTXqFuuAHm/SdVpA4Wb+EgAwTXuFpggXSg83ybvt4EMbWzSCWmzVIBMQuF3pldjAi1+7JIhgOACuTVNkD4UXicrF9UpAy/FSADBBXNrkqCRUPh8XKIQJwy9HDUBNNDNxy+K4u7Tkyevmw8hmFsuAPm0YSaQG/2OyRBAfb05+dGL4u3zZ8Wr9+qPSkC3OgQZLgPm8ZHRYuR0YzY26USwSQagr3/4b2UJdPf5C1YSVR/Vrim31r1PJtN15Pw9tF6YtJ7tkyGABmLE3Hzyurj77Kvqo9z4bqmps657n0zGb7L19+pbeoluOE0C9OYJJ6f6qHZM/i5gRdDEb8nX0mHGxY9dMlQCDaQsgZgm3VoPvtiEwsvihXmOGIYFAGTVBmrlBJCP9Xdz1DAsAKC3z5+KXthT814Y0xhB06Ewrvb69mwFBcQwLAAgh3EgoRGCgkdc3H1HDMPMAdLKxK0GIKvSxOkdBNXsDcQwEEBat9aDb3bVkf1rdJrBH8QwEEBat9aDr5btGdsXebV2iGEggPRuAYogWPS6ZsCIy2PPiGEggPRuDYsgu1D0CLKADjEMBBDMLQ1B0OjJOEyPHeXNzI3cLBlZiG0IILNQMBnUR9IzO9SIjqBI741XExQi4i1ADXWIYSCAgG6pO2IGXXKgTZ4rnhpEDAMBNOrWVFfeIHr54IvimDxv/rdMJgWbZQLUEGTUIJZUVUvixpZqgph+0hhiGAigUbcUAJmHolb3vkQPl9Epz4hhIIDG3ZogyCx6onfescnrxRKGx1ong95mSQB1NHypoWH0coWNeqEN66ZWCjZLBUh+rarlCJ+iBKqbRoPjTC5NIcQ2wzF/xcsiPWOi12EAqrLYoBuvGNTplD0LbgMpX3jsGRO9DgSQyHDjkWhpco+SGDVGiGFwBmg9d4DkoaCBn+b3wnLdWE97APjSRoXYRjJZNxocFEsRARr+aR495UjzlBDD4AJQw82820BjBFkCZGqEGAZLgJSFTvegWIoNUOfvwpgFAggADz9vLEUcB6oDINtMwTCgxYIfxDAY2gh2ljoO1IZBthnHwU+LGCUMUBupobO+5t8XD9B1UwJP2Xh5tBkRDBY2nW7WGp6MZ0z0OgRATSGEaewfqU0Fj9zqWRxAqhafwW8JnI5PEwQ2a1nGyXjGRK8DASQ2NqPKg91eJ/ccHgYzG4kZVZdrcQDpCWpsBn2tkciMRM2fyYFsGnSqNo8ycgRQs7W16ZY44RYax2uzbrtZGnCMkvGMiV6HaUSLkFEbqB1WBoBjlIxnTPQ6IEDX66UDVKMDGls2S8YzJnodEiCTV8y7pIMQIP1tdE/JeMZEr4MCVPTClwPW400eoC44odzxjIleB34ydUDQdGSmD/FgEnBU+brT1iGAYG7pADKvxRICqC1n1nIfyzUdAqhjY0pQOgDpRpJd01kkQGpOCu2ekciYHW5p4moDbiMTQA6RqGzMCMINkFzmEEB6+YlEbWNEEEaANFUVaOag2WWBTTxjohcGgIAjsNbpDEwARQMgmQE2mMZFPWOiFwqATAohnxEfAUljM9owJoDAsnFLkVXF6F5NZKAHqpNRpjPEQVEbTbeKCSCwbNwaBwhMkEvEpyqu/jQcwxsOBBBYVm5NlPlAgiwjbtJQt71PRQCBZeXWVKMh2M+8ALbSDXvginTMRQA5REI1dGtqA0qmf9ZuOqoCJ9xtcncbAkhnE2Lkba0w6bVwfCQT02YOAFmLZdjoTv1eu+S06XhOabE6UAkkfveqEmiqDDJJZ903GRvFsU8mss0cSiAPkWBZ2G/drqvZMu6DLZ1TFNedR2VALRwCCKiDt4GUeWk+RNwz753LvElMAAF1cICEFHmsyfXxdPisLamgWYuCzuXS0NkQQHobwO3ssXS6o8driAn40vDYEEBjNv0HpAYMqdMZlDoBLg2LDQEEsZEgWA9axKqD61KHABIHxRJWgIRqGCQoit4Bg87/sN5DDAMB5BIJuE1VxFw3eNTfQf0rxDAQQC6RMLGRiBkWOv6SScGGAHKwsRrUQQwDAeQSCVuAwieD2oYAcrGxmWeBGAYCyCUSqCKejg0BFNsG8aURQC6RQBXxdGwIoNg2iC+NAHKJBKqIp2NDAMW2QXxpBJBLJFBFPB0bAii2DeJLI4BcIoEq4unYEECxbRBfGgHkEglUEU/HhgCKbYP40gggl0iging6NgRQbBvEl7ZUgBR619eJcKQzM3fCiQA6aDIEUKOZRXxm7oQTAXTQZAgg0u4trHoAAAIrSURBVMJFAJGcRACRnEQAkZxEAJGc5Aegu09Pnrz2cqZxvTk5+dGL8Mm8fX7yw6/CJ1PcfBTFnaDyAtDb58+KV+/5ONO4bj5+ESWdr58VbyL8IO4+LcMW5YcXUF4Auvv8Bc/cGIqQDnMnhm4+eV3cfRajqAsoLwDFjESEEujmk3+NUoURQLVYeR8nEjcfRcjZm4+e8cwNLV6FRWlsBVRyJVCEdKK5Uzai/+5fCKC4bSDWwg2su7+P+HuI1N4KJk+9sKdRekexqsqvI1VhpSsxwhZUaY0DvTqJ0rot3YkyPvPmJE7YQopGoklOIoBITiKASE4igEhOIoBITiKASE5aGEBXWZYdnUsbvv3d4Jj9o/Ox3aSOFgXQ/dnDlwyibbNFhkWhid2khQF0wfgpCRIfTASQq5YE0O1pU/RwMvaPvtxk2fH+0RdZxpi6P8uyFd8n9mz3bPchrzgBLQkgqTypADpnn/vNg8v7s1VR/2ObNyVQVw8uqQSa1KIAenzZfO0AtOXfdw/K3TtOTb2JAJrUogDSlEDse1m97Vg1JrbJu0mjWhJAdRvo9v1zAsiXlgRQ3f2qGzedKuzx5Y4NEFVVGAEE1aIAaseBbk+Py7+OzlmhpGpEVwBJ/TaSWosCqCgu6pFo1kP/8P3zcsOKd+PL7nunGy8AYrsPfcnItTCAVKJ6ykUEEAHkJAKIAHISAURyEgFEchIBRHISAURyEgFEchIBRHISAURy0v8D8fufWoQ5LvEAAAAASUVORK5CYII=" style="display: block; margin: auto;" /></p>
<p>The optimal cutpoint for the female subgroup changes to 3. Note, though, that there are no reliable rules for selecting the “best” smoothing parameters. Notably, the LOESS smoothing is sensitive to the number of unique cutpoints. A large number of unique cutpoints generally leads to a more volatile curve of metric values by cutpoint value, even after smoothing. Thus, the curve tends to be undersmoothed in that scenario. The unsmoothed metric values are returned in <code>opt_cut$roc_curve</code> in the column <code>m_unsmoothed</code>.</p>
</div>
<div id="smoothing-via-generalized-additive-models-for-selecting-a-cutpoint" class="section level2">
<h2>Smoothing via Generalized Additive Models for selecting a cutpoint</h2>
<p>In a similar fashion, the function of metric values per cutpoint can be smoothed using Generalized Additive Models with smooth terms. Internally, <code>mgcv::gam</code> carries out the smoothing which can be customized via the arguments <code>formula</code> and <code>optimizer</code>, see <code>help("gam", package = "mgcv")</code>. Most importantly, the GAM can be specified by altering the default formula, for example the smoothing function could be configured to apply cubic regression splines (<code>"cr"</code>) as the smooth term. As the <code>suicide</code> data has only very few unique cutpoints, it is not very suitable for showcasing the GAM smoothing, so we will use two classes of the <code>iris</code> data here. In this case, the purely empirical method and the GAM smoothing lead to identical cutpoints, but in practice the GAM smoothing tends to be more robust, especially with larger data. An attractive feature of the GAM smoothing is that the default values tend to work quite well and usually require no tuning, eliminating researcher degrees of freedom.</p>
<div class="sourceCode" id="cb31"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb31-1"><a href="#cb31-1"></a><span class="kw">library</span>(ggplot2)</span>
<span id="cb31-2"><a href="#cb31-2"></a>exdat <-<span class="st"> </span>iris</span>
<span id="cb31-3"><a href="#cb31-3"></a>exdat <-<span class="st"> </span>exdat[exdat<span class="op">$</span>Species <span class="op">!=</span><span class="st"> "setosa"</span>, ]</span>
<span id="cb31-4"><a href="#cb31-4"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(exdat, Petal.Length, Species,</span>
<span id="cb31-5"><a href="#cb31-5"></a> <span class="dt">method =</span> minimize_gam_metric,</span>
<span id="cb31-6"><a href="#cb31-6"></a> <span class="dt">formula =</span> m <span class="op">~</span><span class="st"> </span><span class="kw">s</span>(x.sorted, <span class="dt">bs =</span> <span class="st">"cr"</span>),</span>
<span id="cb31-7"><a href="#cb31-7"></a> <span class="dt">metric =</span> abs_d_sens_spec)</span></code></pre></div>
<pre><code>## Assuming the positive class is virginica</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<div class="sourceCode" id="cb34"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb34-1"><a href="#cb34-1"></a><span class="kw">plot_metric</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABNVBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubo5ubqtuq+SOTU2OTW6OTY6Obk2ObquOyP+QOgCQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rbo6rjk2ryKur5OSr5P+2ZgC2Zma2kDq2tpC2ttu225C229u22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///80DS6pAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAemklEQVR4nO2dC3vcxnmFQSler+Smicu1ZbruJXFpu6Lklmwb1w5lpyWbNHElsnZrS9rVurzh//+EYmZw3wEw9xvOeR5pSQAH37eYl4OZwQDIcgjSUOY7AShuASBISwAI0hIAgrQEgCAtASBISwAI0hIAgrSkDtD5z78qf/r+r/bbK7Z//bz86fbkrZeT+xHaKL8+WAhn1iQwGmloM/GkIC2Asr1T+sN2mXUAurznGaAmgdFIQ5uJJwVpAbT3DuPm8v5yACAhuQNIYjMAJCoNgO79hh7k25NfUYB++ijLfv5t8WuWZfu3J4vL7N6faDF8t8x+dlqaWKFRGr57J8v2fs3KipVXYWr2U/z6z8UGf1caC8um2E+xfJMd0t8raMvd17tgCRTrN1VcskfiLCP94R0Sl20mnRTUkw5Af3xAymf78I8EoOJEVmjvtALo/jJ760dSBBu2nJm2S1L6hIHLjOqwX1bVflgJZ5SWnBTv/SVbzuqiTbXHTb15D6BflNvnm2Vrk3Kv2WELIImkoL50APqvE1IA54ttAdDtCaksvl8u2B90UTz71d9xcdy/q1pJrFDO7z2/Pbn3LSmYRa+smv1sl+++pBtQXR9kf/Hy9vfZgrqL/8pTTL37VnGTBIrtF2z7IpVij5cZi1L8tnhZ/LZoncIkkoL60gHoOSnF64NDAtCWtYNIoTCASOmQItj2Gkh1HZL/7x/+ZZn1y6rZz3Z5/1d/qn3XB2yLYrekqqjPYPXudwCihJ2T/dBo56RyJJFIYjSDpg0knpTy0UpWWgCR88iGFNE+O5Owap4BxA4+Aeiw7WLbHlZnhZ2yau3nnHz+smx4lMRURb3pnhNzDkCLcvtyi00FUGcz6aSgnrQAIqV6vshlACKLztkp5hf/+p8/HIyV1fdFyzUrS7kFUFX3URkBSCYpqCstgEhDuigedgqrOekD1DmFFUX6W1K2bPuqucHaNaTQe7z9zz+VrWgGBDsBbbKP6z5Ys/vzzrmpOeX1T2EcgCSTglrSA2iz9zfF/2Uj+quit0sK+LLVJyob0duDelhlu/xbdrIo2rJF57gs0vOMdqwXrf1silZzfvv78q++qByKILQRXWzzZ3Vd0Oy+3kV5nqsa3f1GdANQU6EIJ6V8tJKVHkCkr1PWAmU1v0/KinTjK4DKFfWfcFEirH3Nti/Lim70s48Wrf2UW5Tkld1yVoSXWVOU9e6bXVzSbvz9avteN74BKKtH0IWTgvrSA6j4Iz2sTiPbosVyn4z7XX+ULX5sACIjffV4YJ6zLnQ5uvcV7TuTjf57mf3yRzpmV++HDiS+W9ZcbGDwz1mTetvuUNe7r3dBEviBDjyyJvi2O5BYAUQ2q+pF4aSgnqK8Gr/JJtskMpc+IA3FCNBPH01fqAJAjhQfQEW7S6AxAoAcKT6Aipbuu9NbASBHig8gKCgBIEhLAAjSEgCCtASAIC0BIEhLAAjSEgCCtKQM0PaBwuwqGRPZ9idTN0IUO+LG3nBnaCh9N/7exqIW61QjhSO3NZAkQMYOL9lRkAAphQlKAGhgcxUBIAnR8v1y2UwVoxPSq4np+zlbub+lW2wffEGnhpFDRmZtlQd6+/A3GZl7wxaUO6CHtd77fr3fJm43eG1rxaNBFnn1uaVrSA7VOjZZNdv7gmZyS+9PoneClLmfVmHyXnbNV93vLqv21ls3ErUMU3+R1oGJSFoALQsmqpnF7IgvD68PDsuyICsztsV2SThZkI3YZEA2H4NODKsWVDtoCDmtK47WpOR6dRm8trXjsZu5mn90R8tmHYlHpulfH5RTHItVBUV17g1A/ezI1q3NWsvY3vrrxqKWX7K3hWqBeJIeQIdNPbx9yED6v5ftleUW9eeD0w37M6RHnS6uFlQ76AP08Hk/bneXla0dj+6z+K/6ZAC11hXx6OdlczIp/tW5NwD1s+t8xdayam/9dRNRW1+kfWAikt4p7LR1Ij9nJw02gXivtbLakhTDg9PL1vRiurheUO6gB1Cz3zpuN3jPxgqD/CG3PpuqrEmA/rGX5UzOYfT3Ovf+1t00NvWda9WyZm/ddWNRu6i2D0xEMgHQZfm9i9M7qcP3Trt09QBqVdIMoGYB3UEfoHJxx9Olt2ObBqiM1y7KfPPWjyfk3FLn3t+6nUa1WXtZtbf+urGog5FikrkaiIjU0fQPuV8D0Wr6ITmcm9bdeXR1e0EJWQ+gTs3OAahjY+DQhyqwW2frU1h7XV6edKp+0/WHXxaF2sq9v3U7jWqzzrLqpNlbNxa1d7KM87ZFYwDRg1Ieh+1yB6BWI5rcMMMOFl1dLah2QJqZ5G4v+uNhvd8qbmt103To13j9RjTZUbOOxSNzFqvmLDlRLljBl7lXYfrZ0a9abtZeVu2tv24saheg9oGJSOZqoPLcT07+e7/t1Qi0M7vI635x9Wwz6q0WVDsgHeCPPzxl7ZtFtZh1tfPOarqDcn0nXq8bT/norNs77XSoqxs9WrmXYXayy5vNOsuqvfXWjUXtAtQ+MBHJzUBi5zynuo9fG0gEMq14ANrE1j+Zh+IB6D+iG6SdhTCdA9ISAIK0BIAgLQEgSEsACNISAIK0pAzQmq+h5SLy5EXKKl4A5Dts5CkDIN9hI08ZAPkOG3nKAMh32MhTBkC+w0aeMgDyHTbylAGQ77CRpwyAfIeNPGUA5Dts5CkDIN9hI08ZAPkOG3nKAMh32MhTBkC+w0aeMgDyHTbylIUAuvrsgn7ePFk9el1/ACDv3gBSFgHozep9CtDds+P81QfVxyhAWZYZSc+lN4DSiCSsJEAv3vs3VgPdfH5BKqPyYwwg+oASE+m59AZQGpGEVT2FXT1+nd88PSs/igVvF+IaKEBje4RSkwhAbx5RcsqPch2X0KySLt8uvQH8OUcS1lgNNAhQ3QZSomi+pRFJWFWAxNtA3fRkK6P5lkYkYVUBunt2xHphR5O9sJ30ZBiab2lEElYJIPJPdxxIsGE039KIJKzXkegSoeLfIErzLY1Iwvq/lDHeS5tvaQQctlNU3gGiGQEgba+zsL3CCgggHkKpl4Y5r6uwWY+gIACibSBuJZR4aRj0ugsbIkB1ZspenbhGramn3G+xhgQQ7ZApe3XiGrQmnHJWFlB4baB2kl2E0i0N017rYQd6OqEB1EMo1dIw77UdVqCfHAhAHYQSLQ0LXrthh68ZhAhQK98kS8OK12LY0UtOYQJUI5ReadjyWghbXmgav2IZKkAMIV/zqQHQup56M1EC4QIkesF+UEGVhnWvJYCm+AkZIOnZZ6biAiAiwaMfMkD+7uiYPUCZcAMiaIDWInXooIIpDSdeQ2GrGVpjPfchb4AAUa9qJeS/NFx6zYRtzlsKk44DBSgT/zLG4s4bINW4gQJUX8NT8WqEjcxrJKxCtyUOgMp5TPJejbCReQ2ErRrOUsc5AoBKgnT+NpTCxuXVP8rl0U2vBmICQFbD1sdWtrEAgAxbowRIpt8+HDdsgCav6pmMOy+ANPCJB6Cs9b+sVyNsPF6NC0amUg4aIAWCAJCA6O0wMwJIhiAANCmKj7GUwwZoLd3LBEAj6gwZzgOgtWxHEwANa/DeLp24oQNUS5AgADSseQMk2KMHQEMaub9dJ24UAElc1gBAfNGrXUN3l+rEjQKg6oofAFK1ZtZuII8DoN0KWMarETZwr0QNZDJsjACtBRtBAIgniw/RiQggoWmKAKinwbk+MwRIZHoZAOrK+oMEtQFyK7yBQ1IUICeBVI3TjOrwvSvxm26Nhg3Tm0INNB1CJz2OxgkCQB2NdF1nC9D4lVUA1JalIxU5QKNtaQBUa+JOixkDNHZoAFClqTt15g2QuteKNUCAZJ7xoxM3SoDKQWnDT7lPDCCJZ/zoxI0XID5BAIjJ6kWfZADi/JkBICq7M6fSAcho3GQAErvRfdYAWXlZXSoACT4oYd4Ardu1tKGn3CcCkOi9vLMHqOEGALXl4P6VRADiEjR7gFzcQQeAtMIG5N21it/EC4AwDtS3Sj3KBACtDU8Vjx4guUfhACAiDCRWcvkkJQCkGzYUb2N1+iy3hADaaUgDIAdhUwKo//gOAOQgbFIA9QiaM0DOwiYGUOc9DzMFSOFhmQColLGHlkQMkMrTVgFQKQDk/K0iAMhA2CC8zKr9sHAdb/wAGXtsUrwAuQ6bGEDr9uOAZwiQ+1fzpQdQ80Ty+QGkyA8A6qoiaHYAqfIDgLqaK0DqL7gGQF3N8VKGrxekJwnQDEei5a9/GQm7BkAmw3r0AiCz3kwzLgBS8SYEECUo935EHXp1+AFAu5oZQObeOqjjTQkgOpA4G4AMvjROx5sUQHQcyPfAiCNvpjnuDoC4CqBV6cqb+Qnb9woAdPNk9eg1+eHViuiYfr5/ESJAWv2SAEpDQrp9TncA3T0rkPmg+u1NwdKL41BrIABkO+yOdxqgm88v8qvPygrn5ulZfvf1GQDy6830h03dAXT1+DXlhopURcUpjZzICr1daNDnRfTFEMm/TyPE7zicEDlpVQDRz6tP27WQXb4VvIIP6DId1qE3M3Hp2E8N9Ia1pgvV7SC76al4VU9iAZSGmJo/kQBSngao3QZ6cVQtDRkgv5OEXXijAuju2VHVC2MnLlIN3X0TZDeeeVOvgZovGEDK0wCV40CkEirPZK9Wq/fqjpjd9NS8SddAFt79ruMVAGhcdtNT86YMUGekIoCUkwRIjaAASmNaWRQA3Z7sF/8WEQPk8V5fu944ADpfUIqECLKbnqI3WYCyKNpA1weH5GNz73msAPl8XIVlb+eLBZByqgApNIMCKI1pdb9WACnzAMovCTrXB/sC/AQKkEo72nvKIooDoHxDWmpC/AQLkDxB/lOeVu9LBZAyHyAJ2U1Px+vpqZMWvTsn5gBSThcg6blB/lOe0G7DLoCU+QBdZtnh5VsvYwYoSxAgk49VtzsO9NYPB4cxjwOtUwSI07MMIGUeQEU3nvTkY+7G08OdWhvI6KtlANCUN1tL9sT8pzwusy+3sjsORE5hkY8DUQEg/bATXi5ASYwDUXl4gZYtL++rBJAyHyAJ2U1P2+v+FX6WvNwGXQApAyArYY17+R2CAFLmA3RJTmGHsQPUegeLy7AWvEMdygBS5gKUwsXUdUIADQ5pBZAyD6AEpnMwyREURMpcASDl9PS8Wet/h2GNe2MDKN/snaZwCisVP0CxtYGuD7JK05WQ3fSMeAUJCinlrga/QAApc2sgGdlNz4g3doCGr+kFkPIcABIkKKiUWxq5JhxAylyA6NVUgdNXBABJtKNDSbmvCAE6X5ChoMsFAFINa9A7NiklgJR5AF2z2WTxd+OJSoIE5gYFk3JPUQJE+vAJASQ0OzGYlLsayzuAlHkA3Z7sk5EgeoNz9ABRASBrXh5A+XaZLfLzuCfVdxQxQKNpB5AyFyAZ2U3PlDfaNtB43gGkPA7Q9V+mMBJNFSdAE9wHkPIsABK7Kh9UykxTz4gIIGUAZC+svnfqxBtAyrMASIygsFJmAkCa6RnyxgpQBClrAxSRgnu/xKQiyjjtGogp/PNBTzE02+YEUPhdmo4EHtMXQMrjAAnIbnpmvVEBFMnYJxeghOYD1ZpuR4eVcixXX7gAJTQfqBYAsuPlAZTUfKBakwSFlXLkAKUzH6hWZADF3AZKbz5Qo7Cn97UlMg03gJR5AKU3H6hRNADFMo2bC5CM7KZn3hv0LQ4tASAj6Zn3RgJQNLeycQEqTmFi9zXHBdDELT5BpRw1QKLvmgNA1rzx3I3NA6h8vEtyAE0QFFLKcQN0e5I2QOE+K6VSRE+k4QEkOIQYH0BUIT+tqVLkAFXPB0qtEc0UAUDC71sMIGVuDSQju+lZ8AIgo975ARR+G0j8ha8BpMwHKIH3hQ1o5JJqMCnHD1AK7wsbUAQAxfVwfR5Aabzuia+RoaBQUgZABtMz7x2uggJJObIXDPEASud9YUMCQJYb0cm8L2xIgb47aU0zi+r1HnyAJGQ3PVteAASAtLzBAiTFTwgpcwFK8b6wRkPN6ABSpmOcCQCU4n1hjcIFKMZX3fMASvO+sEbZGgDZHwdK7r6wlvgE+U85FYBSvi+sVogACd1MaCGsjpcHUMr3hdUKE6B1IgOJErKbnkXvTkH5T1nsYaDGw2p5xwFK7AFTHQUKUAptoFkAtDvpxnvK0hWQ/5RnDNDuM5i9pyzXgDYWVs8LgByHHfbK8+M95fWcAdohyHfKAMhCela9YQGkwI/vlOmP0wDdPFk9ek1/fbVard6/aC0AQOa8E3fu2wqr650G6O7Zcf7qA/rri2Pyf2tB3AD1SstvymUuiQFEdPP5RX712QUh5+uz7gIAZM6rUgGFcJR3AKrua65vbb56/Dq/eUrQKU5dq9Vxa8HbhQbBi0AhvYoia/0fo7qT6gk61UNe3jyqeLn69IzUQs0CIrt8W/Z2OmJeU1aqgEI4yjyASnQ2/RqI6sVxd4Hd9Cx7QwPIeVh97zRA3SZPAVA6baDu9AmfKavxE8JRHjuFsft67p4dlZ0ucu66++aiWRA7QN0JXABIxcsFiN0XVj2mjA37kDrn1Wr13lmezDhQOACptYCCOMp8gCRkNz273rAAwqUMC+lZ9gbSBlKtgEI4yvMGqFNqngFK91pY6gBVJectZcUWtG5YM14A5Bsg2TsxDIUFQIa8DUF+Upa+F8xMWABkyguA9LwAyE/YJn4m+0QFI2EBkDFvXXQ+20CpjURLyG56DrzeAcqV+QnhKHMBSvv5QD15BihbJwhQ2s8H6qs6g3gEyH1YqwCl/nygnrwClGnFDeAoDwCU9vOBuqruSAdAKl4eQLN4PlAjnwBpxg3gKPMAmsXzgVryOJAo/TwOM2ENerkAychuek68/gCi/CRYAwEgF2HXCQPUmdKaPEAlQe5T1o4bwFHmAtSZVD8XgHRqAgA0clsPALIVFgClApD6nFKtsJn2AGYAR5kHEEOnurV5NgC5r/iSrIF2Hq4AgCyFzfR7fwEcZW4NJCO76TnyAiBlLwCi8nApw8Q0kgCOMheg7XJmpzAGkNtrCgkDdHuyf3tyOKdGtAeAjMzFDuAo8wAi6Jzv5xuhq6l203PmzQCQmncIoMv5TCijovNyHN7hZ+aW6gCOMg8gMhOooOdybjWQu7CmHuoQwFHmAlQ0gvLzbO9UgJ9UANKbmyzpNfZYmQCOMhcgGdlNz50XAKl5xwGaw42FpQCQmhcAldK6PwttIACkdYeoNEAaXkNWAGTa6xAgY69YCOAoA6BK5LTiAKDd+zBmDlAqog1bB2HymF+IMaLZ10BaD3oSDsub+jjzGshueu68bgCikTS85qwYSDTt1eEHAM19PhC1qj8pTCbsToj0ACLPdplfDQSAlLw8gASnkgEgubBDLzRID6Dbk3kCtFYnSCDs4Ft10wNIcCpZkgBZC5v1PmW8GmGte7kAzbQR7QIgToj0ACLzyWZZAzm4lDELgObaiHYAEC9AegDNthGt3ooGQN020MN5NqLrB2YYDpvt/CDu1QjrxMsDqHq+wuwa0ZYAyjg/iXo1wrrxcmsgGdlNz6WXAaRG0FjYCX4AkM30XHpLgIyHnSFAcx0HWtt979LAvtMDaIYPV6itFi9lDE0XSQ+gOT5cYW0boOEJa2kCNLeHK9RWtX7YUNhW+2dGAM3x4QprAKTm5QI0x4crWAGovasZtYGkZDc9l17bAEl6NcK69AKgvtX4QOLUbtMDqHlU9AIAmQo7J4Dql62IzK63m55Lr1WARvaaHkDNuzIEevJ203Pprawme2G1ANCMAFIgiB926irYmFcjrFMvD6DmfWECQ0F203PpBUAqXi5A1RsLLwWGguym59JrHCAxfpIESEJ203Ppra0KBAEgANQCyHTY8R0CIIvpufQCIBUvAOJYDZ3CBPcHgCym59JrDaCJ3QEgi+m59LYAkiZotBENgOYGkHwVNAbQ1M4AkMX0XHpNAyRYAQEgm+m59LYBkiVoBKDJXQEgi+m59HYAkiRoJCwAunmyevSa/nT1yWp1nOevVqvV+xcJAyT/yN/hsNP7SR2gu2fH+asPKElPz/KrT8/yF8eJ10AASNw7DdDN5xf51WekwnlDMHpxfPf1GQASDSuwm9QBunr8mtY9JU5Pz4pTGj2T5fnbhQZ9McvEezMyth8D2USh4S/65lELoLtnR/Qs1tRCdvl26e1Y9WugTLQmm1MNdPPkqFxat4PspufS2wVIbixoNyx9rRMAytttoKIXVjefUwdIsgoCQGO9sKOyF1byQ85pd9+k3I0nMtKIFmqLpw5QOQ5UVEJk/Ic0n4vP9+qOmN30XHrNAyS2j+QBGpfd9Fx6e1bNU5j4LgCQxfRceo0DJLgHAGQxPZfeXYDEGdoJm60B0MwBkiKIB5CoGQBZTM+l1yRAEhUQALKZnkuvSQi0ai9xBXCUAdCwVaIZzb+Uoex1YAVAxr1mAdJogOuEde4FQMNWACTgBUAjVs1uvHJcF1YAZNw7AJAQCjtjSAAIAK0lOmJ9gGSGsQGQxfRceod6UiIwACAAZBQgzbgOrADIuJdnVRxIBEAAiEkNIFO3BNm0AiDjXgCk4gVA41b5KRkmb4u2aAVAxr3DAMlMjDd5V6tNKwAy7gVAKl4ANGEVG0sEQABoBCCpR7SgDQSA2pK9s8L0E15tWQGQce+AVfJaGAACQF3JAWT8IeW2rADIuHcIIJnbk628a8yKFQAZ9w60gVivSvBZzzIT8cfj2rYCIONeAwCp8AOAbKbn0guAVLwAaMpatoFEXpiixA8AspmeS++wVXAkWqUFPRrXphUAGfcCIBUvABKxCl3KUOQHANlMz6V3zCrywhQABIAGNdU8zqvVGIkGQFxN9ORzdX4AkM30XHoBkIoXAAlaRwmqrpcpNYMAkMX0XHonrCNwyM9ClIhryRoMQLPRyBHKTLycJXKhBpq0jp7BUAMBID2A7MW1YwVAxr2T1iFKMlzKAEBCVj4h2Roj0QBIyMqrgtgyAASARKy7bZ2s9iq3ggCQxfRcekVqoD5B1W/BpmzdC4AkrDsA1b8Em7J1LwCSsGal6l9MhAVANtNz6RVvA2UdkjTDAiCb6bn0CllrZvoAYSQaAIlsNAQQRqIBkJi1TVDLC4AAkCRAAktNxjVtBUDGvUhZxQuAfIeNPGUA5Dts5CkDIN9hI08ZAMlbd1rMWfAp2/MCIHkr54p86Cnb8wIgeSsAAkB61t0pHcGnbM0LgHyHjTxlAOQ7bOQpAyDfYSNPGQD5Dht5ygBIyZr1foggZUteAKRkzXqfEaRsyQuA1KxZ5yOKlO14AZCatfc8oBhStuMFQL7DRp4yAPIdNvKUAZDvsJGnDIB8h408ZQCkas3a11TjSNmGFwCpWjt3yceRsg0vAFK2AiD6IwDyHDbylAGQ77CRpwyAfIeNPGUBgG6erB69bv/ULABAfr0BpDwN0N2z4/zVB62fmgUAyLM3gJSnAbr5/CK/+uyi+alZAIA8ewNIeRqgq8ev85unZ81PzYK3Cw36oFlpGKA3jypeyp+aBUR2+XbpRcoq3mmARmogAOTZG0DK0wChDRSuN4CUpwG6e3ZU98KOWC/sCL2wMLwBpDwNUDnsQ+ocjAMF5g0gZQGAxmU3PZdepKziBUC+w0aeMgDyHTbylAGQ77CRp6wN0IB8jVB7ijuvsLy4AAhhteICIITViguAEFYrrmmAoJkJAEFaAkCQlgAQpCUABGnJKEBvVqv3L6Y3syAy4d+5Xq28fN+7Z6v3zqY3My3ybVerncNsEiAy86OZMORUr3a/mX298BCThX3TTKtxKk5c06ewZsqiS139/T+4L8y7rz3UA2x+qCc1E5obmQbISw109/W/eziF3TzhVenWdfX4d15OYTm/cM0CdPWJl6/26shHG+jq0zMftdDVJ8f0Ngf34lVAxmsgbhDLKg6nl0Y0kft2UPfmGJfitryMd+M9tCxZ/+DIeVwi91/35h99AfSCd4hNAtS989CpfNRA5OvefeO+RfvC0ymMf7o2WgMVVYGn5p2vcSAfX7dovHsZbePXDRiJhrQEgCAtASBISwAI0hIAgrQEgCAtAaAJXWZZtnfaWvDTtzvbbB+cjq1OWgBoVLcnb70kEB3WS9qwcDSxOj0BoFGdE34KgtgHEQDqCgCN6fqgrnooGdsHXy6zbH/74IssI0zdnmTZgq5jaw63ZLXHhN0LAI2pVZ+UAJ2Sz+3y3vPbk0Ve/SOLlwVQl/eeowaCWto+fF7/2AHokP68uVes3lBqqkUACGppqAYiPxentw05jbFl7dWzEgAaU9UGuv7wFADxBYBGVXa/qsZN5xT28PmGDBCVpzAABHHUjANdH+wXv+2dkkqJ14guAWr12+YhADSh82okmvTQP/7wtFiwoN34Rd7txjOAyGqv+boWAFLQ7M5TIwJACgJAjQCQggBQIwAEaQkAQVoCQJCWABCkJQAEaQkAQVoCQJCW/h9YJPEWH3Hg+gAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
</div>
<div id="spline-smoothing-for-selecting-a-cutpoint" class="section level2">
<h2>Spline smoothing for selecting a cutpoint</h2>
<p>Again in the same fashion the function of metric values per cutpoint can be smoothed using smoothing splines. By default, the number of knots is automatically chosen using the <code>cutpoint_knots</code> function. That function uses <code>stats::.nknots.smspl</code>, which is the default in <code>stats::smooth.spline</code> to pick the number of knots.</p>
<p>Alternatively, the number of knots can be set manually and also the other smoothing parameters of <code>stats::smooth.spline</code> can be set as desired. For details see <code>?maximize_spline_metric</code>.</p>
<div class="sourceCode" id="cb35"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb35-1"><a href="#cb35-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, </span>
<span id="cb35-2"><a href="#cb35-2"></a> <span class="dt">method =</span> minimize_spline_metric, <span class="dt">spar =</span> <span class="fl">0.4</span>,</span>
<span id="cb35-3"><a href="#cb35-3"></a> <span class="dt">metric =</span> misclassification_cost, <span class="dt">cost_fp =</span> <span class="dv">1</span>, <span class="dt">cost_fn =</span> <span class="dv">10</span>)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<pre><code>## nknots: 10
## nknots: 10</code></pre>
<div class="sourceCode" id="cb39"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb39-1"><a href="#cb39-1"></a><span class="kw">plot_metric</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABOFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SOTU2OTW6OTY6ObquOyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rjk2ryKur5P+2ZgC2Zma2kDq2tpC2ttu225C229u22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+iRaHgAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAgAElEQVR4nO2dC5vctnWGuXI045HSNkl3E3kd95K449qV7WrTNq6TVVtPkybpWtOkjSVtZzTuXsT//w9KALwTIAHiwgPO9z3P7uySPARw8A5uBIEkhSALJVNHAIpbAAiyEgCCrASAICsBIMhKAAiyEgCCrASAICuNBGjzp7/I//rDX5zWTxz+8ir/6/7i4cvB+2hdlN6eL7RjVkWgNyTVZfqRgrjGApScXPI/DqukAdD2wcQAVRHoDUl1mX6kIK6xAJ18V3CzfWelAEhL4QAyuAwAGWgsQA9+zp18f/ETDtC3HyTJn/4u+zdJktP7i8U2efBbng2/XyXfucyNRKZxGn7/3SQ5+anIK5FfmVF1n+zff8wu+JvcMDPZZ/fJju+TNf+/gDa/fXkLEYHs/L4Il92RWeYh/fq7LFxxmXGkoK5GA/SbRyx/Do9/wwDKKrJMJ5cFQO+skoffsCzYi+PC6LBiuc8Y2CZc63ZeFfcROZxwWlKWve+sxHFRFu2LO+7Ly1sAfS+/Pt2vapfkd03WNYAMIgVJNBqg/7xgGbBZHDKA7i9YYfGH1UJ8obPsOS2+x5nff1+0kkSmbB5c3V88+B3LmEUrr6r7HFY/eMkv4Lo9T/785f2vkgW3zn7lVUx5+1p2swhk1y/E9VlUsjtuExFK9t/iZfbfolaFGUQKkmg0QFcsF2/P1wygg2gHsUwRALHcYVlwaDWQyjIk/Z9f/9MqaedVdZ/D6p2f/La0uz0XV2S3ZUVFWYOVt+8AxAnbsPvw0DascGQhsYjxGFRtIP1IjXPVzDUeIFaP7FkWnYqaRBTzAiDhfAbQum4lrl0XtUInr2r32bDP7+cNj5yYIqv3zToxlQC0yK/Pr9gXADUuM44U1NV4gFiubhapCUDs0EZUMd/75//443lfXv0ha7kmeS7XACrKPi4nAJlECupoPECsIZ1lj6jCSk7aADWqsCxLf3m+KDK+aG6Idg3L9BZv//0PeStaACEqoH3yYdkHq26/adRNVZXXrsIkABlGCmrKAqD9yV9lv/NG9C+y3i7L4G2tT5Q3og/n5bDKYfXXorLI2rJZ5zjP0k3CO9aL2n32Was5vf9V/q3PCocsEN6Izq75k7IsqG5f3iKv54pGd7sRXQFUFSjakRrv5RnLAiDW18lLgbyYP2V5xbrxBUD5ifIrnOWIaF+L6/O84hd954NF7T75FTl5ebdcZOE2qbKyvH11iy3vxr9TXN/qxlcAJeUIunakIIksAMq+pOuiGjlkLZZ32Ljf7QfJ4psKIDbSV44HpqnoQueje7/gfWd20X+tku9/w8fsyvvwgcQf5CWXGBj8M9GkPtQ71OXty1uwCPyRDzyKJvihOZBYAMQue2kaKair+J7G75PBNonJow/ITtEB9O0Hww+qAFA4RQZQ1u7SaIwAoHCKDKCspfuD4asAUDhFBhBETQAIshIAgqwEgCArASDISgAIshIAgqwEgCArjQPo8GjE7CoTI3btt65ehMhuJA17L52hMSpt8rv1hZqdGxsSKQUsgQwBcuZediOSAI0KhpoAkOzyMQJABuL5++WqmirGJ6QXE9NPU3Hy9MCvODz6gk8NYy5js7ZyRx8e/zxhc2/EgfwG3K3l3U/L+1bhNgMvzWrh8UDE5EX2eeBnWByKc2KyanLyBY/JPX8/ib8Jksf9sggmbcWuSupp81hxt9a5nlDzYMqE1BwTl8YDtMqYKGYWC4+v1rfn6zwv2MlEXHFYMU4W7CIxGVDMx+ATw4oDxQ0qQi7LgqM2Kbk8nQdemtXDEy9zVT/8RqvqHAuPTdO/Pc+nOGanMorKuFcAtWPHrq5dVjsm7tY+1xdqnsjWFTZZOY0sAFpX5fDhsQDp/17WT+ZXlJ+PLvfia8i9zg8XB4obtAF6fNUOt3nLwqweHr9n9qv4FADVzmXh8c9tVZlkP2XcK4DasWsksXasuFv73ECotYTUHROXLKqwy1pFvhGVhphAfFI7WVzJsuHR5bY2vZgfLg/kN2gBVN23DLcZeMtMZAb7Itc+q6KsigD/suf5zOow/n8Z9/bVzWjsyzfXimPV3Zrn+kJtolp3TFyyBmibpzur3lkZfnLZpKsFUK2QFgBVB/gN2gDlhxs2TXobZsMA5eHVszLdP/zmgtUtZdzbV9ejUVxWP1bcrX2uL1RlSJHJUQnExMpo/kVul0C8mH7M3LmvvZ3HT9cP5JC1AGqU7BKAGmYCHL6ognh1tqzC6ufSvNIp+k2373+ZZWot7u2r69EoLmscKyrN1rm+UFuVZbSvLboBiDsl98Nh1QGo1ohmL8wIZ/HTxYHiBqyZyd724n+uy/sW4dZOV02HdonXbkSzG1XnRHhszmLRnGUV5UJkfB73Iph27HhS88vqx4q7tc/1hdoEqO6YuOSoBMrrflb5n/yyVSLwzuwiLfvFxdpm3LY4UNyAdYA/fP9StG8WxWHR1U4bp/kN8vON8FrdeM5H49zJZaNDXbzoUYt7Hkwndml1WeNYcbfWub5QmwDVHROXAgwkNuq5sff4qYOIQB4UCUD7CPsnx6FIAPr3GAdpj0KYzgFZCQBBVgJAkJUAEGQlAARZCQBBVhoH0HVXsmNDCmRDOGrekuMYE7UAUHQ2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2ACi0DeGoASAbT5DyeDw2MwVoZ+4JUh6PxwYAmbnC3oYwDACoFAAilRzHmKgFgKKzmSlAsXs8HhsAFNqGcNQAkI0nSHk8HhsAFNqGcNQAUBXjEa1oSh6PxwYAmbnC3oYwDACoijEAopQcx5ioBYCis5krQOaeIOXxeGwAUGgbwlEDQDaeIOXxeGwAUGgbwlE7VoCk2rm7FRSNHJZA5t0wSl/ZeGzmUAJJkwWAwtgAIDNX2NsQhgEA2XiClMfjsQFAoW0IRw0A2XiClMfjsQFAoW0IRw0A1ZNl3Iqm5PF4bACQmSvsbQjDAIDqyQJAQWwAkJkr7G0IwwCAbDxByuPx2ACg0DaEowaAbDxByuPx2ACg0DaEowaAGskybUVT8ng8NgDIzBX2NoRhAECNZAGgEDYAyMwV9jaEYQBANp4g5fF4bABQaBvCUQNANp4g5fF4bABQaBvCUQNAzWQZtqIpeTweGwBk5gp7G8IwAKBmsgBQABsAZOYKexvCMAAgG0+Q8ng8NgAotA3hqAEgG0+Q8ng8NgAotA3hqAGgVrLMWtGUPB6PDQAyc4W9DWEYAFArWQDIvw0AMnOFvQ1hGACQjSdIeTweGwAU2oZw1ACQjSdIeTwem3kCtNthrwxKyXGMiVqOANrtcoKMMKLk8XhsAJCZK+xtCMMAgHIBoHA2swSobAMBIO828wQIG87RSo5jTNQCQNHZACAzV9jbEIYBAFUxBkCUkuMYE7WcA2TCESWPx2MzV4CuzbthlDwej000AL05O/vRizS9+/TsyevyQ0iaLAAUxiYWgG4+fpG+ei99+/xZ7SOXNFkAKIxNLAAxZRDdff6i9pEflyYLG86FsYkJoKzMufnkdXr32Vf5R3bs3Uzyq7Hr7hGqD6Cbj374VfrmCScn/8jPyL8X2OogiE1MJVBV9FQlEJM8WQAoiE1MAKVfP9NvA5m3oil5PB6bWADKK623z5+KXtjToV4YAApjEwtA6auzs6wNpD8OlLMDgDzbRANQjxTJAkAhbOYOkGtX2NsQhgEANZIFgELY0AXo9nzNPvYProasFMkCQCFsZgwQ1kgMYUMVoG1SaDFopUqWYSuaksfjsaEKUFkCaUiVLAAUwIYuQPpSJQsABbAhDFBWBN2eJ8NNICVAHB4A5NeGMECbRbp9cLUd3wbCAlMBbOgClBVA9xcLm14YAApgQxqg2/NTAETchi5A9xen+5NLVpENSZksAOTfhi5A6WGVLNLNw5eDVupk7dCI9m1DGCBtqZMFgLzbACAzV9jbEIZhbgDxxxmnw1bqZBkNBFHyeDw2hAHasv4X64gNSZ0sLDDl3YYuQPZP46/xZqp/GwBk5gp7G8IwDNt010+mC5CLKgwAubUpVy81DMYbMG25bkQbdcMA0NwA0lWPJwCQS5vd8QGU4QOA3NhwdKJqA91fnKbsefyg+jxhUAQBILWNevMRwgDxx6g6BPUlCy8WOrCp6Om6ky5ATrrxAMjaplb4yIqhuQOEZ2FWNuWeEUo30gXIyTgQALKxKbDp232NMEDp3n4cyKQbBoAaNqLwGXQeZYB01ZssADTKJt8ua9h3MQB0++P+ZlC/JwCQsY3JblmzB0ifIABUjRXq7zl7DAC5dIW9DV2AyicVJlsWAyAzV9jbEAfISzA+WJEKAE1pA4CkyQJAmhrFzxEApF2jHzVAoqvhKzk+WJEKAE1js8u7qjMFaEgDyQJAQ9oVPpoXQIcVX+LO8mGq/kDQ8QJUOmhWAGnNJeMaShZKILWaq7jNCiD7NRIbTnLkCnsbSgDtWqsAzgqg+wsA5Nmm88B0VgDpTCUTGkoWAOpRwzmzAuj2PHHTiNbthh0VQLvW5+hw6AKkr8FkAaCWdnJ+jhWgQWH/3aYKf+xm5xjnrza3vnD23yV7m2lLoJ3i79HhEC6B3Eyql3tqtCvsbaYEaNfPz7wAcvRaj8JVY11hbzMhQA18ZE4BQIpkoQprS+6RWQHksAoDQG0PKBwyL4DcNaKPHqB2fWW1UsIIE0+4dOVpHOjYAWrjY/ee8ggTx5ioBYAC2Ni+pzzCxDEmapUAib3CHD3KuHb4ipy9TViAugnvc8V8ADKSRrKOFKBuZdU/pjorgFx2448UILPiZ1w4AMjMFfY2k45Euw+HKkAutv2uS4OgOQGU1VQDI9FuwqELkMsprdd6j1PnA9BO+o6Xq7VaRpj4YEUqf43oowJIauOpFicMkKvXepiOBiDlBHlfQ2F0Abq/OL2/WOtUZDrJOg6A1BPkvU1IoAsQQ2dzmu6HN03VSpaXBgAxgJQT5D2+nEsboK3dvvF1HQNAKhufC5TQBYitVJ/Rs0UJpGXTSV+qPmUTzigTvdw/PLoch02lzl4Zm+Rk+KZayZo5QOr5zc7Xqxthopf7rgHSlk6y/AyCRACQ0ZJRAEiZLI0lOmIGSDlB3mzJscAA8YGatQAn+3V49EWSsBbL7Xly8uXjq8PjnycPru4v+NOIzkXDADnZ7qnQzAFS2RguWRcWIMHEal2ysXr4kmU4m8Z8e/7g6rBaiEVa2E91EWNKRYX77Z5KDddh8wPIeMnDwAA9vqqBxNhY80/e8d4ygNb5w/TsV/uiYYCcPo2/9vMkkTZAnlbMHGGiyLiNeFResvGYv0Wx5h3v7B9+nI8D8qrrsvpUDi/7BMjHXAbSAJnzE74RnbV2aoWLW4BcvtbDNWOAJEkbsWLvJL2wjIUGG0UVVtRaezaO06nCHitKFQ/bPVWaL0AyfkglR55vHJSMCVZI3F+cdBvRjJmiEV1dpN2I1pZWsoa7YZQ8bmYjnbpKKTmKjGMlxMml6M5/+L7ooTM0WDf+ixygNO/Gdy4CQM5s5FOfKSVnRKaqHqL3jzZ6e63nWmP/b0oet7LZjV00nAhArNmjrKU0ATKSXrIGiyBKHrex2QUKx8DEMEO3PVPhTUogh3OiuY4DIItFw6kANF4AyNrGZtHwGQFUe63H2UDiTAFqpKo2+kMpOV6hqcvbaz1C/QRR8riBjWrVOkrJ8cGKVH4b0bMESLnqIaXkOMZELY/d+JkD1N1ukFJy5Pn2v3I5AchIesmaKUBCku0qKSVHnm+RATQ4EETJ46Y2AEioAZCoxjR2XNFPluvFcQAQZYA2CzajY+toRiLXbAG67m64TCk5kwCUFUDsgYi7CWXXMwRIsWmK83AsTaYCiM0BAUA9Nj38kErOJADdX5yyx7IbZ1XY7tr5CpMAiDBAbAbRIt0Mv9l8vAD18UMqOWMAur9Qz0bVnNKqK81kzQ+gXM72vZgMoKwL0AFICQkAcm0TPUB8EKIFEJvWeimGce5/9kWSnO75/Hg2sXXNAZIO8fSMA918dHb2LE3vPj178rr8MAKIaY4Auds4JThAu46aJdDmlK3ww19MXS3Ykdv3L/kLP+W5PoAa40B3n32V3vztV2+fP0tfvZfmH+YAOV7nnwJADjdOoVQCFbjc/vjq/meXKfth/7KC5cdX1bkegJrjQG8YLl8/u/v8RXrz8Yv848gB2pW//IZjbzIAkKoNxJ+nn1w2ANqwx+vVuX6AWuNAWSl088nr2kd27N1M8ujJNaNtZne131GqvxfGIBElTA0gNklMVGHd0oepdxzo7fOn6ZsnnJz8Iz+h+70YaEVT+srq2PQXQKSSMxKgfI3MGkDsYP4StHT9zL5xoLtPn2ZN6XYJdLwADfBDKjljAeJdsXoJxKY6f+eDdXWuD6Cmbj56xiiyaQNF9JXVtZkxQKOkBkjww6sx3gt7OqIXNj+AYhmVmAagxkr1r86YntmOA80LoGieDU8CkNuV6ksBILc2dAFyvFJ9qSgGTgZthp7L0ErOZAA5XKm+1GwAiug9yUkAcrxSPVM8Q7dDNsMFEKnkTAOQ25XqmWYFUEzLHU0DkLa0kzUfgCJbdFaeb5ECFMXj62EbACRT7dVmD6tz5JoFQHFt/REeICMZJmsOAGms4UspOfJ8A0BTAaS1BjSl5MjzLUQvzNlmKzVFD5DGxjFOwnFnMg1Abjdb4eptRVPyeK+N3iL0lJLjBqC+9zQkADnfK+N6JgBpbmJAKTlDAC2XSwDk0aRThYUJx53JAEDLZZ2g/KzByzxygJxvtnI9E4A0d1GhlBwVQMuOagDpv8yjAMj1Ziul4gZIdxceSslRAdRfAmm/zKMCSFfGyaL/KmePzQwBkraBDF7mAUAGNtr7gFFKzhBAsl6Ywcs8CoA8LHGXK2KA9PeRo5QcS4CGXuZRAORhibveVjQljyttjhKgoZd55AD5WOKuXFV5tCvsbayCMdjIklJyxgA0Sr6XuIsdIH18aCVnEoCcL3HHBIBc29AFyPkSdzVFCpAJP6SSMw1A2jJPVpwAGfFDKjnyfIsYIPLLmspsNCbSOwnHg4k833wC5GG3noYAkCsbC4A8yH8J1NOKpuRxiY0hP6SS4xgTtQCQ0kbjVUIn4XgxcYyJWurVOfpkkqyoAQoRjhcTf8S01BwH0pgOzWWSrEh2J+nYGPNDKjk+WJHK86a7NUUFUHcvJz/hzAug+wsAJCTZTc5LODMDSOcpmNCYZEUIkCFDlJLjgxWp/DeiS3Wzg5LHGxIAmZZBlJLjj5iWWg9TNa2MkqVuRVPyeFNjCiBSyfHBilQBGtERArRDN15XARrR8QG0Cxa1eQGk8yKikFGyAJBbG7oAeX2YKhtboeTxmnbjgiGVHI/INBVsOodsbIWSxyvtRgZDKjmOMVFLDtDQy0BjkhUbQBhI1BIAakvEMcWjDD2FAKh4Hh9FGyiPIwDSVECAuv0wSh4v1PcWictwxtscMUDtbKHk8VxFDNGI1hQAaqiMIADSVLhGNBN1gKr4ASBNjQNotIjveLxrfEAaCjeQyNUsgih9ZZlqDWiUQJpqArRNkrXGbk/jAWoSRMnj180OGADSVHN9oId/FCu8DMkwWTvpn7Q8fg2ARqmzvMva/Y6FkQA0RdQAkFaydoq/KXl8mubZvABKt6wKc7tONBcAcmlDGCBv60TXVcsoSh6fZohqbgDpysYTNAFqrwUKgDQFgIR25a/RwVBKzizfCysU/GuuYdLmBwDpyv/iCkw7+T9kPD7ZfP95AeRvcYWd/D8yHgdAoxVmcQXiAEmWkgJAmgqzuAJtgGRLkQEgTQV8L6xS4Gl/GgB16jAApKnw3fhragBJ10IEQJoCQPLp8wBIU8HWiW6oeHfGxGZMODomAMhK0+xYSAggxfs7AEhTzgBa9qannU07fVe0PePWRPX+FwDSVK0KSwqNqsKiBQhVmJXcVWG9BFEFSLkQIgDSVCCAOgo58Vhtol5IEwBpqvNWRpJoPM+w9wQJgHpegAdAmmpOaWWtHw9TWmUK+PIVAPIoydN495PqZSIAUN9i9ABIU4EA6mbV9AD1bmYAgDTVfBrPdpgfX4X1NKMleRVuKVQVQI6DAUD6Q0HyZMUFUP9uKgBIU04fZZgSNKXHB5YgA0CaCgWQRABonA1hgMQTeY1piU48MSlAQ2sgAiBNNVfnWLChoK3z1TkU2k3lcY3d5ACQplrdePZmT5hxoOvpANLZjxAAaaqzOsfpOICWy/4GkDTDAq2lC4A8qrXhHBsJ2oyowpbLnCAVRgDImQ1dgNirzYt0M2aJu+gA8hAMANJXJ76DAElhCbSfQMtEK1AApClX40BlG8hkLGgSgPTCBECacjiQaDaMKGyC7KlEGAYAVE+WMUETADTc+hkdDADSlyJZpgSFB0iXHwCkq2AASRvRIzpiVh7X5gcA6crti4U9HTEaAPkMBgDpS5ms5bUhQMYE2XjcICwApCnXrzYbERQYIJOgAJCmwgGksAkHkFFIAEhTzhdXMBpIZL8MCRrtcbNwAJCm3K/OYUBQSID8BwOAOrr5+EWa3n169uR1+SHUmyx9ggIVDYFqSgDU1puzH71I3z5/lr56r/jI1ZssBUCKRjQA8mISAB2hHoC+/uG/ZSXQ3ecvWEmUf+Sn+pO1lDI0KUAhhpsAUEeMmJtPXqd3n32Vf2QH3800cNPlUnJQvZOt/z1usYuuNw0C9OYJJyf/yE8MfS+k81uVy8H77l8HmjWCEqgjaQnENJSsgQnSbRvPI3yBJj4CoI5uxrWBrnV7YmEACvXyBwDqiBHz9vlT0Qt7qtsL49IiKAhAOwDkUx7GgYR0CArxlHMX7AVYAKQvjWR1AerZU8knQKRhAEDKZHXb0ZMAFHAdKwCkL41kdUcT+3Z18zXTK+RangBIXzrJ0iDIO0BBl6MGQPrSSZb8gYbKxstk5bBbcgAgfWkma4ggzwDtAJB/+d2tZ4AgvwCVr2AQhgEADSSrn6C6jfMXtqpXeAjDAIAGktUCqGdrZNcA1e5HGAYA1Jus5QBBqfqUUiMuIwwDAOpN1vK6S5AyZ7XIyB9s6b9hqopaxDZHBlBfM8gYoOq5xABCjbOEYQBA/cmSFEFqm2GCOs8lVBjtAFAoBdh0V02QKUCScLoIdVdAJAwDANJJVpcg+Qjf+LXnasRIVtAkDIMTgGQr5DrGRK0g236rCNIHaHBQp7zg+AAq16dsXBRKEwEkMlwbIN1BneyuRwbQcu4ALWu/B21UBOkP6uxka0AThsESoGWpzkWhFAYgBUEdgDTGd/o8rjAnDIMNQCU3824D9RHU6VFZrQCutCUMw0iApIVO86JQmhagZpbbAaS2JAzDCJthePh9QylII5qnW2rTyHVJA6YL1FAvbETUorER7BzfOFCeeqlNm6BmQ0i5IENL0e5/quG2qqGzFG8qHCtAkraeyqbgQbUwp+JylcjAMMKm0c1a6gfjGBO1ApZA3bq7r0Gs3SKOeucdPYCaL0kdHUBFM9oEoGvtPrnG8w8qMJjZLOsyDsYxJmqRAKgLAadHWgyl3euGND0MZjY1ZmRdrqMDSE1QadPGoPy/vxemOa8sHoBKdPI2j87omeK+oRT6abyiMG6iUP9HPTdjDhunVDbLqps1sLzS8QHUUN03mjbyabCz2PeiHFbWAMcoGMeYqBUcoLqXtG12nekcJrOiyQJUoKM1tmwWjGNM1AoPUK0QMvF44zXT6CfVqx+jOwrGMSZqTQFQSVC3S95HRjlObYYPNYCa4IwI5ygBan3Rloox1SEypNN9BkUFoFYfa2w4AKg8YO69qACqypllvY9lGw4AKo/MGyDVSLJtOEcJkIwgzYkJTY3gZwqAtNvIAMjCE8tAORuwS954dg6AlHLjCfON5seF4wmgDjJpixedTjoAsvBE5vERNg5MhosG+QVtZFr3oVSgOsZErWkB0t1Uwy4cpYkSpGY11ECmxw4AaWtMsiQuT+WH+z1jeH3TRA5MCwhJ70nrWRUA0taYZCkAMiXIxuMDADTrploVZbj50Ki4OTZxjIlaIedEK2zMqrGRHm+EoShwmkfGhRPABgB1bEwya5THl21+ymJGRcpUo9euTBxjotbEjehcBgSN8Xjr9jpjfABIUzQAMqjGDMNZtkx0h4gBkKbGATRaLOsUZ/wEJw1cGQnIWBOVQKIEaNholkEm4SxLE+NmMUogTU1WhbH8bLZu9TJZL5z2fQ0jB4C0NXkbyLT3rBNOcx4OZRgAkI0nGo1oPt/KGiBxl+qWDqJGzgYAqWwkjxDMwqmP7tRuQRgGAGTjCdkc0MYzTB2bwrJ2B6NX8DWjRsgGAOnY5JPP9WhoPi8HQNcAKFfVNKo1i1vn28R0Sy7CMAAgG0/o25QYtR5j6XXPCcMAgGw8YWLTmt/lK5gYbACQhY05PaOCIW0DgGxsZjaqDIBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYAKDQNoSjBoBsPEHK4/HYzAEgid51dSMa4cwsOf4EgCYNBgCVmpnHZ5YcfwJAkwYDgKAjFwCCrASAICsBIMhKAAiykhuA7j49e/LayZ369ebs7Ecv/Afz9vnZD7/yH0x681GQ5HiVE4DePn+WvnrPxZ36dfPxiyDhfP0sfRPgC3H3aea2IF88j3IC0N3nL3jmhlCAcFhyQujmk9fp3WchijqPcgJQSE8EKIFuPvnXIFUYACrEyvswnrj5KEDO3nz0jGeub/EqLEhjy6OiK4EChBMsOVkj+u/+BT8WfUgAAAGASURBVACFbQOxFq5n3f19wO9DoPaWNznqhT0N0jsKVVV+HagKy5ISwm1eFdc40KuzIK3bLDlBxmfenIVxm09hJBqyEgCCrASAICsBIMhKAAiyEgCCrHRkAG2TJDm5rB349nedaw6PLvtOQw0dFUD3Fw9fMojW5ZE6LBINnIaODKAN4ycjSHwwASBbHRNAt+dl0cPJODz6cpUkp4dHXyQJY+r+IkkW/Jw4sz6w01PGOAIdE0C18iQH6JJ9HlYPru4vFmnxww6vMqC2D65QAg3qqAB6fFX+2QBozf/eP8hO7zk1xSEANKijAkhRArG/s+ptz6oxcax+GurVMQFUtIFu378EQK50TAAV3a+icdOowh5f7dkAUV6FASBdHRVA1TjQ7flp9t/JJSuUZI3oHKBavw2S66gAStNNMRLNeugfvn+ZHVjwbnzWfW904wVA7PTUUSauIwNIJtRTNgJAAMhKAAgAWQkAQVYCQJCVABBkJQAEWQkAQVYCQJCVABBkpf8HPMWYbKgKgEsAAAAASUVORK5CYII=" style="display: block; margin: auto;" /></p>
<div id="parametric-method-assuming-normality" class="section level3">
<h3>Parametric method assuming normality</h3>
<p>The Normal method in <code>oc_youden_normal</code> is a parametric method for maximizing the Youden-Index or equivalently the sum of <span class="math inline">\(Se\)</span> and <span class="math inline">\(Sp\)</span>. It relies on the assumption that the predictor for both the negative and positive observations is normally distributed. In that case it can be shown that</p>
<p><span class="math display">\[c^* = \frac{(\mu_P \sigma_N^2 - \mu_N \sigma_P^2) - \sigma_N \sigma_P \sqrt{(\mu_N - \mu_P)^2 + (\sigma_N^2 - \sigma_P^2) log(\sigma_N^2 / \sigma_P^2)}}{\sigma_N^2 - \sigma_P^2}\]</span></p>
<p>where the negative class is normally distributed with <span class="math inline">\(\sim N(\mu_N, \sigma_N^2)\)</span> and the positive class independently normally distributed with <span class="math inline">\(\sim N(\mu_P, \sigma_P^2)\)</span> provides the optimal cutpoint <span class="math inline">\(c^*\)</span> that maximizes the Youden-Index. If <span class="math inline">\(\sigma_N\)</span> and <span class="math inline">\(\sigma_P\)</span> are equal, the expression can be simplified to <span class="math inline">\(c^* = \frac{\mu_N + \mu_P}{2}\)</span>. However, the <code>oc_youden_normal</code> method in cutpointr always assumes unequal standard deviations. Since this method does not select a cutpoint from the observed predictor values, it is questionable which values for <span class="math inline">\(Se\)</span> and <span class="math inline">\(Sp\)</span> should be reported. Here, the Youden-Index can be calculated as</p>
<p><span class="math display">\[J = \Phi(\frac{c^* - \mu_N}{\sigma_N}) - \Phi(\frac{c^* - \mu_P}{\sigma_P})\]</span></p>
<p>if the assumption of normality holds. However, since there exist several methods that do not select cutpoints from the available observations and to unify the reporting of metrics for these methods, <strong>cutpointr</strong> reports all metrics, e.g. <span class="math inline">\(Se\)</span> and <span class="math inline">\(Sp\)</span>, based on the empirical observations.</p>
<div class="sourceCode" id="cb40"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb40-1"><a href="#cb40-1"></a><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">method =</span> oc_youden_normal)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<pre><code>## # A tibble: 2 x 18
## subgroup direction optimal_cutpoint method sum_sens_spec acc
## <chr> <chr> <dbl> <chr> <dbl> <dbl>
## 1 female >= 2.47775 oc_youden_normal 1.71618 0.895408
## 2 male >= 3.17226 oc_youden_normal 1.54453 0.864286
## sensitivity specificity AUC pos_class neg_class prevalence outcome
## <dbl> <dbl> <dbl> <fct> <fct> <dbl> <chr>
## 1 0.814815 0.901370 0.944647 yes no 0.0688776 suicide
## 2 0.666667 0.877863 0.861747 yes no 0.0642857 suicide
## predictor grouping data roc_curve boot
## <chr> <chr> <list> <list> <lgl>
## 1 dsi gender <tibble [392 x 2]> <roc_cutpointr [11 x 9]> NA
## 2 dsi gender <tibble [140 x 2]> <roc_cutpointr [11 x 9]> NA</code></pre>
</div>
<div id="nonparametric-kernel-method" class="section level3">
<h3>Nonparametric kernel method</h3>
<p>A nonparametric alternative is the Kernel method <span class="citation">[@fluss_estimation_2005]</span>. Here, the empirical distribution functions are smoothed using the Gaussian kernel functions <span class="math inline">\(\hat{F}_N(t) = \frac{1}{n} \sum^n_{i=1} \Phi(\frac{t - y_i}{h_y})\)</span> and <span class="math inline">\(\hat{G}_P(t) = \frac{1}{m} \sum^m_{i=1} \Phi(\frac{t - x_i}{h_x})\)</span> for the negative and positive classes respectively. Following Silverman’s plug-in “rule of thumb” the bandwidths are selected as <span class="math inline">\(h_y = 0.9 * min\{s_y, iqr_y/1.34\} * n^{-0.2}\)</span> and <span class="math inline">\(h_x = 0.9 * min\{s_x, iqr_x/1.34\} * m^{-0.2}\)</span> where <span class="math inline">\(s\)</span> is the sample standard deviation and <span class="math inline">\(iqr\)</span> is the inter quartile range. It has been demonstrated that AUC estimation is rather insensitive to the choice of the bandwidth procedure <span class="citation">[@faraggi_estimation_2002]</span> and thus the plug-in bandwidth estimator has also been recommended for cutpoint estimation. The <code>oc_youden_kernel</code> function in <strong>cutpointr</strong> uses a Gaussian kernel and the direct plug-in method for selecting the bandwidths. The kernel smoothing is done via the <code>bkde</code> function from the <strong>KernSmooth</strong> package <span class="citation">[@wand_kernsmooth:_2013]</span>.</p>
<p>Again, there is a way to calculate the Youden-Index from the results of this method <span class="citation">[@fluss_estimation_2005]</span> which is</p>
<p><span class="math display">\[\hat{J} = max_c \{\hat{F}_N(c) - \hat{G}_N(c) \}\]</span></p>
<p>but as before we prefer to report all metrics based on applying the cutpoint that was estimated using the Kernel method to the empirical observations.</p>
<div class="sourceCode" id="cb44"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb44-1"><a href="#cb44-1"></a><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">method =</span> oc_youden_kernel)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<pre><code>## # A tibble: 2 x 18
## subgroup direction optimal_cutpoint method sum_sens_spec acc
## <chr> <chr> <dbl> <chr> <dbl> <dbl>
## 1 female >= 1.18128 oc_youden_kernel 1.80812 0.885204
## 2 male >= 1.31636 oc_youden_kernel 1.58694 0.807143
## sensitivity specificity AUC pos_class neg_class prevalence outcome
## <dbl> <dbl> <dbl> <fct> <fct> <dbl> <chr>
## 1 0.925926 0.882192 0.944647 yes no 0.0688776 suicide
## 2 0.777778 0.809160 0.861747 yes no 0.0642857 suicide
## predictor grouping data roc_curve boot
## <chr> <chr> <list> <list> <lgl>
## 1 dsi gender <tibble [392 x 2]> <roc_cutpointr [11 x 9]> NA
## 2 dsi gender <tibble [140 x 2]> <roc_cutpointr [11 x 9]> NA</code></pre>
</div>
</div>
</div>
<div id="additional-features" class="section level1">
<h1>Additional features</h1>
<div id="calculating-only-the-roc-curve" class="section level2">
<h2>Calculating only the ROC curve</h2>
<p>When running <code>cutpointr</code>, a ROC curve is by default returned in the column <code>roc_curve</code>. This ROC curve can be plotted using <code>plot_roc</code>. Alternatively, if only the ROC curve is desired and no cutpoint needs to be calculated, the ROC curve can be created using <code>roc()</code> and plotted using <code>plot_cutpointr</code>. The <code>roc</code> function, unlike <code>cutpointr</code>, does not determine <code>direction</code>, <code>pos_class</code> or <code>neg_class</code> automatically.</p>
<div class="sourceCode" id="cb48"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb48-1"><a href="#cb48-1"></a>roc_curve <-<span class="st"> </span><span class="kw">roc</span>(<span class="dt">data =</span> suicide, <span class="dt">x =</span> dsi, <span class="dt">class =</span> suicide,</span>
<span id="cb48-2"><a href="#cb48-2"></a> <span class="dt">pos_class =</span> <span class="st">"yes"</span>, <span class="dt">neg_class =</span> <span class="st">"no"</span>, <span class="dt">direction =</span> <span class="st">">="</span>)</span>
<span id="cb48-3"><a href="#cb48-3"></a><span class="kw">auc</span>(roc_curve)</span></code></pre></div>
<pre><code>## [1] 0.9237791</code></pre>
<div class="sourceCode" id="cb50"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb50-1"><a href="#cb50-1"></a><span class="kw">head</span>(roc_curve)</span></code></pre></div>
<pre><code>## # A tibble: 6 x 9
## x.sorted tp fp tn fn tpr tnr fpr fnr
## <dbl> <dbl> <dbl> <int> <int> <dbl> <dbl> <dbl> <dbl>
## 1 Inf 0 0 496 36 0 1 0 1
## 2 11 1 0 496 35 0.0278 1 0 0.972
## 3 10 2 1 495 34 0.0556 0.998 0.00202 0.944
## 4 9 3 1 495 33 0.0833 0.998 0.00202 0.917
## 5 8 4 1 495 32 0.111 0.998 0.00202 0.889
## 6 7 7 1 495 29 0.194 0.998 0.00202 0.806</code></pre>
<div class="sourceCode" id="cb52"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb52-1"><a href="#cb52-1"></a><span class="kw">plot_roc</span>(roc_curve)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAAA+VBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYzMzM6AAA6ADo6AGY6OgA6OmY6ZpA6ZrY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmOgBmZmZmkJBmkLZmkNtmtttmtv9uTU1uTW5uTY5ubqtuq+SOTU2OTW6OTY6OyP+QOgCQZjqQkGaQtpCQttuQ27aQ2/+rbk2rbm6rbo6ryKur5P+2ZgC2Zjq2kGa22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb25Db29vb2//b/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+52EZeAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAM9klEQVR4nO2dC3vbthmFYTf2lGypknZedvMu6RZl3dyt0dbZVbPu4naKsigK//+PGT4S4kUkKEgH5Efa5zxPKvkIYonzCgApUZ9MQqnKaO/AfRcBKIsAlEUAyiIAZRGAsghAWQSgrN4BLEymH32V/rn52w+N+ejnt0npr1/2vU+aUgNgTq/tX2+fZX88SHGsJtlfH/e9U4rqH0AafLL5q5na/87MYxv925fmzI6B9YX5+F9J8u0zc9n3XulJC4BNeyrDYepcU/prNTlzU1LyemIeXAkocTazc/m3MKcv042sL86zMeSms1FKC8D/5jbtzczRsFme3RZ//XPbeCkT0slVBcBHE3P27eQyffTSzVq2yVilugakL+FM89Pr0l9Om5lN9nVKqgCQTV3yt32ObWGX7H9Mdp85HmkBOPn4NtkLYDVxM1QZQDpMFpaMtHcttvPaCKUzBb2ePJZZfj8AtxqXAaTrgwQvM9DSDafxzkFKa8DSZIk2rwHfPM5WVT8AubHMCOBwudlinh7xNB8FrS8ciXwKkgGSDRgHwM5Bf5IBkyMarbQAZCH7zgPyqd++sld2cFhev5DGBYDV5KdytmBb/NE+f8Y1IFjb9XJhZMZfN54J52tBNsNcujsPnhUALA03l6WaNvyfxiE1ADZBmT2q7wW9fTmpvhdkT8RO0j+/mZjH/ynWgC1AC+3ZuN894ruhyiIAZRGAsghAWQSgLAJQFgEoiwCURQDKIgBl9Q3gv1sV96oan48FQgCwjwVCALCPBUIAsI8FQgCwjwVCALCPBUIAsI8F0gWAd7/5Or19/9mTT7/PbzINKLhYPhZWBwDePPkkBfDhixfJdz/e3jgNKLhYPpZWfAA3T7/MRsD7338tg8HduEcHFFwsH4urwyno3W+/T97/7pW7scYPrDr43/UsdylYtM3F2lBJDsCbT9Pk3Y17bECv3GDfVFRrj4XV5wgQDSHQvb5pjNzXHgurQwBjWgP8iY94DfjwxfPsKOj5AI+Cai/yu7YIy79hnQfsmVbuEoA2xQt0r988k9+Hw9A2Reiw3w9YOwkgahAthyv3+USsTWiHa4nrH01hgYwFQAeLZywfC2QcAGzw+kH7fCyQMQAYyFTj87FAhg/ATTv6Qft8LJChA+jw+D2WjwUybAClRVc/aJ+PBTJoAMbjxwkulo8FMmAAWidWh/pYIIMFsHvIrx+0z8cCGSiA+hmXftA+HwtkkACa3kjTD9rnY4EMEUDj+5j6Qft8LBACgH0skAEC2Pch+NB8LJABfkVpgLvUoTgCYB8LZHgAevsoMZaPBUIAsI8FQgCwjwUyOACe/AcQtM/HAiEA2McCIQDYxwIhANjHAhkaAKMf6KE+FggBwD4WCAHAPhbIwAAM+QIsn48FQgCwjwUyLADQN1W0fCyQQQEY9iWIPh8LZEgATL/BxfKxQAYEYOjXgPp8LJDhADB9BxfLxwIhANjHAhkMgOFfBe3zsUDiA8i/FvzdE9GL9PaTvd+UJ4BIqpYHkkodNy9KD3s7RgCRVCkNIUU6Pvz5Velhb8cIIJIqxVFkKNgpSSaipL1e0P26GKik6B0vlwdKb9/9ujwKfK8s4/E7fOXG8rG8Oh0Bb/IiHfk64OsYAcRSeQ24eb51CcCnDo6C8vJA2cQjw+DDX/Ycho7hy3g+H8urq/OAtF5QNhPZ84Cnr7aPejpGAH3J0zEC6EuejhFAX2rumGnqWNfBxfKxQAgA9rFACAD2sUAIAPaxQAYBYBw1IXw+FggBwD4WCAHAPhYIAcA+FggOYH1xchXeuqljIylL4/Ox+GKMgLkxZ7eBbZs6RgCw1hfGTINaNnWMACJIEJxe72/X1DECgLUw5txORQETUUMHjKdjXQcXy8eiiwBgMzPmUu4sA4ZAQwcIANP6ImTq2aqhAwSAaX0R/PJPCKCmvgF0sQejFtr9RV7L/zyoff0VNJrylD4fCzDaCAhUvQME0KvqHSAARPblL6dgoiMX4d3aEPqBHupjCaqPAALoV7UOEACq9UXY23CZah0gAFhyJBp8ILTbgREV6fb5WHpxpqC5MYGfyux2gACwpxeaH3cURADY053moUehBLCrCADC55+kBmCIv4x3qI+lp/xWBAEonwcQgPJbEQTAEQD7WCC6n4iN6sd6fD4WHwHAPhaf7idiBKB8GEoAXdYL2hYKKoyEAGqKfhhaqheU1SeoFhCqdoAAuqwV4UqkVAoIEcCuOqyW4goFFUatXtA9vyQoVYQMsmloexRa1AtyhYLKBYR2RkDTjzXov6IP9bH0Yrwbep4sTq8X7jC0UjFL1oGqUekAAUQ6DN3MzvMTseqUbwG0rAEEEAmAfC6/zI+CtvWCXKGgUgGhhABqwgFsZtPlyZVMRJmKekGuUJD/PIAAoqwBq4k5D/p2jKjSAQLQfTt6ZD/c7POxQAgA9rFA4kxBx30iRgBJnEU47I3oTOUOEECi+3Y0ASRxRgABAIqwBhz09bxyBwggiTMFHXtZCgEkPAyN4GOBEADsY4HEALAw5nJxxFsRBJDE+Tzg7N/ZO9IhKneAAJJob0dfHnNhFgEkBBDBx+KL8SU9mYJCvytZ7gABJHEW4aWcBgR+V7XUgcb8BxDooT4WnuKVIbwoRaR4HsARIIIBLE6uDqhaSQC7QgHI5/FyDnDEIkwAIhDAZjaVj8Qu5Vgo6FS41AECEOFXR2ej4IjzAAIQxQCQvvgJ4DjBU9Cl+1A48MKgUgcIQAR/R+zsNl0CloEVa0odIAARfBg6lyPQzSy0WkSpAwQg4okY7GOBEADsY4EQAOxjgRAA7GOBEADsY4EQAOxjgRAA7GOBEADsY4EQAOxjgRAA7GOBEADsY4F0Wa7m3a+kVsS2ak2mUgcIQBQdQFGdRuoTSL2IrGqNU6kDBCDqsFzNG8Fw88JVrXEqdrs5/wEEeqiP5dVhuRqRveeq1iS75Wp4WVCq6DFUqtNInQhXtcY9XLxuOAJSdToC3n/23Ln5OlDsNgGk6nANsEdB+fJLAD51cBS0rU7j8ndVa9zDxW4TQKquzgPsIJDjf1l+XdWaTMVuE0AqvTNhAkhFALCPBUIAsI8FQgCwjwVCALCPBUIAsI8FQgCwjwVCALCPBUIAsI8FQgCwjwVCALCPBUIAsI8FQgCwjwVCALCPBUIAsI8FQgCwjwVCALCPBUIAsI8Fond1Dq8LSsURAPtYIAQA+1ggBAD7WCAEAPtYIAQA+1ggBAD7WCAEAPtYIAQA+1ggBAD7WCAEAPtYIAQA+1ggBAD7WCAEAPtYIAQA+1ggBAD7WCAEAPtYIGoAPPkPINBDfSwQAoB9LBACgH0sEAKAfSyQLusFuXuFkRBATR3WC3L3CkOU7zYBZOqwVoS7VyoekRBATR1WS3H3CqNaL4hK1WG9IHevUkAoaXgFab1yY/lYXn2OANGAgovlY3mprQH6wcXysbw6rBfk7hWGaEDBxfKxvDqsF9R+HqAfXCwfi0uxcq52cLF8LBACgH0sEAKAfSwQAoB9LBC176kcek489vY+EUBP7X0igJ7a+0QAPbX3id9VVBYBKIsAlEUAyiIAZfULoP2Kibb2Tb/I1NbeNQzdfqnSe9v2k/yTjbD9D1CvAPZcMdHSvvEXmVrau4bB2xfJZ6ft27dtHJ6w/Q9RrwD2fFrW0r7xF5la2ruGwdtPMsZ7tp/cPP0yax62/yHqFcCez4tb2osqv8i0p71reMj25ZW8Z/tJPgWF7X+IegWw54qJlvZJ0y8ytbR3DQ/Yfnq7Z/tJDiBs/0M0mhHQ8ItMre3Thgds/02+mLauA+MeAcevAY2/yNTWPmt4wBpw87x4WksX3o16DdhzxURL+8ZfZGpp7xoGb99NPHu2n+QAwvY/RBrnAb4rJlraN/4iU9v2XcPQ7W8nkj3bzwCE73+AeCasLAJQFgEoiwCURQDKIgBljQ3A+idXNW9hjDmp27lWD69sm5PPH16VnbdfdbKDh2psAOb1pBen10myNJdtT1tf7D68etiCrEeNC8D6ov5S38zSbOdnty1PrMdNAMdoeV7PbTObunurh38wRjhsZsbIsJDbc/E/n9g78tSKM53bu8nivOdOVDUuAI0v3KWNMntscnq9mZ0n8i9ZnN3Krcw99jnyNPtvx1laSm4AqekOAMhWYXldTy7TBpKr5LxtWwDYcVIWj6773P+a7gSARPI+u00fs6EKD6tpCiIpA9hxZPpRnoHGC0BSnlYfyAG49Xg/gNWjvyvPQOMFsOtIpukU9Oh6eXJVaeudguwC8DPlGegOAMhODWR5LRZhOwQshe2CXF2EnZOdGSyM8gx0FwCks5FMR+lhqAQqB5sZFXfQuQVQcpJ5vm6ramwA2nTEuZX2MdC9B7CY7m/Tre41gNWk9f2LXnSXAIxSBKAsAlAWASiLAJRFAMoiAGX9H0yjc1Ffc5t0AAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
</div>
<div id="midpoints" class="section level2">
<h2>Midpoints</h2>
<p>So far - which is the default in <code>cutpointr</code> - we have considered all unique values of the predictor as possible cutpoints. An alternative could be to use a sequence of equidistant values instead, for example in the case of the <code>suicide</code> data all integers in <span class="math inline">\([0, 10]\)</span>. However, with very sparse data and small intervals between the candidate cutpoints (i.e. a ‘dense’ sequence like <code>seq(0, 10, by = 0.01)</code>) this leads to the uninformative evaluation of large ranges of cutpoints that all result in the same metric value. A more elegant alternative, not only for the case of sparse data, that is supported by <strong>cutpointr</strong> is the use of a mean value of the optimal cutpoint and the next highest (if <code>direction = ">="</code>) or the next lowest (if <code>direction = "<="</code>) predictor value in the data. The result is an optimal cutpoint that is equal to the cutpoint that would be obtained using an infinitely dense sequence of candidate cutpoints and is thus usually more efficient computationally. This behavior can be activated by setting <code>use_midpoints = TRUE</code>, which is the default. If we use this setting, we obtain an optimal cutpoint of 1.5 for the complete sample on the <code>suicide</code> data instead of 2 when maximizing the sum of sensitivity and specificity.</p>
<p>Assume the following small data set:</p>
<div class="sourceCode" id="cb53"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb53-1"><a href="#cb53-1"></a>dat <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">outcome =</span> <span class="kw">c</span>(<span class="st">"neg"</span>, <span class="st">"neg"</span>, <span class="st">"neg"</span>, <span class="st">"pos"</span>, <span class="st">"pos"</span>, <span class="st">"pos"</span>, <span class="st">"pos"</span>),</span>
<span id="cb53-2"><a href="#cb53-2"></a> <span class="dt">pred =</span> <span class="kw">c</span>(<span class="dv">1</span>, <span class="dv">2</span>, <span class="dv">3</span>, <span class="dv">8</span>, <span class="dv">11</span>, <span class="dv">11</span>, <span class="dv">12</span>))</span></code></pre></div>
<p>Since the distance of the optimal cutpoint (8) to the next lowest observation (3) is rather large we arrive at a range of possible cutpoints that all maximize the metric. In the case of this kind of sparseness it might for example be desirable to classify a new observation with a predictor value of 4 as belonging to the negative class. If <code>use_midpoints</code> is set to <code>TRUE</code>, the mean of the optimal cutpoint and the next lowest observation is returned as the optimal cutpoint, if direction is <code>>=</code>. The mean of the optimal cutpoint and the next highest observation is returned as the optimal cutpoint, if <code>direction = "<="</code>.</p>
<div class="sourceCode" id="cb54"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb54-1"><a href="#cb54-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(dat, <span class="dt">x =</span> pred, <span class="dt">class =</span> outcome, <span class="dt">use_midpoints =</span> <span class="ot">TRUE</span>)</span></code></pre></div>
<pre><code>## Assuming the positive class is pos</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<div class="sourceCode" id="cb57"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb57-1"><a href="#cb57-1"></a><span class="kw">plot_x</span>(opt_cut)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABaFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYZGT8ZGWIZP2IZP4EZYoEZYp8aGhozMzM6AAA6ADo6AGY6OgA6OmY6OpA6ZmY6ZpA6kLY6kNs/GRk/GWI/P4E/gb1NTU1NTW5NTY5NbqtNjshZWVliGRliGT9iGWJiPz9iYj9in9lmAABmADpmOgBmOmZmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq+SBPxmBPz+BYhmBgT+BvdmOTU2OTW6OTY6OyP+QOgCQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+fYhmf2dmrbk2rbm6rbo6ryKur5P+2ZgC2kDq2tpC2ttu225C229u22/+2/9u2//+9gT+92dnIjk3I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrbtmbbtpDb29vb/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///92SWBlAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAUjUlEQVR4nO3ci18b15nGceHEpDesuNvu0m1l403ThngvDbHNthvbos0mISEJbNs4bCBxTbut4QCpAM2/v3PO3DUaaUYPus7v/XxsiUdzOX711ZkZWaLhUZRQjWkPgJrvAhAlFYAoqQBESQUgSioAUVIBiJIKQJRUIwO62rr5Mp8tV95KwRrn/7pfaQAX68t9c2rMNauAjm4AaC5qfgFlC0BTKgmQ/+f3P2ws/Yf9+c/NxqtfOA7f3m80/uGZfU5XT/1w20tlg9eIH73aajQaq24/gSTn488/bLhVfXVHjRtfOChR5i9gd/YsAhRtkhpziYAarjb859nevnrff5rPm/bu0rb/nP64GdxLssFrxI+mAJ03/WW9U3+No2jVq61Xmo2bf7NQ4uxi/ZVmtBE/jzdJjblUQMv2SVz2n7+lD71v/R/9fOkdz/tL02b+o1f/k8kGr5E8mhzCggll78b+1daNZ9aTW2w1GkCU+Tv7p2hnNo82SY25RED2abaHl/OmnS/sMxncswAu1t3JiP/cJ9ngNZJHU+dAR24ycxT+7/e/bToi9sHwXCfMgp3ZR2yebHLkxlDlSj0HehmcCNtDTHQvqKXt8Fn3ASTZ4DWSR1PPvcXgFg4OS+Ec46UPVQ7QargzmyeblBtEDa7ZB2RDfxazB6kf//cf/rqeBpRkAJpSXROg4JgRHJo2wiWSo0qSDV6jLyAfxUfry9HZdHAOFAFKskBrcgiLNkmNua4JkH9K/I47B7anxB/6V9Fb9hwoOa+NssFrpAElk8d589+CI5h/gu1fnSe7tVCizN/Zqpc+iQ43eT1dogrrmgAFxyF3UR4ePlajK2v7JMbZ4DXSgKLLePtzIzhpDhZLA0qy8D0Df0H3aLxJasx1XYC8v9xvvPqlu3d+v9F45Z3ovb1/fOalssFrpB69uG+v6INy1/XBe4OvfriXTFQ3XyZZ8K7lj55FG4k2SY25xvi/8cn/LlCLWwCipAIQJRWAKKn4RCIlFYAoqQBESQUgSioAUVIBiJIKQJRUAKKkuj5A3z7zzl/v/wGuotxfZdCifR6O6zT1SQ3/fr/V/Cy34+JBFo1x8EPUtQEa1OXqz9nQh3sAFazWu4nRlABoQAFIeogaHZD9MJf9pOnr79sPdNlPt6+6g8YH7l6zEX8OfjV+BuJV7FfFwgWD1dOPBY9suC261eKtfBBs9WK9sfR+iCa8bzfpFgtWO7/9u8aNP9pNBVvP7nE72Vm4RS/+d3h79j/wjpZ7xxuOIdxJclv7Ej5Qthx+ttl+wms56LL/lDXd1/3sX/ZTXhvu080hoOCTrBvR07kdPDHB6vHmok0k5zXxVprRVu2nHQNA0f3ofOe8uRFsNbTYtJ96W87u8fXtZOzBFu2WokVP3YcaN3rGm/mXhDvx+OS1JwByx43T8APzyXMT/Bhmf7efKUzOZSMRWUDBovHmok0kZ8DxVtKLhk97dN8+fHs/3nq8ifSmkkH2jt2tFt53YoJNpcYbjyHayW0+bR3U6IDcB0fD58bvegQofKrCv07dd2vCZyI6V+l5Ot3qqc2lHw73lWzF/+vILRo8g9F9+9ieO9548Vaj5zscXBpQeuwRoHBRe/gKj2CZ8QZjiHYS39a+xgroYn1pO0VhRECZrQwA5E6IwtMhAdD57S+DI1h6vNEYop2kbmteowOy37uJDwO39/sBck/U6dLwQ9jt/XhzeUCZraQOQO6x5Mjnlk0k5w5daUDJzhJA0f2rrX9PHQzD22gM0U4yt7WuazuJDs82s4Dsy7YZA4pWsWe+V1tLwelGn5PoeP4KJ4L0Vly+nDqJXo5Oop2l6CQmAhOeGWf2mDqJTgMKF42/BOKlxxSNIdpJdKs1fxHqei7jbcf3Gsu5cyD/TGHpo/AIkqziroF/9fZ2sEq4evoyPl47eCbTW7F/FVzGh2cp4UDCTb3fZ4/bPe8nhDNQuGjmm7LxYuEY4p1Et7Uv+Y1E8WU4g69irrCqFIBydcT3WSsUgHrKvblIlS4+zkFJBSBKKgBRUgGIkgpAlFQAoqQaGdCLuhX9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yBSAqhb9yNR8Afrmp79ZWXnrxYvnb6x85xN3891fvDfREcwWoOdv/tfKa+/F/fh6xd1MtOYM0E9+9uLr73314uOfvfjfH7ywN1+/VmtAb3zvq699Mx+7tjx/8xPXlonWnAH650/sn+c/f89/8X1iG/anX9cb0Fu2A7YRfk9sWyZe8wnojZUVf+r+5qdf1R2QJfPxW7YtthHf/GRlahPyfAF60x3pmYEsoGQGsok9wE+05hJQeMznHOj5Gz+IG+GfDfn3ADSwIkD+MSy8+Ph+zWegX6avwj7mKqxyWU6TrVnqx1ROm7M1z4D+9J8rkz9pBFC25hnQdIp+ZApAVYt+ZApAVYt+ZApAVYt+ZEoGZDzTt643boxx2xXjoQ2ZwCBmsB0AKhsDyABIiQFkAKTEADIAUmIAGQApMYAMgJQYQAZASgwgAyAlBpABkBIDyABIiQFkAKTEADIAUmIAmcqAOg8P3e3l49baSXxTc0Cdd1utdrotACqqs9ZdB6i70/aO70U3NQd0+WTX6zzYTdoCoKI6uPN5MANdPj20k1F4U3NAZ5bMQTtpC4CKK2xP59GJfeGFN35wy6/UYv/SW32SXDRov0nN5u9BD7pQ2I/x1Vjb0edJU4cUAjpbc50Kb8LHUhhze+6T5IZXDvoMvuTssWsz1ZYoncQgxtqO/JM2aOkqgHIzUL0BXT7e9FJtAdBQQMPOgeoFqPNuO4S0YOdA4wNkp2x3FbbZ/yqsVoBiPz39AFABIPtnyPtAtQJ03LLVTrUFQCNVXQEVN2QCgwAQgKQYQACSYgABSIoBBCApBhCApBhAAJJiAAFIigEEICkGEICkGEAAkmIAAUiKAQQgKQYQgKQYQACSYgABSIoBBCApBhCApBhAAJJiAAFIigEEICkGEICkGEAAkmIAAUiKAQQgKQYQgKQYQACSYgABSIoBBCApBhCApBhAAJJiAAFIigEEICkGEICkGEAAkuJFApSq3J4X/ReNT63m7ReNDypmoN6GTGAQizQDAQhAAKoSA8gASIkBZACkxAAyAFJiABkAKTGADICUGEAGQEoMIAMgJQaQAZASA8gASIkBZACkxAAyAFJiABkAKTGADICUGEAGQEoMIAMgJQaQAZASA8gASIkBZACkxAAyAFJiABkAKTGADICUGEAGQEoMIAMgJQaQAZASA8gASIkBZACkxAAyAFJiABkAKTGADICUGEAGQEoMIAMgJQaQAZASA8gASIkBZCoCunzcWjuxd45bttru9u4hgDoPgyZk+gGg3uru+GTuRT+d+ZYO2sxAthWRmkw/AJSbgJ4exq817/LJrtf9dBdAPps7nwddyfYDQL3VeXTi3LiyU5F/SLMHMr9u+ZVaMrfnBf890eHLqrAf46u5+j3R9qAVAXK3nQfpWai2M1AMKNsPZqDeSs9AZ8HZtJc67gMo2w8A9Vb6HOhgM9cwAAFoMKDuzmZ0FRZM1HYa6n7GZXwEKNsPAOWnIPc+kO1WeCQ7brXuJIf8egOyfzL9AFDFqjGggoZMYBAAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFK8SIBSldvzgv+i8enVXP2i8SHFDNTbkAkMYpFmIAABCEBVYgAZACkxgAyAlBhABkBKDCADICUGkAGQEgPIAEiJAWQApMQAMgBSYgAZACkxgAyAlBhABkBKDCADICUGkAGQEgPIAEiJAWQApMQAMgBSYgAZACkxgAyAlBhABkBKDCAzDNDF+oa9Ob2xD6B8DCADICUGkBkM6KgR1XIJPwACUA+geAYqVwACUC+gSgUgAOUAnTfdIYxzoH4xgMwwQFdbpc5+AAQgzoGqxwAywwBdbQGoOAaQGQao5DtAAAJQ0SGswUl0YQwgMwxQpQIQgABUJQaQGQaIQ9igGEBmGKCQ0dvbJfwACEAFgLzTmy/d7eXj1tqJu3fcarXuHqaCegPqPDzsaRCAMoCCQ1h3p+0d33PJQbsnqDWgM/tiyvUDQHHtBTPQ5dPD8LXW/XQ3G9Qa0MGdz4MmZPsBoPgkeik4B+o8OvEun+wGU3Wr1U4Ft/xKrZfb84L/ovFQTWE/xlfl2tGn+yOuVmJDxUM6W4v603mwa2ehJLBV2xkoBpTtxwzNQPnul9pln9XyT2Oy3lBAyQvM1UE7GwCop0GTGMTMA3Ifa10N7vcc4g/anANlAc3sOdD0AB3Z66+L9UBQd2czvMiwc3X3s8MkAFC2QQAKqudbGcHbHLZbx63WnV2P94FSgOyfWX0faFYADa4aAypoyAQGMeOAsocwAGVjAFU8iQZQNgZQxct4AGVjAAFIigE0HNDV1mrp7/YACEA5QHvLXulvhwEIQL2AuIwfFAMIQFIMIN4HkmIAlbgKO+V9oMIYQFzGSzGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQD6HoBpSq35wX/PdHTq7n6PdFDihmotyETGMQizUAAAhCAqsQAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMoKqALh+31k7cvc67rVbb845brdbdw7oDStqS6QeAequ70/aO77mWPdn1Og92vYM2M1CqLdl+ACg3AT099DoP7Qvs7J7rVvfTXQCl2pLtB4B6q/PoxM09Yd+e7PpztzuSed4tv1JL5va8yL9oPGlLYT/GV3P1i8bP1lKAujub7iiWvOrqOgMlbcn2gxmot9Iz0OXjzTCNj/t1BZSdmFPnQZMYxFwBSg72/lVY3KbaA0q1BUADAdmjVnC5Efqxk3f3s7pfxidtyfYDQPkpyL3h4b/a7Psd9nTRv72THPJrCihpS7YfAKpYtQVU2JAJDAJAAJJiAAFIigEEICkGEICkGEAAkmIAAUiKAQQgKQYQgKQYQACSYgABSIoBBCApBhCApBhAAJJiAAFIigEEICkGEICkGEAAkmIAAUiKAQQgKQYQgKQYQACSYgABSIoBBCApBhCApBhAAJJiAAFIigEEICkGEICkGEAAkmIAAUiKAQQgKQYQgKQYQACS4kUClKrcnhf590RPtebq90QPKWag3oZMYBCLNAMBCEAAqhIDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgAAkxQACkBQDCEBSDCAASTGAACTFAAKQFAMIQFIMIABJMYAAJMUAApAUAwhAUgwgAEkxgKoCunzcWjtJ30uCOgPKtQVA/au70/aO76XuJUGdAeXaAqCiCejpodd5eJjcS4I6A8q1BUAF1Xl04l0+2U3uJcEtvwrXu9aavV80nmuLN8F+zF47BgzpbC3qT3gvCWxN5nU/ezNQri1RvyYxiNlrx2gzUJ0B9ZuBANSvyp8DjXGcs9exOTgHmkg8HFB3ZzO+3NgMrsI2+1+FjXGcs9exXFsAVDgFubc57GtsyPtAYxznDHYs1xYAjVS1BVTYkAkMYgbbAaCyMYAMgJQYQAZASgwgAyAlBpABkBIDyABIiQFkAKTEADIAUmIAGQApMYDMtQLyqn0CZuGXnolBTGFpAF3T0jMxiCksDaBrWnomBjGFpQF0TUvPxCCmsPQMfsqWmqcCECUVgCipAERJBSBKquqAir4b3r8677ZabXfvuNVq3T0cvHSyTIlt24XDjZfYtvv+ROmxu6VLjb3u7agMqPC74X3Lfm+q88B9d+qgPXzj8TJltm3rLPhnD9/2mf0Xlx67W7rU2GvfjsqACr8X1X/X96L9dT/dHbikl1mmzLY9L/pe3/BtH9z53H6NouTYg6VLjb327agMqPCbmYUVLONPktEMWLxkvEzJbYevmjLbtv0pP/aom0PHXvt2VAZU+N3worLfv7Nj8Ge/YS+NZJly246WKLVt+yIqPfawY8PHXvt2jH0Guny8mfxQ+sBfbttn6RO/Idse5SVXYuy1b8eYz4H8U/f0P6R0x8od9A82y2+7U+GgH192DB977dsxwlVYwXfD++853ql9eXQ/G3KOGS9TZtvJNFpm27YH5cfuXqBlxl77doz6PlD+u+F9K3xzwi7t370zbBIOlim57Wj6Lbft1BsfJbYfbrTE2OveDt6JpqQCECUVgCipAERJBSBKKgBRUgFopDp/fXvaQ5iRAtBIBaCoADRSASgqAJWqq61V/++jG/vnzUajsWoBOUP2r6utRuPG/rRHOK0CULk6uvnSKrpY3wgcJYCutpaDh+tZACpXlov/5+8vg/spQKd29nGwalkAKlf2GOammVP/ELaUBnTUcLU67RFOqQBUsk5v/s03dLG+tN0zA9X36OUKQCXr4u0Pbu/7jOwklJmB/J+mPbZpFoDK1l5j2dnxzpsO0MX6qn9gW/JPon1U9VUEoLJ12rDnyXv+GdBH6xvunNq/ov/V28FlfG39AIjSCkCUVACipAIQJRWAKKkAREkFIEoqAFFSAYiSCkCUVP8P7P6udIOIq4sAAAAASUVORK5CYII=" style="display: block; margin: auto;" /></p>
<p>A simulation demonstrates more clearly that setting <code>use_midpoints = TRUE</code> avoids biasing the cutpoints. To simulate the bias of the metric functions, the predictor values of both classes were drawn from normal distributions with constant standard deviations of 10, a constant mean of the negative class of 100 and higher mean values of the positive class that are selected in such a way that optimal Youden-Index values of 0.2, 0.4, 0.6, and 0.8 result in the population. Samples of 9 different sizes were drawn and the cutpoints that maximize the Youden-Index were estimated. The simulation was repeated 10000 times. As can be seen by the mean error, <code>use_midpoints = TRUE</code> eliminates the bias that is introduced by otherwise selecting the value of an observation as the optimal cutpoint. If <code>direction = ">="</code>, as in this case, the observation that represents the optimal cutpoint is the highest possible cutpoint that leads to the optimal metric value and thus the biases are positive. The methods <code>oc_youden_normal</code> and <code>oc_youden_kernel</code> are always unbiased, as they don’t select a cutpoint based on the ROC-curve or the function of metric values per cutpoint.</p>
<pre><code>##
## Attaching package: 'dplyr'</code></pre>
<pre><code>## The following objects are masked from 'package:stats':
##
## filter, lag</code></pre>
<pre><code>## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union</code></pre>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAAB8lBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZpAAZrYAtusAwJQZGT8ZGWIZP4EZYp8aGhozMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kJA6kLY6kNs/GRk/GT8/GWI/P2I/P4E/gb1NTU1NTWRNTW5NTY5NaX9NaatNbqtNjrVNjshTTYNTtABZTU1eaatiGRliGT9iGWJiPxliP4Fin9lkTY5mAABmADpmAGZmOgBmOjpmOmZmOpBmZjpmZmZmkJBmkLZmkNtmtrZmtttmtv9pTY5uTU1uTW5uTXluTY5uaX9ubqtuq+RvTW95TY6BPxmBPz+BYhmBvb2BvdmOTU2OTVmOTV6OTW6OTW+OTY6OyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQtraQttuQ27aQ2/+fYhmf2dmliv+raU2rbk2rbm6rbo6ryKur5P+2ZgC2Zjq2Zma2kDq2kGa2tpC2ttu225C229u22/+2/7a2/9u2//+9gT+92dnEmgDIjk3I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrbkGbbtmbbtpDb27bb29vb2//b/7bb/9vb///kq27k///r6+vy8vL4dm37Ydf/tmb/yIP/yI7/25D/27b/29v/5I7/5Kv//7b//8j//9v//+T///9kbTulAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAgAElEQVR4nO2dj4PkxpXXezdeRoKcb9Yhv7jJHZfBgRAEyUFYxwd3Ry6DDSQQwIaQ3eSICWEvjZN4QzhyJuv0OrZ3zjnI7VrpHjyBnZ3V/0m9+vnql1Rqdasl9fvaO63Wjyq91x+9+qFSaVaRSB002/UJkMYtAojUSQQQqZMIIFInEUCkTiKASJ1EAJE6iQAiddIaAJ3OuD72elVd3rr2IOWQH85mB/aOcGTz0ef/6G5NHpe3DuoOTj25tjLpxk7AyRmsaJD06ewq3/NsdiLWXtUHXv73D81mH/gnD9C+s1rr+9L6AIGxib/RmW9tGkCntXmMBSDEQUw2QHekt8yBl7fEdkh3/ABxsy5/KC+TBJ35u44YoOQTUEoBCO9yfvhXr9y2157NrrGA/6tb4MiE5HrU2gAxM4/kb/Qui65XvshWXf4RW/iS3vFXz89mT70OF5S6tMy+FkAsqXcP2dqzw9nHHsgDP/y6uOyO2F6/eF6kr1Nk6RzOnvpT+P1wniLB09kR/23dAz8sytw/+ZBcpw+4tNNR+xrZ5yeNlidwceOIrX7qNj49aZzISVhh++bihhVxHChOr/6Y+dZaeyqIurhxMBmALn/EbOK+lCH1RMXZE7kf8ysT2wsBhPa1APoo2/fKtw9FXD6XByqAnnpeJqtTFOk89fyBkyf38x2RBj5QpaiLArm/Aciko/c1ss/PGA0ncHEDtvH9z0wuYJzMSVmB860FiFEiPYMj0JHj/6GoQx3oSF1qV18HHx+wfx9/wBe4mMu+BDszV5yZH0zt6wDEdn0XUjw/hNVwpf788EAVYZDEn/FfTqV4cePKdyCgu3meH57Aj8OOOuP86QNVimzdAfz6+hwVQDodlLuWe374BFh2Bw8uf2SdngRI5QRW2OcZ8ylHA7wlPIZQ+eHsykf/8/+2/G8xvjN1AOjjKppX/+tP/uMh/zE/8Ad/qneT3rrD7MR1ILmvA9ABXHjXRILnJn4LgMCNELxNimIXAS3Ok+91/RWIepxsfaBK0azjwgDJdM7d0sM7P3OOsJ6vZid19a45PXld3UVljn2eMZ/yTFlS8gzxSfz5Hx3KMnD8AAmz3lXRXMR8uNSgrOLNexBEA6YzCyC9r1sHUr8q++HPjINMJVr8yCpFkSCvu1h58sOv/s8bJ6xmgg40KaJKT1VZdSCVzlng57HPDw4yJyBhPAWq1ekZ43jS3GHWedYVYdJFsMIprC5//rsQ9sdfhEkD7sAFDdF89tH/9D9+wf348+eNU0IAmX03BpCVJ2y7w2oQB7BrO4BUOtsCyDrPOoBQDcFDRVS/JgaQ8Jsu3v/8m5KWUBFm9q0BSP4QKisMkF2E6ZJI58l2+cNbR9XptR+FyKtsYqQNTjoody0fIHMCogiDAsstwhyArPOM+VR3PvBk9VpZTorznQpA786UL1llkTVg4bL8jQeidQYKVaLNvjUAsWrsd6DPA3ylWnqqpWQq0V/iFeLKzpOt+8B1BuyVDx3YTSyVogfQ7ItOOih3LR8gcwIsmPxGuBJtAOLXkHWeEZ9Wpr11RwZgdZ684//dw0lEIBljdWNcfNNLqvvONLpRK8zsGwVIlThHPKsj9EvgFGdWM153GZ7OeI0VMsQHqhRdgALpmNy1fIDMgRc3PnAoyyKnGW8AMs34WNcmQkUydsaQNKWZKvN0FwYu/3aq9QH68G35G/Hus+9AcOWdZR/XPjpX3X6mEq33rQOIH/gBaHBcPD87+Av0m+sUoUbx1E90B6DJE/wuG26XzoGQogtQ9WeHs4/9hZ2Ozl0rAJA+AbYa+hdfxwY7AIEVD5zzdH2qYEC9IFfv4g4TuBd2hWczeoBIWLoCtacigDqKACKFJftjnBa9JwKIFBYBlCQCiNRJBBCpkwggUicRQKROIoBInUQAkTqJACJ1EgFE6qRtAnR+/Tb73117dvUuXvur1+M7/+p1+/sZun2Il939hC6e48ndmYl74LHPJrHE4cTkR4pOnSH5kcMcL6SmPjDtBCB7n+By6HsMoIjrxcCIU7gxfhD/TDCiJo+QLm6cJO2HkhwpPKDJAnRxg8cB/muyfWOfKUZE86g9os1+ewXQ+XV4PupEjA6DJ7Ce+dbs6o/5yqPzQ/No1ZEE6A5/vkBc6/CrviKKML7PyTk/SqTAdn5lBsUKd6fI5ojnIsad8kNxMjIfnobKU+vsQCYjKIp9cosg/7syI3USMmNxgtK667e10coJwgyRpdgGa4S1cht3A3KPOi/thaPzWOoj0BoAyWGVYqyoHKksVs7EFv67nHJO2P9nfHjyCRwKz0pc3FAb+BBp4cUD4dFDUaxgD6viRh2Kk0H5VGoZnygkcybgO4l9CosOKp3R+aExjX+HE5S/9vXbxmhhqjIDktHbZDhBJlruQeZhCwKpr/uj9ql1ADrh9vOfAGrE/Lv5c/32/+Xj+riHpJPOnxGzTshnVWDDM/LZDQ7QiQToxJR78tP6waVP1TLKp1LL+EQRQEexT22RysgyT4YqA5BttDFDnxZqIiATLfcg87AFgdRb/zY70DpFmLD/jA8+Nf7Af874IAi5kRVfsgQ75Yc8w118RwR66Trza6jiRR0sB3WqQ3EyKJ9KLbsnmhSBrpuRxkeqbMPf0U/sGn3HTJJhbQMZEy33WJ/GgqBLh69tAHRx44pyOvfMMz8RJZgFUCWeQW4ESDa1QwChfCq1HDjR5joQB0iNz1YAoe81AEkzYE8PIGNiDCBswf4BdOW2idjWH+6HMx2BLm/9gQzmbpA3JYR/aaoiTD2kIA/FyyifSi27J5rSChOhQB4rio9n7urvNkCe0ZXm0NqmJEyMAYQtiKU+cK0PkFVntAGCy+pQA2TmMhAP4nEH8d/uXNaQlEdlJVo8l3yFbxPP0MDyAapEi2WVD+ynlt0TFY9RHMQ/1Y4qI1OJVhmfoPN0jVZmQDJuJVptcwFS5iFPRVLv8Lv2pvUBqnSb06sDscL/yn9Bl55+2BM141WV5c5Mt7d5C5rXGlgj9g+fu80naoFcRIeO34yX+fD95PLlrSPrRHVNJPqpdpQZ6ZNQGcsTlB+u0bjmhbeB5DYXIGUe8lQs9RGoj3thuKmy/cy+2LxP7fHj+N0Goz4AOj1q3mdjOuuYGQHUTtsHiPeK9af/1jHaEUDtRMM5SJ1EAJE6iQAidRIBROokAojUSQQQqZPaA3R/Wpq0VT2IAJqyVT2IAJqyVT2IAJqyVT2IAJqyVT2IAJqyVT2IAJqyVT2IAJqyVT2IAJqyVT1ocAC9/dmnP/hTvvTWJ55++vNbzq0vgIxV9++/89XerOpBQwMIvPvGr8HS25976f5bv/nSdrPrCSBjFdMb/V0WPWhoAL39O9+9/9ZvfZct/Qwc/sdb93XPVrHA+tf/PgG0Pb312z/lsUfILG1NfVv1zr/+t1SEbVE/+yAC6J2vfnK7ud3vCSBk1RufpDrQNoUj0Nuf3T4/fUcgtkQAbVOotvDWJ7bv6N7rQG88Ddr6hbFZSOo0NICg1BLtlX746asV9knTCqMItFWJHhN2uYprdRqtMGPVfQJo68b3q0lb1YMIoClb1YMIoClb1YMIoClb1YMIoClb1YMIoClb1YPaA1QqmaXab/5ilbJTcx6RVfWpVu6Wzla51gzJqh5UA9CjF46Pb/Klxy8ff/o9tTrJDQRQfHFfAHr8tdeqRy++xpaevHqzevgZtT7JDQRQfHFfAHofkLkHIejx19+sHv3zN+X6JDcQQPHFfQEIBFGIlWVffk8u/TpTL6dFGotqAXry6lfg4/1PK4BASdcRRaD44v5EoMcvc35QBAIluYEAii/uDUCPXhBtMKoD1SZNAEWk+eElGbXCYkkTQBE9PAbdhNBD/UA1SRNALZXkBgIovkgAJbiBAIovEkAJbiCA4osEUIIbCKD4IgGU4AYCKL5IACW4gQCKLxJACW4ggOKLBFCCGwig+CIBlOAGAii+SAAluIEAii8SQAluIIDiiwSQVFZjMwEUXySAhLLKIogASsq/MY/IqgkBJJXJf6T91toRKAPeM/9iSLhghnitJuVIEcjXugBlgp7MsyXB3iG6OilHAshXhwiUUQQKpDwoq3pQlyIsw7VoAigp/8Y8IqumBxC1wsIpD8qqHtSlH4gACqQ8KKt6UKeeaCrC/JQHZVUPIoA8q5LyJ4CkCCDPqqT8CSCpbjdTqRlPALU+Ap89AUQAtT7COnvqiSaA2so6ewKIAGor6+wJIAKoRmpOF5hn4VPh6V0y15YEe4fo6qQcCSBfdQC9r6i5dxOttc+eACKAYrr37A9EBHryvdfQavvsCSACKC5ZhD1++VjOGB2YZJNGJe63UgCC2aJNFHLwz0qX/sYLZojXalKOFIF8JVWiQboe5Jw9ART5jC4SQPZZZw1OIYC8hPYNIJgn+sn3Y7O0EkDhz+jingEE/x4eHz+rG2Lu2RNA4c/o4h4BFJR39lm9UwggLyECyDpdAij4GV0kgFxDslqnEEBeQgSQfboEUOgzukgAuYZMCqDWc44QQC0VMCSrc8q4AGr/tBsB1FIBQ6YDEEwZ0fJRAQKopQKGTAugliO9CaCWChmSTQSgdeYcIYBaKmTIVADic45kFIFaiACyraJKdEttaJbWbCoAtZ8yggBqqaCV0wGIWmHttPYkm7amNLB1SrZsX5uaaDyjCJSUf32q8VUTikBhKycEEPUDtREB5FtFALXQxt6VEb0JOT6AqCOxhQiggFUEULo297aeWP8JAeQlRACF3DAhgFo9sE0AtZQ+ycK2kABKyb8+1fiqPQAo1n8yRoDaPG9LALWUPkkCKJTyoKzqQesCVEghWwighPzrU42vmhxAJTDkWBjugBslQC2etyWAWkqfJAEUSnlQVvWgTnUgTBABlJR/farxVeMESE3v8vjl40+/p1bqkyysejSsD3bAjROg9Ae2CaCo1CSbT169WT38jFqLz54AIoDim/Qkm4+//iaaa8o6+8L6NiWAkp+3JYDiUnMkfvm96vHXYIIgf5LNwvo2pcF8U7Jli0qdoUwABLLxL6xvoVsAFIG8hPY3AoGcsy/wt9EDlMOWXCwnPm9LAMX1qLEOVGqCxLdAD+6oACrzSvFDACUpBaAnr34l0grjKtC30QNU5rleTHtckgCKS02yGe4HCgEU6IAjgLzz3BuAgvINKdC3sQOEijACKEUbGZFYmG8jBwhXohOftyWAWipgCALI78EdFUBii6lGE0BNWhsg61otpgWQVsrTbgRQS6lzxLUFRtCkANIhiABqUociDLVXEEBeD+4YAdJ9iQRQkzYEUFlMCyDcG92Q9OgB+tXr1fn12/76s6t3E47uUoTZBKmFaQAkCdoDgACeXQC04jK2mIEdzi2AkQIkCEp42o0Aail5iqtVxf6ZGFRZIWgCAHGCpgXQ+fVvH85mR+fsz0lVXd6aza7ePedrrr8y0+sO2J4XN2ZXXtkyQAwf9ic3tiiCpgIQ2DYxgA6vPahOZ/Dn6t3LW4yU02sPeAQ6NOvg38WNI/Zv+wBBEZZrG3AImgRAvK+r8Wm3UQF0ov5cv81LqIsbJwIgtI794Z+nfRRhpW6MVSUOQdMACPq6JgUQVHXUn9MZ15GuAwFALBDxbfzzme0AJCUq0StYzNU6Nb51UqNBJ2SMDRBAUqFKdF8A2fiLICS/mRA0kQgEljU97TbWCHR25bYPEKxTRdi2+4HU6cpijH+TBE0HoCqfLECXt1iYYcSwahBapyrRB1uvRJvTFUFIfFME1Xp9iK6O5ogJmhRAvMkOEefO7MBa11MzHp+uKsYMQFWd14fo6niO+XQA2oY2M8Xdigd7ECcoGy1Abv86rDcEEUC+NjRHIjTqRXu+kG/dqvH6EF0tv+nOCbw7AVSjjU2yqYuxQr6yLf6AxhBdLb/J/nX7sCwv3aMIIKXNzdKquqUL3pGYjTQChQDSBBFAvjYHEA9CkiCoAymExgXQqvDqQBBMc+coAkhpkwBxgnJVDYqPbh2iq8WXQlSiC/cwSRAB5GujAJWiGNMzT2XhG/NDdLX+XhR+EVb7sBIB1FL1buBBCCa/kyP6Qp3SQ3S1+roslpggA1BeRpIeEUC/DGt9eqrNAwRBiBFUmIfzRgXQkqGPCVLrBUEEkK/1AVoul2ELV3ZvXOb1KQ7R1fIbB4gVYyvnsEze70Pj5yIpD8oqRz0DZKZUeHh8LKdLxK6uSkwQqkWsVnlhaqKVQWgMAFX8ulAEIYAYQRUawRtLeVBW7RQgNLXmvZtovTzFpROCkFkQf3KrFMisBtkQXa2+glGIIL1eEBRKmgCKyEwr9eR7r6H18hR5BFr6tjAxeop8lVu9b6hdP0hXWzsaghyAcirC0gEyE9uxsuz4mAchPMkmu1bhT+DIFR+uWBU5Xpll0L3Y6Vz7E5hVrKxVGQy8hGfhdnRKm1C/AJmpNR+9+BqKQg7+qiCzijDeI1cWOW77AkD2U4dDulbdHZciBpn1GW+F5aogowjUCJA9taapB/mGcIScOpC4pVSooYqiDMjsZ36G5GpvR2YUIwgBZLblOQGUAJA9tWYdQNzbAadAEFJDFUt0i3WIrg4cAwShFPCzJnnuHzBEq3YKkJlaEwqzJ9+PzdIqFO4VYgQVuuKZjQwgTpDplLYfVvKr00O0KgoQ+7m2DpDsB4Ig9PD4+FldlkXdEGqTQTGmnc3v0ftmxxd3DBCzyHQpeo8K5M4YjyFaFQNoKf/ZAF3cmF29W11+45XZ7OiM/asuvvCt2ZXAU/OJAEVU4waDkN4GrTGNkKwHDdHVkQSXpSHIu61nV6eHaJUH0NITAujOUXUqnso4PzyAx8Iublx70PRwz4bvhYXaZPzefF6qVkw2SFfHcmRBSBGEhumq3fLczPQ3RKs8gOoi0MVzt1nMuXv5jdsV/GNf4YkfWOwRoFKEIVwjqnhdWviaezoboqvjOSKCAo8K5HqmvyFaFQMoWAdiJRg854MBeg6e+jnpGSDZJsOl2Yq/HlP34yZNZ9+QR3zVpgHSBEWeNYnVhaKLQwAo1Apj0Qc+HID6j0AlD0B2dUgWY/lwr9W6HAVBsWdNhmxVK4CgDnR27YFVhB1UZ/IR+n4BggikEOIbeL809CviWZuG5Or6HBlBvAGQ+dETijCrx31IVrUDCB5HvW1HoN/rtxWmJegRCMkNapRQbibdGZKrG3IUBMHIFHtggZou2/S4D8qqdgB54nWgBm0HIPVtabqoYYSQGCWUqykvhuTqphw5QYUYU5BZfYoyJuUE0BYAKlGDjD/uIMZIMISyXI80jvtnQABBPS4rVAGmAxFqmrmV6ejieABK0dYB0tWhVbniUYhvzPMsZ64PDK8ZLkDWQN1KBCLeq6UKtdx0dtWdCgGU4AbXSIYQPPS5hCgk5iRnBFUN0xY05BFZtUWAipU92p4xZLCCNaazq+ZUCKAEN/hGLsUoIfFDlO7Q+527ujHHgsfPVaGm9hNrGUAAkbnDl4fHvRJARkluCBiJrlX2mcHI+1y4PuqfIQEkI2hRFIWxRijDo1/D414JoA0IRr8ul3IQLAy7L1YwcnosY0ThtrCYVBQg0mt5IQajLYXyCormgVo1DICSrqPgBSMbZKKbepUtV8tslbkD/IYbgXCRuzSRKBNbRJ06L1XTgEMk97ar1vsegZLcUGtvxe8OyP+yzB7gN1iASvw+ELhNzyHSWHGGcKusFBAJmnDVmgBKcEMTQODwZSamUhEIhSoOQwPISU8yJIMRTG0mG2XW/Q4OkfViIwKo2Q0pAImZVARCVbAjbuAAlWLoPS/PxBpokIFFbtMgz4tI6yxoAgGUABCfCINXJjhCsEIE/N3UFpJyDKTHghCv0skwJJpqqqPa7J/nskBz5xnarlWOJgaQrFJwhNSUePmuagtJOYYA4rMxiNGhUCXKdDnGMFIFXJEXWS4q3aZe1IdVjhA0iykApH8EPnRdIZTlyMfoYPvmWTjbxqw3DpCIo2oN9DVmq5WgRlJUquJatttgV2OhaIFuyypHmJ/FWgD578/oDaC5lG82/xVUbbrMchWLsEpv1pgNu3pNq9yU7XNkUUmetTCw1PVtZKFdZm/WqikBVM75YhggtigRUmMmnF2Yo1cr/ifu2HjWfp7bAghFJLMq4/cAWdHGe8AKE4nKbV8WMYAWViGm4PjIN/lbC8XTPecffv7qT8wTPpV4zeFOAVIBKAwQj/9AUBF+vYm4DcX+2lWIwQFkdRbJT1G1Rvc/DESMqzzj18Y2rPIAWnhCAB3yIa1yZCu8gw4/4fPcbfjYGUACnkUZKcLkYqH6UYzQDaecTzsEa1AtaU1Xo9HxmwYosBO7Mipr8mBtVcmtEjfTuluVGoEWdj1aAcTgAFS+cBdGksE3NLoVdmBbNgrQosbVaJsOPPPFYg5f6129UrfsZeVhqUbo86eDBFxiU47vFrRyNS42OgO0cNaErbKKKbEe2m2idFuuDNARq+pfOtUWIBN9fIAkOR5Ad/grejcGEKoTuyePNsEH2lAJkObeIWpRNoOLlTUiH/8K/GdAwcltFefiKcZ6VzNKlzGA6i6LEECLahHIwVuMvLyXD7ErVnqUSIZaENgoGPfYZFWLOlCwH6g2AvG3ijUVYZe3TtIB4p6b+7aobV6Np2SrhK8ridDCPxouyMyfp5mn6iLrUCT8a13sEVfPV/MV+3+OXa1PsrIIagIo0LYMWFU6XPqXhWj460YbN81YtQpalXtT6HsF8zoAqTqQDRDfeP12PUBAGZKZZNMsGVfPmfMW87AAFf1DowobK8SkuXPkeeyU1WqR+cFeZrjILGSFz5wiDSqlIYAKtGq+QsWGfVnE4yo3xTsxdklU5UJfKBGrwAk4nWC0LkUrHzCyOMoypwEaKBj5qhzGbKv+sw4AyVaYU4SdzmZPPX/SUISd4ofIzCSbaLpNDBAHIqg53oSuzErSJFwHYcp1abi2IF29mC/sn1ftoIs00RUMTTXr6LxcZkuxiv06cLcKfo8Cu9rjwc3foUBuQJdQJb0SsMoNbM5l4VwVGiPNEURMCJmZVUESVpgEAzW7VgCtJzsCiTdBi/kYzART9lRT0iO8RhML12zbwtmgeII1AjIZwfBOfgRAoQpq33O8l13D4hjp0snuhISLc7mEL8DZPF6JnvPEFwu3ZuxRwE3ggbYqxcWCgm+jVSXasCj8QF5hjBhIogFqxaVcmVpIM+Eb/NshQLbMFHdmyUyyyQmA+OKLOzO8Ce+lwhQu+uQatNIObH7Is0rOBQOI14+zFZohMy8KdukWGfvg+LA182K+CI0ZXLCNlS6ZTS5io/jf5Ou7YVFv1cJBBQ6Z8/8dOTyxky44H/KbjEtL2QcgNBeDtuvmMd02QHYl2kyyaZZA+jqyA69dWwiXArixEApTxmWBg72Y52XLjpPelAviWi2ly+eAl/z5M52SZZXiwYocfP0CnZkV+WTTMskqZVe1QApdFrYzVRNNdj5qrHLeL6Y5he/ZjoswDFAoAlkAte4xwQCpMGVvUVdw6GB1EVv5uF1uC14iLBaluVbnEHbgXhSDSvw0Cw54CCDEg0ne/PiV6dTyr6M1rSrn/lXhWQXE62RVqmLskWqWwHfoiF1gq3oGyK5E19eBPCtbAoRdb+1k/4J1eYRXzavMjl8ivLP/9RAdVB3z+4GicVVe594Gt/+nlVWqbGu0qvBrUG4rbGiVaDPJplnaLEDBnQKNrLhfQ67OPFcvBEALtMrrcmvM0QkjcYC2YlUES7u/AFG2G4AcmUk2Q/1A2wKo7Ozq0LU6r/C1ug5ApR1GWgLU2ar6VCv760AAiijJDR0AshLazLXqBvt1AQqGgTSAulpVn2rMKkfbB+h0Njs5bZiSKskNOwQo6VpdC6BwOUIAGd259osbJzAIZHoANbk6Kcdg0gSQFmvGQ0u+YWbgJDcQQPFFAijBDQRQfHG6AFWnUIRd3DjqlCJpsELQFNupRJ9BNxDxM1XVA+QP1UhRj9O7kHat/gCSk5aTpqVmgMRgMvVxNsMfYRFAeyTGjScboDv6yZ7TA06B+YiIANojNUYgOaBevrdHPc1TO1s0AbRHagRIDoSW7+2Bp1HhTQfyI6z2AN2fliZtVVuAVAQyP79810r8lSsE0JStigP0yyBAqA7EPmDJfBBAYU3aqtYAmVbYFfUwqv4ggIKatFWtAFpP7TsSd+2bDWvSVjnaOkAwB8ysLl5N19W7Po0NazcANY0EmrKrd30aG9ZuAHKejSeAxqvdAJQ2O8eufbNhTdqqngFqGkrWi6vf/uzTH/wpX3rnq0//5Ze2nFtfABmr3vrE03/lu1vObXdF2Gznleh3vvr5+2/8Gl/848/f/5n0+hbVs1Vvf+6l+2/0ZlXPAKVpu5a//Tvfvf/Wb31XLvWgnq1667d/2odl4Z9uHwDiDv7cS3zp30ymCDNWTTsCDaAfCAotCdAnPs8dv2X1bBWqDW1TuwHo8tYRa4g1Nea3azmOQNrp21TfVv3mS/d/tv1a9G4AAnTk7didAYTqQP9gOgAZq1As2qZ2B9DpQVNjfruWv/PVT6JW2FSKMGPVpCNQdYfT0/Bw/JZNF3UEuFzZUn89JlvOxVj1s6d7bBpEAMrRXwTQnRP/kMZHNSyAWCWoulMzfLEPV/etSVsVAYizk3sRqDtASdq1bzasSVsVA+iXeW74MQD9Pnpbj/iQywRQXJO2ygMo94QBEvPTW8NaW0aglPmBdu2bDWvSVnkANRZhd07QwHr5ioN0gJLmB9q1bzasSVsVAShSif4hDIM+sd/W0wqgtOlddu2bDWvSVkUAijTj78Ar5rpEIJofqCHHic8PdOfAf7SnXRFmzw/06IXj45t8qddZWgmgnQH0+7wLx5pg4fJWq1YYnh8I5qZ/9CLMTx98Ww8BFE55UFa1A2gtxZvx7wMy9yAEbWum+iG5OilHAqgNQKD423pII9TWASJYhQUAABVaSURBVHLHA8E7Dqro23qmGIFy2JL7h9UkTRFIy+0Bevwy52crb+tpdkpjHpFV3YqwvCrjL3AngOoBcoaSPXrhpgRpn+pA1jugCaBWANnPhWl++n1bDwEUXDUKgOwuxIfHoJu9v61n10VYTkXY2gAlDapPcsNYAWKV6Jwq0esCBOPJmpXkhrECBFsoAq0LUNrkCkluIIDii9MFKG1yhSQ3jBkgKsLWBShtsvskNxBA8cXpApQ2uUKSG0YNECJosgBlW4lASUpyAwEUXxwCQBkmKLivnHacAHIWkwAyBE0UoMyKQTGAknAggPYQoMwuxdRm8Vaeiy98i7/mgEeg8498Ez/qQwAlAqQJmhpAmScDkHo5z41rD86u3pUAHeLX9xBAew8QikCZG4HUy3lunMjHMWQ1CN6hIV7fQwAlA6QImiZAAE+gEi3eysM54k/3IIDk63sIIAKovhnPiisAyItA8Vdf0DtTI8p3fQJbkN+HiAGSb+W5uHEgn+cxANVMG0URKGJVXp//2CNQACD5Vp6L534PtcIkQPL1PQRQOkCCoL0CSKj+DZcEkLuFACKAoms6AcQJ2kOAWooAilpFAKWIAIpblRNAzSKA4lYRQAkigGqsygmgRhFANVYRQM0igOqsygmgJhFAdVYRQI0igGqtsp5zJoACIoBqrSKAmkQA1VpFADWpFiA1pwvMs/CpPZnexckxNlcHAZQA0PuKmns30dokNxBA8cWhAnT5jdq7qJHHNGoAuvfsD0QEevK919DqJDdMBqDYk/KjBmgutWWAdBH2+OVjOWP0HkyymTnfpzQyUQOE/loAXd6CEYkwrOz8w89f/bH1TE8HgGC2aBOFkq6j0UagrMqcxMJPyo80As09WQDdOarkEzznhyeV/UxPB4C4dD0oyQ1jBShjW7LMOmxSAKEQpAOQAehDR2I82cUX9FhW80wPARQ6BQegjP9fZtZhwSflxw0Qo8fwowG69df+8YNKPsGDATJr1gQI5ol+8v29aMZniiG0+94A9I3bp0fq+UEMkFmzDkDw7+Hx8bO6IZbkhvECVOESTOwXelJ+5ABhfnAlmjXEzFt67Gd61gEoqCQ3jBWgQCV6mgBF+oFEiSWe6qnsZ3oIoNApeACJQsw5LPCk/AQBWk8EkGcVAdRGBJBvVeYd5j8pTwBJEUC+VQRQCxFAvlU+QP6T8gSQFAEUsCrzDiOAYiKAAlb5AHlPyhNAUh0AsjpMJgWQMY0AatL6AIW63NLsHaKr7R19gNwn5QkgqbUB4vetU5wyRoC0ZQRQkzoBFGqvJNg7RFc7O/oAOU/KE0BS6wIUHPiQaO8QXe3sSAClau1JNjMY/OmO/5yOApaNf3DrMACSjAcHPqRdMEO8Vt0dMz/5PJw0RaCWkqeYQyus3Zu1xgDQcrkUC5mbQm69U37UAK2EdgsQI6jtyyFHANCSbREEeQCVeV6FBieOEiD1Z5cAMYdOrwhb8v+4Mi/5PBhwxwgQxKBfegBd3Jg99bu3+bu74XUZr8xmR2fs3xYBml4znkWg5ZQBWnlCAMHQ1Su34SGM82fuXt6CZ3uiT/NsACAW0PPJAQR1oBqAJl2EweB5+XQqW4Ql+Nc0bfT6lWh2upigiQAEWxBBOPm8nE4dKFSJPv/IAwEQf+XB1gGC082n2ZEYBigXFntJjxCgcDNeRiB4XxgUYX0AhAmaEECIIC9/f3j0ZACSdSD+LMb12/0AVOaTBMi05PcKINYK+0sMmdPZ7KnnT3oCyBA0KYA0QX7+3vDo6QBUpb+rmQAyG4IACYL2C6DLW7HXWm4TIE3QxADiBIUA8oZHTwag9dR9TPSEASqzUP7O6FYCKC41vcvjl48//V4UoOSXQ44AINFBwhd5CCKAugCkJtl88urN6uFnYgBJgqYCUFFKgGQhFkjaHt06IoC2oZRJNh9//U0015R39pMCCPCRAAFB4SdPrMGJBFBcao7EL79XPf4aTBAUnmQzG/9YPaVVtWL/q2/LyJjL6djbXakzlAmAQP6VkCW+mm34EahgRRj8UxuXkUeXcopASu0iEChgSJb2Zq3hA4Qr0VzLyKNLue+GIVrVg5Lmia6vA00JIBaEWBgyhxFATUoB6MmrX6lrhZWcoOkAhAmqlpFRl/7AjiFa1YOSJtms7QcCZUnvtRkFQHxpZb4tw4PmCCCpDc3OkaW8lmQ8AGmCqtjTt5U3MmiIVvWgzQHU/FqSEQGkCIJvwUFzlTewY4hW9aBNzQ+UTQwgSRAHaBlMjwDi2iBAjW+VGBVAgiD+TRNkp+cM7BiiVT1oYzOUQaBveCnAuAAqoT+xDiBnYMcQrepBGwWoYU73kQEEBAmA1CBpAsgXAVRj1aoQ3xRBbnrWXeQhWtWDNjfJpkXQNAAq5V2xTI4v89LDAzuGaFUP2jBAtVNyjw+gsrIIIoB8bXCaX0zQVABalZwgMG0ZSi/3rRmSVT1o4wDVTIg7QoAqRhAglBFAEW1yonEUgiYDEHQHFTIE6XkX0G58GEKwMh1dJIBibkAhaDoACYIEQGryKbybnrZsiFb1oLUn2azY9eiukuM/Jzbic1VVBZi25P+5yvOJmdtSa0cgd+werFchiE/9Er9ghnit1uUIMSizJp/Cu8XuiUUX9z0CyVPED8Dok9eFWHAmnbECpAkqggBF7olFFwkg6dTVahkBKDIZ3GgBAoKygtemvdtiVAdqK3mKjB7zBJU+eVOI1dg7RFc35CgIKvwN1AprfYRyKS/C3HataslPqwgrgaBM9Ah591VhKdQjHV0kgECFqESbWoFcL0d1TKoSzVVIefdV+VKgQzG6SAAZl8KVaV+SZq4Ot3Lp2zkkVzfnqJudzn1VsZQP0qoe1AUgYAgIwh7Fk73EbosN0dXNOTJ4CtHstO+ryqV8iFb1oG4AwZ/VyvIofkVA5K7GEF3dnKOYd4Ebbd0WU0tJI1ka8oivmixAMgiFAYrc1RgpQNmK/V9KgvwDqpSRLA15xFdNESC1VBQrfp9IyprhPfiWtpECxBpiUISJtlgAoJSRLA15xFdNGSDeJDO3Ge1XBIResjVKgDJRiQbrCjsB89k8EKE+j5pV0wYIgpC5T2TP8J779o4SID5rdJbxWjTqkbZTbhyI0JBHfNXEARITEkReEeD1KY4TIPGORkVQ5d0V459NXWBNeURXjREgM6XCw+NjOV1iFXW1TZBlpOvQkQJUwjsaRT2oLCrvrpj4bOiEb8wjtmqEAKGpNe/dROujblipLiEXILeFO1aARPUu43cAC++mhvysvw3YnEdk1QgBMtNKPfnea2h9jRtWoRneVd3At3N0AIlvnKBKE0QARWQmtmNl2fExD0LhSTaNVkW1DM5MOalRe6wiVMEoxcD4RD5CcVLGNikOkJla89GLr6EoVHsdrcTdRvxCcFU38C+UkUagEipClS7FAnWg+G3A5jwiq0Ydgbh0PajeDTDKY5llhTepTp57do4XoJI3xniXop2yaoXlNV1C+wKQPbVmKkAQhLLlsvDfRGdGSk8AoFI255elHmUWqAvt3KoeVNcKU1NrQmH25PsNzXh0dZZLKMi890Dp965PAaBKEbQswq0xbm8kof0ASPYDQRB6eHz8rC7LGt2gxpp5G4Y8/DwpRys9QRAE22BrDBRukO0NQBE1u2G1WjKIsszdkOf5ZIowWIQuxaU3RhHvlIc6pQmgJjfAyJlsVSwz775qblUNxg4QC0JFUWR8oGvMqkCDjABqcsOKD/9kjRSOkNqgGyjTAQiKMdbgFAPDY1YxhOw7ZARQkxvE4E/4Cy16z9489w8ZKUBqkAcgFL8soOS2ptEmgOrdoIafw6tvloGZ3lXVYAIAqQd0C+/xFGsxxzc4CKAEN8hvslvRWQtVAzlWsT4YDR+gTIyUzuD+/DI8kSKXTRAB1OgG/W0FI80UQqhdljfNwNCcR2RVzxGoVHNMQHNsyWcR8veHIswfFxU1gQByakRLcXMsq6xHNiIuHRlAKxjHsuJrBEKFP4mQrETnwSYZAdTwDRy8BISysrIf2ZgGQDwCsf9hTcF7psOvRuDy7pEFTSCAHCMLQCiDBhnqGeJF2GDuWyflGExalsvqZeFFZkqx0P6i7K43gQDyjCxW6KkGvkZVogVDIwZIp8wbY0XGexaXy+j+9pizoAkEkG8kvMtW3eBQG81sQru/7ZiUYz1AqjyD3gtACD99iGeGKfEtjqAJBFCwtiCe7OSVIS48aAjVqEcMkOpD5Q+PFahvyAXIqgDuG0DrCmakYmEehoFCdajKxNSVRhCI+GjQ8Q4JzcBCZqa0i88HIxf9nfnQV2HreC3uLwKpV7KvxKxUEIcKU4rJnXbX6Z+UY0IEEv2KsmtIVP/4pEKFCUbmkJzfJ7Nf1RfINpr1OCNQs3PDRsoGb8ExknNW2tPBaKcmpBpfteMiLKsEPDwSiS3OACkrnTx3bnMEso1mvV8AlboiAD1DhapvZs5OeR6tEI0CIDz5sfrCsOJXjT/GrnQ6VIPZRrPeX4BKMepsyWMRU6F3ysssl25tyiOyaqcABXYCiPiEpKxd5gwdAsnusLq7rdGs9w4gbCc4lf1bcgFDXDm0zaQve+1ya7YjlnQTQGLMB7tY5Ps2ChOJyhLdU86jgXd9q3rQ7gAqxS0AOWscMJQt+fi+zDg4r6sRRVYNDyBZtTaTLNoQ4Ra+pIgASgBoxetCJa5uZmp8Fq4t9NNj0mxHLOkUgFalrFpLCw1EK7vWxNVQKyKAdNXAeE4uZ6K6mXH3miO4Q+0BROFs13f1+lYlAWRBYiiCFikrsx2ASrda1Dh2ah8BCmyR97VZbYELGNK75M4AonC2jVnvBqDSdHip9ZmgSNwjzFZWR7U+UFPUNHaKANLXKr/focI9p8jslGd+cB8LQFZC6gtQJNoSKzWNub83tzkvaoe+EEAm2IuOOHVbUgQjuUkE/lwpkC1+d0U7V69vVQeAeJnNezMybbC4/WEbxWzO/CYaetajEt+N6ZuFpE6DA8iuUlaGJP6UB96W2yiptdlawX59qzoBpC+LTGmFUHJ2ktaaYn2pejzEqrwwl05v2hJAizov1rsaHarWQ+9JhkByd8AkZbk3MQi4NSutiDUYgEprJpOyNCShoGRfO8pYdLHxBPHFt1lI6rQuQHOpgFOcTT4blf3V3mkRAKiEBgu/+1Gs+L00I/sXVl51rlXX1bhOa1lVxpGJWrVw1iRfFkoWP2gHTZI2XXa7SoGZ2lQOVJ4NCiAzyaZZqjRA3Cdzx2a1bVEt5s4G9APINYswQBX2tRXsF8ViJX2o3p1jwVTyOgXsJA9BroV/OsHAtRr5cW2rFqVrFTrfdS4Lf1PoJFRNG5dvUE8Em3j/tniVEGCm2nPbRMZWyiSbaLrNSrt6vmD/RbSYo236AI0VrAlf0DVxLdjlJt7FrZDinUiFahuvFGzL+Wq+Yv/PdUGHKbOtckKFBZCyOHLC8csiPVoHshWms13YH/FNFnDKKkWV+o6t6kEpE0zZU00p03GUwf5jmvP/pSsDWInjS5TAQmqu/0jhHzDka8fVlhcVc5m4Vos6V+sTqbxYofKdl9XcMmgurCjNZeEkYHxVWs7CWDqbglbZ144pmGGgiLkk+Q23ARVhZoo7s4Qm2VzMK8YQLFjiayq5zV4/r+QRVSxyhYKZFlssbLgCAi8yTtQ3KTkcGxYh1s9FmAoMEtRRE3/Thszd3K2z9I83iUZjdbIyPw15NTjf1yZhTaVMsmmWQOpSVe70LhruWXeD9QOI49kFrS9WHb9E8DKXIkIpxc8Ly6uBSrTcyTRqLKtkQFGJWXGQVYDmpXfCzIrSnJqJoPicTEj246rtBuRCm9QFSsD8Aqz+bQrUxZy151RS28bGqF0EAsmzV/Vox6NSgXjNfwB+RCUpQ6igWkEVKBiFvxY2XIFsYafC7GSKVbiCF9rVAqgFdrU+b/mTe8mr37F0kp6rtbhmZ1eUwPQKwedcbL5RnlXW8YGLTXiUG2lZ1YPWrQN511JTjwn6AXTdMrS/XcmxUrXZCrra3imQqljFXK33sawSASWWfKiFrvOTP+LCvyyckOxa5RvlAWQwLYMXW2V/HQZAZpJNswRqdm7E1V4vXqwTIJZqB4BKB6AiApD1S3nJB61yG5L1+QdSbQaojPjKa8SafXpT0iSbgX4gz8o1AKrdKZTq+q528/RcrY+uan7q4Fm6KdfnX59qfFU4KQcgdFK9aWD3wtyENuZqJ0/P1fq7DWl7gBoArk81vmo9q3oQAdTVKteaIVnVgwigrlYRQC2V5AYCKL5IACW4gQCKL+47QFp1bw4Lb/v1NXZKzXETWTflEdzmrhykVdsTAdQiDwLIFwHUIg8CyBcB1CIPAshXfxNMkSYpAojUSQQQqZMIIFInEUCkTloLoPePjz/1pvO0j9LDY77R3gYrj49vyo18cJrcwezH14aOfvQCHBpOOJKt/LCzrf7f32Vr7T1wam2tclIfqFXb1joA8ffwfsZ52kdJvB48sA1GVvON74PZcgezH18bOhoG0z568bV4wn62eCedbfV/WA4P/7a1B95xLatM6gO1autatwhj7nZeLC/05Ht87LS/DTzGN9579gcwRk3soPcTa0NHvw+uuHczmnAgW7STzra69zf+5T9889E/+1d4Dze1tlah1Ads1Va1LkCMcHusvRSLnhBe/W1wSaiNzDi5A9pPDn0MHs2+RBMOZIt2wtn+i3/65uN///fwHm5qba1CqQ/Yqq1qPYAevfDsa87TPmrLi/za8LbxL3IjOFXugPbjP0D4aBiUHUs4lK3ZCWf7/t9krv53fwvvYafW2iqcejVUq7asdSNQLej3bnrb3te1OrYxeq2Gj3788lfiCYeyNTtZ2TZfqy2twqnXRKBdW7VVrd2Mv3czXtQGtt37itn4KFBbsFxtH/3ohZspmaKtZiec7eP/0FxbaGcVTr0asFXb1DoAyRBpP+2Dtj35/pvuNlHjkxvBOLkD2k8VAe7R0tORhIPZ6p2sbJ/8178D7RW8B06tvVVW6tUwrdq61opAD4+PWW0h2mMS2KYqf3xjfY+JezTq9ohm6h6odrKybeoxaW2VnfpArdq2qCea1EkEEKmTCCBSJxFApE4igEidRACROokAInUSAUTqJAKI1En7DtD59W8fzmYnuz6N8WrvATq89qA6vXp31+cxWhFAJxCGbu/6PEarvQcI2CGA1hcBRAB1EgFEAHUSAUQAdRIBRAB10r4DROooAojUSQQQqZMIIFInEUCkTiKASJ1EAJE6iQAidRIBROokAojUSf8fksdgNhwAHxEAAAAASUVORK5CYII=" style="display: block; margin: auto;" /></p>
</div>
<div id="finding-all-cutpoints-with-acceptable-performance" class="section level2">
<h2>Finding all cutpoints with acceptable performance</h2>
<p>By default, most packages only return the “best” cutpoint and disregard other cutpoints with quite similar performance, even if the performance differences are minuscule. <strong>cutpointr</strong> makes this process more explicit via the <code>tol_metric</code> argument. For example, if all cutpoints are of interest that achieve at least an accuracy within <code>0.05</code> of the optimally achievable accuracy, <code>tol_metric</code> can be set to <code>0.05</code> and also those cutpoints will be returned.</p>
<p>In the case of the <code>suicide</code> data and when maximizing the sum of sensitivity and specificity, empirically the cutpoints 2 and 3 lead to quite similar performances. If <code>tol_metric</code> is set to <code>0.05</code>, both will be returned.</p>
<div class="sourceCode" id="cb61"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb61-1"><a href="#cb61-1"></a>opt_cut <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, <span class="dt">metric =</span> sum_sens_spec, </span>
<span id="cb61-2"><a href="#cb61-2"></a> <span class="dt">tol_metric =</span> <span class="fl">0.05</span>, <span class="dt">break_ties =</span> c)</span></code></pre></div>
<pre><code>## Assuming the positive class is yes</code></pre>
<pre><code>## Assuming the positive class has higher x values</code></pre>
<pre><code>## Multiple optimal cutpoints found, applying break_ties.</code></pre>
<div class="sourceCode" id="cb65"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb65-1"><a href="#cb65-1"></a><span class="kw">library</span>(tidyr)</span>
<span id="cb65-2"><a href="#cb65-2"></a>opt_cut <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb65-3"><a href="#cb65-3"></a><span class="st"> </span><span class="kw">select</span>(optimal_cutpoint, sum_sens_spec) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb65-4"><a href="#cb65-4"></a><span class="st"> </span><span class="kw">unnest</span>(<span class="dt">cols =</span> <span class="kw">c</span>(optimal_cutpoint, sum_sens_spec))</span></code></pre></div>
<pre><code>## # A tibble: 2 x 2
## optimal_cutpoint sum_sens_spec
## <dbl> <dbl>
## 1 2 1.75
## 2 1 1.70</code></pre>
</div>
<div id="manual-and-mean-median-cutpoints" class="section level2">
<h2>Manual and mean / median cutpoints</h2>
<p>Using the <code>oc_manual</code> function the optimal cutpoint will not be determined based on, for example, a metric but is instead set manually using the <code>cutpoint</code> argument. This is useful for supplying and evaluating cutpoints that were found in the literature or in other external sources.</p>
<p>The <code>oc_manual</code> function could also be used to set the cutpoint to the sample mean using <code>cutpoint = mean(data$x)</code>. However, this may introduce bias into the bootstrap validation procedure, since the actual mean of the population is not known and thus the mean to be used as the cutpoint should be automatically determined in every resample. To do so, the <code>oc_mean</code> and <code>oc_median</code> functions can be used.</p>
<div class="sourceCode" id="cb67"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb67-1"><a href="#cb67-1"></a><span class="kw">set.seed</span>(<span class="dv">100</span>)</span>
<span id="cb67-2"><a href="#cb67-2"></a>opt_cut_manual <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, <span class="dt">method =</span> oc_manual, </span>
<span id="cb67-3"><a href="#cb67-3"></a> <span class="dt">cutpoint =</span> <span class="kw">mean</span>(suicide<span class="op">$</span>dsi), <span class="dt">boot_runs =</span> <span class="dv">1000</span>)</span>
<span id="cb67-4"><a href="#cb67-4"></a><span class="kw">set.seed</span>(<span class="dv">100</span>)</span>
<span id="cb67-5"><a href="#cb67-5"></a>opt_cut_mean <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, <span class="dt">method =</span> oc_mean, <span class="dt">boot_runs =</span> <span class="dv">1000</span>)</span></code></pre></div>
</div>
<div id="nonstandard-evaluation-via-tidyeval" class="section level2">
<h2>Nonstandard evaluation via tidyeval</h2>
<p>The arguments to <code>cutpointr</code> do not need to be enclosed in quotes. This is possible thanks to nonstandard evaluation of the arguments, which are evaluated on <code>data</code>.</p>
<p>Functions that use nonstandard evaluation are often not suitable for programming with. The use of nonstandard evaluation may lead to scoping problems and subsequent obvious as well as possibly subtle errors. <strong>cutpointr</strong> uses tidyeval internally and accordingly the same rules as for programming with <code>dplyr</code> apply. Arguments can be unquoted with <code>!!</code>:</p>
<div class="sourceCode" id="cb68"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb68-1"><a href="#cb68-1"></a>myvar <-<span class="st"> "dsi"</span></span>
<span id="cb68-2"><a href="#cb68-2"></a><span class="kw">cutpointr</span>(suicide, <span class="op">!!</span>myvar, suicide)</span></code></pre></div>
</div>
<div id="roc-curve-and-optimal-cutpoint-for-multiple-variables" class="section level2">
<h2>ROC curve and optimal cutpoint for multiple variables</h2>
<p>Alternatively, we can map the standard evaluation version <code>cutpointr</code> to the column names. If <code>direction</code> and / or <code>pos_class</code> and <code>neg_class</code> are unspecified, these parameters will automatically be determined by <strong>cutpointr</strong> so that the AUC values for all variables will be <span class="math inline">\(> 0.5\)</span>.</p>
<p>We could do this manually, e.g. using <code>purrr::map</code>, but to make this task more convenient <code>multi_cutpointr</code> can be used to achieve the same result. It maps multiple predictor columns to <code>cutpointr</code>, by default all numeric columns except for the class column.</p>
<div class="sourceCode" id="cb69"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb69-1"><a href="#cb69-1"></a>mcp <-<span class="st"> </span><span class="kw">multi_cutpointr</span>(suicide, <span class="dt">class =</span> suicide, <span class="dt">pos_class =</span> <span class="st">"yes"</span>, </span>
<span id="cb69-2"><a href="#cb69-2"></a> <span class="dt">use_midpoints =</span> <span class="ot">TRUE</span>, <span class="dt">silent =</span> <span class="ot">TRUE</span>) </span>
<span id="cb69-3"><a href="#cb69-3"></a><span class="kw">summary</span>(mcp)</span></code></pre></div>
<pre><code>## Method: maximize_metric
## Predictor: age, dsi
## Outcome: suicide
##
## Predictor: age
## --------------------------------------------------------------------------------
## direction AUC n n_pos n_neg
## <= 0.5257 532 36 496
##
## optimal_cutpoint sum_sens_spec acc sensitivity specificity tp fn fp tn
## 55.5 1.1154 0.1992 0.9722 0.1431 35 1 425 71
##
## Predictor summary:
## Data Min. 5% 1st Qu. Median Mean 3rd Qu. 95% Max. SD NAs
## Overall 18 19 24 28.0 34.1259 41.25 65.00 83 15.0542 0
## no 18 19 24 28.0 34.2218 41.25 65.50 83 15.1857 0
## yes 18 18 22 27.5 32.8056 41.25 54.25 69 13.2273 0
##
## Predictor: dsi
## --------------------------------------------------------------------------------
## direction AUC n n_pos n_neg
## >= 0.9238 532 36 496
##
## optimal_cutpoint sum_sens_spec acc sensitivity specificity tp fn fp tn
## 1.5 1.7518 0.8647 0.8889 0.8629 32 4 68 428
##
## Predictor summary:
## Data Min. 5% 1st Qu. Median Mean 3rd Qu. 95% Max. SD NAs
## Overall 0 0.00 0 0 0.9211 1 5.00 11 1.8527 0
## no 0 0.00 0 0 0.6331 0 4.00 10 1.4122 0
## yes 0 0.75 4 5 4.8889 6 9.25 11 2.5498 0</code></pre>
</div>
<div id="accessing-data-roc_curve-and-boot" class="section level2">
<h2>Accessing <code>data</code>, <code>roc_curve</code>, and <code>boot</code></h2>
<p>The object returned by <code>cutpointr</code> is of the classes <code>cutpointr</code>, <code>tbl_df</code>, <code>tbl</code>, and <code>data.frame</code>. Thus, it can be handled like a usual data frame. The columns <code>data</code>, <code>roc_curve</code>, and <code>boot</code> consist of nested data frames, which means that these are list columns whose elements are data frames. They can either be accessed using <code>[</code> or by using functions from the tidyverse. If subgroups were given, the output contains one row per subgroup and the function that accesses the data should be mapped to every row or the data should be grouped by subgroup.</p>
<div class="sourceCode" id="cb71"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb71-1"><a href="#cb71-1"></a><span class="kw">set.seed</span>(<span class="dv">123</span>)</span>
<span id="cb71-2"><a href="#cb71-2"></a>opt_cut_b_g <-<span class="st"> </span><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">boot_runs =</span> <span class="dv">1000</span>)</span></code></pre></div>
<div class="sourceCode" id="cb72"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb72-1"><a href="#cb72-1"></a><span class="co"># Using dplyr and tidyr</span></span>
<span id="cb72-2"><a href="#cb72-2"></a><span class="kw">library</span>(tidyr)</span>
<span id="cb72-3"><a href="#cb72-3"></a>opt_cut_b_g <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb72-4"><a href="#cb72-4"></a><span class="st"> </span><span class="kw">group_by</span>(subgroup) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb72-5"><a href="#cb72-5"></a><span class="st"> </span><span class="kw">select</span>(subgroup, boot) <span class="op">%>%</span></span>
<span id="cb72-6"><a href="#cb72-6"></a><span class="st"> </span><span class="kw">unnest</span>(<span class="dt">cols =</span> boot) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb72-7"><a href="#cb72-7"></a><span class="st"> </span><span class="kw">summarise</span>(<span class="dt">sd_oc_boot =</span> <span class="kw">sd</span>(optimal_cutpoint),</span>
<span id="cb72-8"><a href="#cb72-8"></a> <span class="dt">m_oc_boot =</span> <span class="kw">mean</span>(optimal_cutpoint),</span>
<span id="cb72-9"><a href="#cb72-9"></a> <span class="dt">m_acc_oob =</span> <span class="kw">mean</span>(acc_oob))</span></code></pre></div>
<pre><code>## # A tibble: 2 x 4
## subgroup sd_oc_boot m_oc_boot m_acc_oob
## <chr> <dbl> <dbl> <dbl>
## 1 female 0.766 2.17 0.880
## 2 male 1.51 2.92 0.806</code></pre>
</div>
<div id="adding-metrics-to-the-result-of-cutpointr-or-roc" class="section level2">
<h2>Adding metrics to the result of cutpointr() or roc()</h2>
<p>By default, the output of <code>cutpointr</code> includes the optimized metric and several other metrics. The <code>add_metric</code> function adds further metrics. Here, we’re adding the negative predictive value (NPV) and the positive predictive value (PPV) at the optimal cutpoint per subgroup:</p>
<div class="sourceCode" id="cb74"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb74-1"><a href="#cb74-1"></a><span class="kw">cutpointr</span>(suicide, dsi, suicide, gender, <span class="dt">metric =</span> youden, <span class="dt">silent =</span> <span class="ot">TRUE</span>) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb74-2"><a href="#cb74-2"></a><span class="st"> </span><span class="kw">add_metric</span>(<span class="kw">list</span>(ppv, npv)) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb74-3"><a href="#cb74-3"></a><span class="st"> </span><span class="kw">select</span>(subgroup, optimal_cutpoint, youden, ppv, npv)</span></code></pre></div>
<pre><code>## # A tibble: 2 x 5
## subgroup optimal_cutpoint youden ppv npv
## <chr> <dbl> <dbl> <dbl> <dbl>
## 1 female 2 0.808118 0.367647 0.993827
## 2 male 3 0.625106 0.259259 0.982301</code></pre>
<p>In the same fashion, additional metric columns can be added to a <code>roc_cutpointr</code> object:</p>
<div class="sourceCode" id="cb76"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb76-1"><a href="#cb76-1"></a><span class="kw">roc</span>(<span class="dt">data =</span> suicide, <span class="dt">x =</span> dsi, <span class="dt">class =</span> suicide, <span class="dt">pos_class =</span> <span class="st">"yes"</span>,</span>
<span id="cb76-2"><a href="#cb76-2"></a> <span class="dt">neg_class =</span> <span class="st">"no"</span>, <span class="dt">direction =</span> <span class="st">">="</span>) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb76-3"><a href="#cb76-3"></a><span class="st"> </span><span class="kw">add_metric</span>(<span class="kw">list</span>(cohens_kappa, F1_score)) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb76-4"><a href="#cb76-4"></a><span class="st"> </span><span class="kw">select</span>(x.sorted, tp, fp, tn, fn, cohens_kappa, F1_score) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb76-5"><a href="#cb76-5"></a><span class="st"> </span><span class="kw">head</span>()</span></code></pre></div>
<pre><code>## # A tibble: 6 x 7
## x.sorted tp fp tn fn cohens_kappa F1_score
## <dbl> <dbl> <dbl> <int> <int> <dbl> <dbl>
## 1 Inf 0 0 496 36 0 0
## 2 11 1 0 496 35 0.0506 0.0541
## 3 10 2 1 495 34 0.0931 0.103
## 4 9 3 1 495 33 0.138 0.15
## 5 8 4 1 495 32 0.182 0.195
## 6 7 7 1 495 29 0.301 0.318</code></pre>
</div>
<div id="user-defined-functions" class="section level2">
<h2>User-defined functions</h2>
<div id="method" class="section level3">
<h3>method</h3>
<p>User-defined functions can be supplied to <code>method</code>, which is the function that is responsible for returning the optimal cutpoint. To define a new method function, create a function that may take as input(s):</p>
<ul>
<li><code>data</code>: A <code>data.frame</code> or <code>tbl_df</code></li>
<li><code>x</code>: (character) The name of the predictor variable</li>
<li><code>class</code>: (character) The name of the class variable</li>
<li><code>metric_func</code>: A function for calculating a metric, e.g. accuracy. Note that the method function does not necessarily have to accept this argument</li>
<li><code>pos_class</code>: The positive class</li>
<li><code>neg_class</code>: The negative class</li>
<li><code>direction</code>: <code>">="</code> if the positive class has higher x values, <code>"<="</code> otherwise</li>
<li><code>tol_metric</code>: (numeric) In the built-in methods, all cutpoints will be returned that lead to a metric value in the interval [m_max - tol_metric, m_max + tol_metric] where m_max is the maximum achievable metric value. This can be used to return multiple decent cutpoints and to avoid floating-point problems.</li>
<li><code>use_midpoints</code>: (logical) In the built-in methods, if TRUE (default FALSE) the returned optimal cutpoint will be the mean of the optimal cutpoint and the next highest observation (for direction = “>”) or the next lowest observation (for direction = “<”) which avoids biasing the optimal cutpoint.</li>
<li><code>...</code>: Further arguments that are passed to <code>metric</code> or that can be captured inside of <code>method</code></li>
</ul>
<p>The function should return a data frame or tibble with one row, the column <code>optimal_cutpoint</code>, and an optional column with an arbitrary name with the metric value at the optimal cutpoint.</p>
<p>For example, a function for choosing the cutpoint as the mean of the independent variable could look like this:</p>
<div class="sourceCode" id="cb78"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb78-1"><a href="#cb78-1"></a>mean_cut <-<span class="st"> </span><span class="cf">function</span>(data, x, ...) {</span>
<span id="cb78-2"><a href="#cb78-2"></a> oc <-<span class="st"> </span><span class="kw">mean</span>(data[[x]])</span>
<span id="cb78-3"><a href="#cb78-3"></a> <span class="kw">return</span>(<span class="kw">data.frame</span>(<span class="dt">optimal_cutpoint =</span> oc))</span>
<span id="cb78-4"><a href="#cb78-4"></a>}</span></code></pre></div>
<p>If a <code>method</code> function does not return a metric column, the default <code>sum_sens_spec</code>, the sum of sensitivity and specificity, is returned as the extra metric column in addition to accuracy, sensitivity and specificity.</p>
<p>Some <code>method</code> functions that make use of the additional arguments (that are captured by <code>...</code>) are already included in <strong>cutpointr</strong>, see the list at the top. Since these functions are arguments to <code>cutpointr</code> their code can be accessed by simply typing their name, see for example <code>oc_youden_normal</code>.</p>
</div>
<div id="metric" class="section level3">
<h3>metric</h3>
<p>User defined <code>metric</code> functions can be used as well. They are mainly useful in conjunction with <code>method = maximize_metric</code>, <code>method = minimize_metric</code>, or one of the other minimization and maximization functions. In case of a different <code>method</code> function <code>metric</code> will only be used as the main out-of-bag metric when plotting the result. The <code>metric</code> function should accept the following inputs as vectors:</p>
<ul>
<li><code>tp</code>: Vector of true positives</li>
<li><code>fp</code>: Vector of false positives</li>
<li><code>tn</code>: Vector of true negatives</li>
<li><code>fn</code>: Vector of false negatives</li>
<li><code>...</code>: Further arguments</li>
</ul>
<p>The function should return a numeric vector, a matrix, or a <code>data.frame</code> with one column. If the column is named, the name will be included in the output and plots. Avoid using names that are identical to the column names that are by default returned by <strong>cutpointr</strong>, as such names will be prefixed by <code>metric_</code> in the output. The inputs (<code>tp</code>, <code>fp</code>, <code>tn</code>, and <code>fn</code>) are vectors. The code of the included metric functions can be accessed by simply typing their name.</p>
<p>For example, this is the <code>misclassification_cost</code> metric function:</p>
<div class="sourceCode" id="cb79"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb79-1"><a href="#cb79-1"></a>misclassification_cost</span></code></pre></div>
<pre><code>## function (tp, fp, tn, fn, cost_fp = 1, cost_fn = 1, ...)
## {
## misclassification_cost <- cost_fp * fp + cost_fn * fn
## misclassification_cost <- matrix(misclassification_cost,
## ncol = 1)
## colnames(misclassification_cost) <- "misclassification_cost"
## return(misclassification_cost)
## }
## <bytecode: 0x00000000247b08c0>
## <environment: namespace:cutpointr></code></pre>
</div>
</div>
</div>
<div id="plotting" class="section level1">
<h1>Plotting</h1>
<p><strong>cutpointr</strong> includes several convenience functions for plotting data from a <code>cutpointr</code> object. These include:</p>
<ul>
<li><code>plot_cutpointr</code>: General purpose plotting function for cutpointr or roc_cutpointr objects</li>
<li><code>plot_cut_boot</code>: Plot the bootstrapped distribution of optimal cutpoints</li>
<li><code>plot_metric</code>: If <code>maximize_metric</code> or <code>minimize_metric</code> was used this function plots all possible cutoffs on the x-axis vs. the respective metric values on the y-axis. If bootstrapping was run, a confidence interval based on the bootstrapped distribution of metric values at each cutpoint can be displayed. To display no confidence interval set <code>conf_lvl = 0</code>.</li>
<li><code>plot_metric_boot</code>: Plot the distribution of out-of-bag metric values</li>
<li><code>plot_precision_recall</code>: Plot the precision recall curve</li>
<li><code>plot_sensitivity_specificity</code>: Plot all cutpoints vs. sensitivity and specificity</li>
<li><code>plot_roc</code>: Plot the ROC curve</li>
<li><code>plot_x</code>: Plot the distribution of the predictor variable</li>
</ul>
<div class="sourceCode" id="cb81"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb81-1"><a href="#cb81-1"></a><span class="kw">plot_cut_boot</span>(opt_cut_b_g)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABWVBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QdxMgfx8swyc0zMzM6AAA6ADo6AGY6OgA6OmY6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshe0dRhtbVh09ZjoqFkuLhmAABmADpmAGZmOgBmOmZmkJBmkNtmtrZmtttmtv9m2dxuTU1uTW5uTY5ubqtuq8huq+R11dd/3+GOTU2OTW6OTY6ObquOyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rbo6rjk2ryKur5P+2ZgC2Zjq2kDq2tpC2ttu225C229u22/+2///Ijk3I///bkDrbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vy8vLzpZ/1k4z2qKL3j4f4dm35mJH7raf/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///8KnZ+4AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAPWElEQVR4nO2d/5/jRBnH04Vr/dJbRegibF2Kp0IXROXUW1QOpIBexS/0KCrugagLpcd2u/n/f3C+ZiZpZpLJJJ1u+vm8Xttt58mkmeedeWaSTGeiGAqqKPQB7LsAILAAILAAILAAILAAILAAILAAILAqAJhHTN/70GBf/mhm/ARlVR1AdDAz2FOGuWkziKkKAObS9Z+isc2e/wnKqjKAeNkfkNfHd6LoSRaMxLv1Gakcg3j92yjqvCw+rc+68+jgw/jj75DEn8TxajRY9KMnJ3UW5Kaqeg34oEMcSPxIpL+TLmdRaiw/PdGPbp2L2DUmAJ7qi2x7L482gFQA4t+XacKtc/WOAVr2nz4nL12OixjZxqQSsMTVKOqerz+IuvWX58bJA4B0cRxPOxP1TgB44sd/41szAKIh+Oeff9ePKADCiWyM5sGnDfiYeHLZZw3xggKQ77h9KnuqHABz+JJFKQaA0ZojBvkAoCewGUD89zu8p6oAkMDz1Ft/+WQEALp8ARhCENM/fkMaXAWAM+JtAP2cBKa9lk8I0ptevREmEBbRM+e8o0Q/SQDdc9pZvXVO6sIzaIS5PBpheiG22Q1ldtkNpTjYJ37GRyJxNXqib7mU3itVB/BdFsCXyYVY8m51h3Yy6YXY0+f80794I8wu1X5PIhdpAwgv482kvVKQu6GiEYZiAAguAAgsAAgsPBELLAAILAAILAAILAAILAAILAAILAAIrIoAFgez5W31OOWxuK9G0vRkadtIy2hueD5vyWpKf7x5hy/vQHdH1QFon/QiZj1T5PyYXhjnjzCyZTXYCr6txMFsW7sAwPUst9j2AsBqFHXu8xDEHrOP6etgefhmdPBXGoLuswcxrKzL228z2+0JexzTlSnijOdpNHtX+xyLXSxFVrkTno995SBxZZJF/zZ+BJpNfCvbozjouhzoK3cAq9GA/DEAvNj9MWchytqnzye70iVsG/JHk1hyX4wciuM4SUucKbc5mMl0scdkxBELV/OkBaKb0aTUt4ns2u7lt4o9yufTuyB3ACz68MIsD2csiRd7LACMVWOsALBcCzGOQngvSRMAstskAMZxsvuvz2O9rZdZ0wD4ptruU3sUB70jcgcwZw/YD5nbpjx4qBNLQCEnZRYAy6W40D2l0rKfxS5SWdjLgj161hnGGQAiu7Y73SwPekfkB4A1CHrNbhzAatSZaHmqABAHXYPz6lDFEKQFDuWpjdCjA2AjJ0Qu6T09Tf/MAsbhLA8A8+qiUxyCDmfa7jMAzD3f7atKI9yVjTBjwU5LrRkUjTBtq9dnHWlLNbhaC2prhEXWNABaAfoJAJkl9W05jXBSf8axPOj6fOglv27oQgwFmkZddZLdFw0D6e299OJE2tL9xc0+ZOqz2AXPmm4DSATvvCMinMqS/jaRXe+GJrm7yUHvhnbxXpDn6bk7Z3cZAUBgAUBg7SKAvRIABBYABBYABBYABBYABJY7gM/ylJ9a1uyZvYkvb8DV+QIAAPDODgAAUF0AAADe2QEAAKoLAADAOzsA6IW4qM0HWzK3DkAeAQAwCgBaBuDicwBwUt13Qy++vKh5jy0XasCNqwH2QgCAo+oHkEcAAIwCAAAotgOALnshAMBRAAAAxXYA0GUvBAA4CgAAoNgOALrshQAARwEAABTbAUCXvRAA4CgAAIBiOwDoshaC+B8AnAQAAFBs31sAlz/9iP1/dHR09P2PRKK1EADgKhuAr6TXH97TUq2FAABXWQA8fPaPvAZcv/dAS7YWAgBcVSYEXd0lIYhVgm8TWXd38eWXGBjkpDIALl99oNUC61mEGuCqUo0wVdIOWAtBAeQQAACjAOAGAPjquU/j6/dLd0MBwEWFAOgfuQ54NukIWQsBAK5q4EoYAFwEAABQbAcAXdZCAICrAAAAiu0AoMtaCABwFQC0CgDzPwC4CAAAoNgOALpshQAAZwEAABTbAUCXrRAA4CwAAIBiOwDoshWCA9gkAABG1TtfEB0WhJFBTkINuHE1wFYIAHAWAPgAqGHFLAAAgGI7AOiyFaL1AMR69KmVS+kCu3SB07cPZ8vDNyO6lGtq3dJkIwBwsBv8L9ajz1u7ly3x2+9mVullG/HldQHAxW4AINajV77Vlq+fi6WoUyteaxsBgIvdUHixHn12/fbUMu/Z1dvlEusA4GI3Fp+tRw8ANZqde0Gr0TjlWxmCZNTR1rLXQpAIXlYAq1GZlb5thWg7ALkevVzCfqMRpj7PrnPv0ghPLf0lAKAExHr0Ygn7ZPl62g29LwBk17lPNioBgO0pGgBAlXtBC8O5a79a22wDKAJa00yyFWJfAdCwb4wybgDmrLZMLYHIVoh9BUDdZowyDgBo8GK9pYWlCtgKsbcAqivdC7KFHilbIQDAWWkAhad/DAA1CwB2BgBtR7iS1kT+Uv7q7tFznwJAwwA271fI+YKu370XP3peptoK0XYA/85XXQAySuYLuvrVR9q0EbZCtB3ARb5qAUBOf3oJRpW0AXK6mtc+ja9+SecqKJgvSAwLau+4oIv/5qkeAHnSJuvgAKgsZ5GoAC2uAWEAqBoAAM0C4GFI9UIvHdsAAPAEMO3SB5vzbDf0+t2fl+sFAYAfAFIB6C29RaoRpn9lrwMAwBsAfYpT/Up4zwGsz8yPUko9klyfDeiN7anx6Q0AWAEYnVwWAH2G1rU+CwAACwD6WHLCOzHrN+5H0WDBHi7SB5NjBiDVwckFUEqWQuwBgP9tKF0DpoN4LgbG9bs0ZfXihA1YSWzbALBBoD0ACkIQdffqh7P1G5OY/tGPRCRF2WwA2NBT+xNhALADGLFhEykAU+pRZbMAMA9eAYCyAPgZrgGgt5h5CNo8+7MAjMPnAKAsABrnF7fONQA0UQxi5DYLgPUZACSJFQGwrpBeA+iDrifvjJXNAqDoEgwAGr8SjtAIy8QQAMrJUggAAAB3804BIA0G/7EBAAQBML31Cb8jDQAGAA0+lI/F7ehxDbejWwug4WEpAKAl5qqXr7oAxHMagugzmWoAEv+3F8AP8lQfAPYLnIIfyFgkhwW1d2BQ4wDKyHwWoQYAQAXzrgCQ9yE8bkUAgA8Aqjl1fdFNaXMhAMATgO8PNAAgK9s4CQCw2oMAkCEI1wHVADgMRjEA4NcBBY/FzIVQALIE2gPgWxvSAJQfjGICUEbmQuwDAHsNKD0YBQCK7N4A7INRAKDI7gmgaDAKABTZPQEUDUYBgCK7J4CiwSgAUGSvAqCSAAAAiu0AAAAA0JgZAIrtuwOg4YfyAKAStyULAPXr4EdHR2LmmrgSAHKSePloPwFoswQ9vKelmwthAtDrnR7bCQBAjtQMEdfvPdDSzYUwAjg+PbYTAIAcqTlSSCw6OmKVwD5fkBoXlB4Y1Dsl8mqpWiwzADVL0OWrD7RaYD6LDDWgd4waYFaZGsCUtAPmQlgBWAkAQI7SswSVAKD5XwdA/A8AZtl6QXKWIBqMrt8v7IYWALARAIA88esAWgnIdcCzSSwyFgIAqqjGK2EAqKLGAVD/A4BZWwRgIQAADjIWAgCqqCEACQHmfwAwa5sAzAQAwEHGQgBAFQEAAHi7EADMALj/JQAjAQBwkLEQAFBF9QFI+d8AwEQAABykDlJ71A4AVeUDgPizpwqRB0D4HwDM8gNw7ApAI3Bxoe5XAICD1EFqDq0C4HMAiGsA0JOFKAcgIXChXSwAgIvUQWoO3QDAnWsD0OsBQFwFgNJparyPNipIjgxiA4KyYhl6vd4LnqNaWyLvGsBP6fwaICuAXgNo1CI6PhkOh0mDgBrgIHWQFQFwUQBDrRPVnIfbDoARyAWQ+D8HAPP/8KRn8UFdHm4vgJMTAPCWD4AhJ+ADQHWimvNwiwEMkyrgCoD7PyEAAA5SBzkUBADAQ3UAIE6Ms/4nAJT/AcCs4ABkE9Kch1sNQMagHACa/zcASP+z7D0AcJI6yKGqAgBQVTUB6G0A6J2UASAIAICD1EEONQIAUFF1Acj63wUAjWANengvAJz2sv4fWgAo/4sqEPKH9C0BcNLzAFDwO3oASEkdpOZBAqCX9v9QJ1AIIOBMBu0AQFyqA+hZAej+F/xsBNgDHA8Ptx0AcyF7vtVL+b8kALaZeTIP9vTs9NjKoK0A1HQ16l2cB+BEPt/q6f7XCVgBnPCpDHIcTJ2fZDcjaCkANV2NNnFNnAOAuPDkBf6OTyCVii6bANL+Z5ud5rUD0v3a2N58Bi0FoKYqSE9asAmA6IVhjk5O8gBk/U8JbJ7iyvup7LmhKBbthK4WAFCTdah39ulqNlR23b/EphxoWDQwvZF5c8PkbkVydF4dKjNdjXpHVf4s8jwJt2S+UTWAams+2JJ5ZwEUtwFaoP0iLzF3y9zE15N3/ymdXXOWfctcB38jz/4LLXFbKjNdjXpHlVtaAKiqUtPVGK4DAKAGef1ESb0FgKoCAAD4DAAqSrtu+SIvUSn36k3b8vWcHRVlLzoke+5v5iW+npfYsHwAlJfL5XPt2T2/vGEBQGABQGBtBwBkFAAEFgAEFgAEFgAE1hYAfKUtQFNB1+9qE1c76/IVry9vXs0DYLN/P1+8nUkP77FHctV0dfde/Khy7m1oOyFIW4fAVfRpkMcXp1eh2EFtB4BHDbh87Q8eIQgAqC5f8Qri95gbq4mFII9vb17bqQEeJ6HnOUwa4Z+9BwDppeCcdPVr3yDi14o0reYBpEcVueuhVwgiX+zTBWteW6gBqRVo3HV116cnTy5CdroXiivh0AKAwAKAwAKAwAKAwAKAwNoBAI8/jJe3J7kmUzrJYts0x7yzCg/A5GSLzZal2LxbAoDACgBgfRZFXeqm+1F063zZj6IBcdny9tvsHXkZEyP5Fw0SVyZZJvSFb8iz6zZuGbM9br9YFbV9AOuzLvtb9g9m7D/xKvVen3hzTl06P5itRgQC+S8A0M1okgAwYVlE9mR3chdJrpuh7QNYHMzYy7I/Vt6k3mMnPk/7mpzZ0harmJIGwDdNdid3keS6GQoAgMYN6SZyXksAwrniZUFCUEe6kjk5zgAQ2bXd6eYbo90EsBp1JporAaBOLToTLQQdzvIAMK8uOsUh6HCW7A4ASirbCIvWNQ2AVoB+AkBmWY0G5H9nwrLkNMJJ/RlvvVSVFbgbSv7H06i70QZMSQvwDg9PWhbWO33pxQnPIrLr3dAkd3f7xaqocBdinoHiRsUZiwAgsAAgsMLfC9pzAUBgAUBgAUBgAUBgAUBgAUBg/R9GNCLtzxxN+wAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb82"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb82-1"><a href="#cb82-1"></a><span class="kw">plot_metric</span>(opt_cut_b_g, <span class="dt">conf_lvl =</span> <span class="fl">0.9</span>)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABU1BMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8Qos7YzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmZjpmZmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SOTU2OTW6OTY6ObquOyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rbo6rjk2ryKur5P+2ZgC2kDq2tpC2ttu225C229u22/+2/9u2//+84uO+0M/C6OnGhX7Ijk3I///L3dzM8vPbkDrbkGbbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vu1NLz2df4dm3+5OL/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///87YnuoAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAY4UlEQVR4nO2d/XvdtnXHKTu+9K6VdW2mdK3SdG2XKWtmr7O6LV5apduqdm3mq9GRtGRLnMijrq9nWb7//0/DwTtAAAQJkCBlfJ/HlkSQIHk+xPsBUGyzkqpI/QBvujKAxMoAEisDSKwMILEygMTKABIrA0is/gCOv/Mr+tuXP96TAzZ/+Zj+dn1452lrPF4nba/2F95PJh7AeSfbaf4PFUMBAIqdI/zLZlkoAM5vJQYgHsB5J9tp/g8VQwEAdt4mdj+/vbQA8NJ4ADqcNgsAtz7GD3l9+DMM4MX7RfGdz9CfRVHsXR8uzotbf8Sv8cWyeOuIXkReGlvzi7eLYuevyLuS90UXiXjQn/+ITvhreiG6ZI3iQcfXxQH+m0Gn0fMoyAOg8DW7L8QIV9I7/f5tuC85rfNDRVcIgD/cg/fb7P4BAKCMCGnniAG4vSzufA2vsCbHyUWbJVgPbHheYB3o78riIRYqsLW3YJ7bS3KcpIU1i3HNT9cAfJeev10vpVNorMWBBKDDQ8VXCID/OIQXOF5sEIDrQ/hYv1wuyAeFXm+PfUfoub9gpQR5qeNbj68Pb30GL7bQ3lXEs1m+8xSfgHW1X/z50+vfFQt8NfqPZhE8eslc8ADo/AU5Hz0KivG8IHdBfy2eor8WUhbU4aHiKwTAY7DC1f4BANiQcgBeigCAt4NX2GgFBP+Gt//z+39aFvq7ing2y9s/+yO/7mqfnIGihU+V50A8+gYATOgY4sF3O4bECXeCB8NPIMoA/4fqbS2rggBAPrCGV9wjOQFJpgQAeXgAcCBfRc49YKm68a5SPMfw83s046UWZ6Zaq3na1gBgQc+nZ6wZAOW0zg8VXUEAwCrHi20XAHDomGQR3/3nf/9q3/WuX6KSr6BWkgCwtIcVBUCXh4qtIABQEKPXI1kQt7MOQMmCkEl+vb9ghmPZLcnXwWgar//+B1oKE4OSDGRdfMDrQCL6YyVvEVmWngUZAHR8qKgKA7De+Qn6nxbCqGH8Agx0LtVJaCG82efV6s3ypySxo7IQVe6oSY4LXDFcSPGsUam7vf4d/erQx4luggthdM6f8m9RRM+joPkUK7T1QlgAEB+090P1tpZVYQCgrkG/QppM9+BdoRrKANAA/gmhNyLlMzmfvis+6a33F1I89AxKjlYriQnOC2EKHr2I4hxXQ2+z87VqqABQ8Ba890PFVxgA9JEcsGxgg3Ls29Buunq/WHwtAEBLibentltSBaSto1/huh+c9F/L4ntf4zYPjwc3xN6hKYc0rP6MFMkbuULIo+dRwAN8hRtupAjfqA0xBgBOe9r1oaJrlr2h66I1T+7SdZFUcwTw4v32jpoMYDChcscjM84ABhMqKd9pPysDyPJTBpBYGUBiZQCJlQEkVgaQWBlAYmUAidUbwOZej9GJLhfBuS8GcUSYlMZNAR0B9II8M2UAiRWSBW3ufbIUQy14QJsNbO9tSeDeBp+xufcID62ARWHUg46nbHY/LqDvnRygEWCr89j3eLw3VEEAlsimbGSV2G15cLV/QL1KILAgZ2yWYOcFnEQGo0h/Mh5YYQdYBAIATgHscKT3nZzCAByIXGWzS0D831M5kJ7Bf947WpORbmxQfJgdYBHoAHYHGIidkMKyoCMpWz8uyLAeHkDdkQLZmWDke0fn0vAqPswP0Ag0ACLem6kYAM6pQa/2C/AV2jlS6WgApNEsAkAcwBHoAOjhkJecsuKlABCy8RrsudZTAM6CdsG4a8m7CQfLBygkDQDPsm6iogHAWTk18GbZACAVwuDwQYyOg9kBFgH4wIG3D/71gMcb6X0np3gpYE1991CevfNr8SHTMx/hnJxVQ9ncGnwtO8AigHrnB+/h2I+LBT98QzVOQ+wGf8GhygASKwNIrNwdnVgZQGJlAImVASRWBpBYGUBi9QZwwSX92pAjrOdlA0RpCoppY6cygAwgXlgGMIK1MgCqVNbKAKhSWSsDoEplrQyAKpW1MgCqVNbKAKg6vnYdy1oZAFW3166f1TqCDABrHAD1MyQNQQaANUpnXH1JVNdj3G1eGiEF1Pj7J5JSQU4BWMMDkMyvIMgAsAYHoNmfIyjLUj21rkXyyADa5fnaTfsDAmT91dmKIaixcEB8S/a8LKaNnRoYgMn+yPonVYUArBACZnpKJr4le14W08ZODQugvjRZH6sCnZyUpZo2oluy52UxbezUoADqZ5dm64NO6U8VQR3bkj0vi2ljp4YEgPKWS4v1JQAqggzAV62vhvP2S2H+E02n0u8SgjqyJXteFtPGTg0GgBStDEAJ+b0dgIygjmvJnpfFtLFTQwGgVZtLbv8GgVPtb46gzgB85H5+VrW8FPZvA8ARZABecj4/r9pjAOWJHwCEgBLIANrlfH4FALL/SScAz+oMoF3O55cB8PJXI2ACwJJABtAu5/NLAEphd5WAEQDPhKJZsudlMW3s1NAAStnqCgEzAELg0kEgAyByPb/oXtOaXzKBDABrUAB1qVtZImABgAlcOjKhNwnAy59/Dj++eRf0EP/8i89pmOv5a2b/ppUFARsAIHDpKAbeIADPhbm3z3/w7fbJQynQ9fzUCaI0WZkTsAJABDIA0JPv/9vPGYBXf/fb7evf/FYKdT0/DLLUOP83WJkRaAFgJfDmAOBZENI3P0QQHuCMCOlPkByX4RHGy/LUosoWwFUSJ5ZY7zhp+QGABLB9+TdyKnB8QBhAaf/MK0uQaC27WgNvZAqAEoCIlwOO5xf2t+QzlTFI6jA6LTMAEAfw5D475AmA1f8tGX3lBcBM4E0EQDIeSAav/8WjGlqL9petpK3UoIqLXWYn8KYBgH+4CMDtge/zipDj+UUCsFd1hLGV38Rl1kzoTQLgkuP5fQFUzWECngKsBBoedTcJwPXhHt1DvV0xABgOawA0AqpH3c0DcEz2KvUi4AIguuC6ApCrqA0CyPpIxKPuRgKgq3OuvVZJ7QagaWyz/ZU2gpIJlcT8BIAFQQZgAVBVq1WLW4pKgIZxAtz6HAAgaDCYOwCyEvrVvte2ufbnl2qhYMkKzB8AAHypV5LOxK86gtkDIBsX+21b7AeAWt9EwNFEEH6j2KOxWlkA6AjmD6CDHAB4FlRVsrV8BuUpAR5GPBorKwAVQQYgA8Afv2Itn0H5k0YrWSegAZALg/kDOC+Kg/P23ZJBLQBo1tMHACLQCYBIBrMHcHznq/2D4HZAXZ7wnF+1lo9XhBGATMAEgCKYOwBUDeULxrfKDcBiLQ+vCGxuw1hBCwCMIANoAyAT6AbAHqWMwN5NNAsA23PIgkLbAVALDQSgezEqBOwAXN1E8wAQpR2AymBHfuHhlgJhDgJOANZuopkA6KCeAASBgQDYEGQAGoG7d+86ADgyoVYAZgTzAID39PLbsMgOoHRXWSpmfwcBMwASrwcAE4JZAOjSGWdVXZ5WZy5hvyAMwOUbZPQeckesqAx6ifEUvzu6LQXgNHC3LQWYk8CJ3rthSwGQCOaXAsYCsKruEgQuAFYCvgD8e0onA4DsrxbYDkDNgJZ+g7Jkff5WBFEAqIlgDgCu9gum9kRgBVA7AZCxLd7nbyGAwywEOgCQE8EcAHRSLwB8bIt3OUsEpFFiOpJpSQP+AKREMAiACNtzxQdQmrou8YcvDy3yHs+71Pa4A1V1XY8BgCeCWQDAvXGeW8d2AlBiyXYRXc6VYdiShhkJnNoJWHqq5wPgeAFNgfOg8QAAIIx9JsxfqmapTtmHryaM+AAIgTgAGvvePypg13YoPnc+2X282f24gH1j+c6lykmtAK7IaExYNRQVAdTcZcktbwCwkj58EaQCMDoPdQeAs6EoABr73i/vPAWTQc3xav/W481yQTzb6N69yl6+XgAgpjgAhBsbMYFuf8VaOoFTbn/DMjfdAUAiiANA3/ceb1R9j+yFfA4ADmg7ai1tDU5PagdwfbgHLYHjkCyoFinAbRKt2doBgL0Ydtyt61iN5a31fe93cd8NGUcnO4Yj2+M/xHbLbD/3dgCQwy22x0GD8rgWqpUBPtZSCcQHsDrr5lRtfW913/vIALrICcDHJJYmgjJ9KSaAbk7VjjdX970XWRDLdXB/QiML2rXk62YAVz/q2xLGtdA+AEhJvZJayZRBNACN/rnuABr73uuFMNicFcLiJN9CeAIAygaABoFTe1usDYCFQIcU0Nj3/hEpFKAa+ogC2NJqaOOkkQBUqzAAopvCTCAEgL9Xe9vrm+BYCk93a20GAJrrDPUHYEwE4QAg27fmMrMBwCtPumdc06XRQsAHgOdopY/ZJcFYri2XGRWAVgnqBmBF0TVcExsujSEAmolggt3RaQGsGp5xuktjGAA9EUwXAJuoTRcKevXgXb5kgQ2AWgnqDIAmAccyQzEAaIlgggCI+HpBZH2C158+xKumEA0JoDmfSXdpNBPwB9A6Wjms1SU5xgP4ekF0iZRXv/hcWsDGBkApg7sDYKWAnUAcAG2jlWkA6OMB1Nx0oaCXH31LFy2wrxeEAJydlf7eOyaHHu475PAT6uAgFNNzqLaoe0xczvEACoAuFASLddBVI7auFFA6vjuPILj8zD6Jng1XhqaAlciG/FNA/b9GRQSgjgeIDAfKAZECBgUA1585MqEbDKAxHqAC8CgD9FpoTAAnVWwAK8d4fRIAjfEAam66UNDrT++31YL0SlAvACtYkMNOICIAx3h9GgC62HpBdKGg9nZAHACr0gxAmUVvInDjALhkAaDWQgMAWJNATAB2hwnzS4+QBfn5JboA2BZ26GItNjBjIhAVwMo2uzUJAN+1msYBYCMwXQDXh/b5dV5DktaB49EBnNmSgDSJ20CgB4CVxWWoDQBqfTUAWI3sC+D6cFIAzElALPEXB4DFZagFAG7/agBgWPKIdOVc//JRUeyt8YxTyNcPMACj26c6P8DPLdQKoNEM6A3AXgpwAmkANLsh1BRwvLc9p45xywUcuXrvCDus8DAHADY/oHch3KgE9QdgI1C5kkAvAKsyUgpg5r760ePrXx5t4R/8ucW9+yLMAaCTbABUF8Q5ADD6SrQAsJUB+BveOVIAHMMnLcLmAsBBwFoKjwjAVAsCI5MvXAIANRuSBZmHGaOuFzQGADG3LxoAk9diTwCQz6/vPJUAwEHqxEjCXABC1wuKCsBMQJ7e2iAwAQC4KiSnAPio33r/QIQ5AAQvVxMXgJHAIAAMmVAfAL0UG0Cpv1tUAHfvyjO84wFoEkgCIHS9oGYCCAOgESCGR/9XbIKxTmD2AALXCxoEACEgZTxkGQP8dzwATWetNAA6aAwAxFmxUhc14OtIoKTQI0pbkL60hFFvGgAy18zgHc1K4dK7yJ8HgMB5wsMAMK03KtpiCoIQABoB80uXFsUC0GWesEFQCZJ8e1q8crzOKfFpDS+hSnIP8onISx5mLP/TqFgAQucJi1po6yxJWOSzMXG1+bkKh3VbCvC6m1cK0PxFEwEImifMcyB4FXWecMP8cJmdgH6ZG8CKTfELA6D6i6YAEDpPWAFAEBi9cbgvgpVA47KGc26jMUy2lwkCIBNIAiBwnnCtAQAEzdeWlzr3B6CvN2rsEW3Oxp8bgC7yAcAyIoP5yWU2i/UDsDpzIPCqj0lPZ9RoAHrNkOFlsLKlgmwTZWYKvsxiMIO1LIs56Xs7sLVB+gGYTAqICECkAuPEIDMBk7XMizkZNtcwLY6TCIDLT2IIAI0ciFm5NExOpJcZCRitVXkCMC6OExsAusOMAACCC108qBeAE9PA8HgA8C10AB2cUQYBwMrgrotFGgiYrVUZABh3NwH76wgiAWh2Q0gA/J1R5glAXcrGAYBhGACAOwV4O6MMAcBUBvutU+VfZZEX8jDkQfplnf20vQEYy4AOzijTAtAk0AGAc38fww6IkQCYakEdnFHaAHjICqDPis06Aau1qm4AJATjAmhzRrEACJofwCpBvZbM9i4xpXUkmnmQ8TKKYFwAbc4oZgBh8wOCAGgEHNaqugKgRcEcWsJh8wNMRUCHxSJ9qywNAF5bjM0DQNj8AFMR0GW1Tt+xRdEt10gC9stM/bLTAxA2P2DSAJx91dMBEFIIGytBndar9VvjQ1rGoBOAMzuCyQC4PtR8sti8+JcfwmIdbNkgJ4CQnVu8Ku3yOhI6gbYRsdbO78QA9EKYrRcEC0TAgh1k2SArgFUoAEHAbUlrEmgdkjQngg4AhnVL0Qphvl7Qc1if4MlDumyQFYAhB+q6ZLZHq8kAgO+e6+VqFAJgAKllgNZ9LS3WgVIBXTZoa14vqCb+OWHbd/m5+FQN/6BOG4w5bhL08P3knKQnAMBCHXTZIHpE/5SNRUDXFMCSQMun3EgCJ62b7Mlh9o6P1ClAFwfw6sF9eoiXAxMDcNLYwd4OQM+Hpg/g5Ye8+B0YwEVrxw0OMxFwbzHWHCswrbGfGIAtC6L2p8sGmQGYa6GjAhA72bcDUMYtJwOAiI7ngOh6QVD/h+KXLhtE1ABgsn93ABdePoZmAK7dTWYEwNl7LTQUAEygNS+pzAAciWBOAPp0RSQC4L2stGm0bKoA/JxDNUuai4A+AC7cfcc0zJgEHGu6WgZrpgWAFsKu8RshEwC788+IACzZ0BwAdNJwAC58/MyNeZBjLSdLlFqRP2cAxhyoH4AL1+gJM5cpCYjBMu91JNQSJzWA/pP0cBnsuTvICAA6zKIvpwSg/2aecQE4Ji8JcxkIKJsvVebLDAQmAyBgkt70ADQmb1gjLKcEoO8kPUsZ3BeAzwzKVgBGz2kjgakACJikBwB8t+iKBMBAgADAbdxGNjQHAAGT9PBqfREB+ExhbQIg44PPQJSB14D9Gb9ZagBdpAEw50D9AfjMoGRDwlVFTX/5TBZmwBOB0y+I3SwD6A5A+uqfqQAwA54I3I5Z5cwBQCXIuFFmfwDtU1ip6fESnhYAOBlUbcOVwmMlA/ADQEyPbgsmdgMgDNoBEAIzBmDOgUIAWAgQ0xMRu4P9CQIzAMKg1Te0nDMAWyUoCICBABhfvq4WNgYEVgAo1DR5WwEABOYEQFFZBToEmWNtePToZ9SXkpQ/GqqxX5vTYSi5X1AnqSnAUgSEpQAlCfCcx5ICaCKw6rLGMbo84+TcLaaNnZoHgNK28WCtGdmOgAC4aK4fogLga7mMpUkDoATUafYuAI5UcPmMEWgkg/kDqG32jwCg1Fc5UK+rNQAIgQ2AIKAxyADsYY41JmwALIkAwmr5SoNXxGzLAFikYBgArWEGAGYEDQAg4hw0mfGATpItUlYW+w8PQCEg2gFNBCR7MkUora0i3WUsxQFgyClaLTkogCYCmj1ZYqR+ctKxsTR7ADIBtSXcqCHZCPAiX67sjqU4AGxFQFIAaiLwAKAcG0uRUkAPSw4PQEHAC2i/u8W0sVMxANRJAVw0jWxCwMOaBGYPwF4JSg+AFwVSDcnnbjFt7FQUAGlTgCBg7o7WxwpuIoA+lowIgI2J2cYD8IhNbSWQAcQI45trWhA4CGQAscJwUG1Cof1dNy/ToxpLNw+AkESidiWB+QOIYa0YYbYgCYOJwNwB1JMHQMIogyaB6QJgM+VfPXj3B9/yH0TiWR050KQAXOCkwJJBrQXpp48lFwC2XtDrTx9uv/kh+0HFH9XRCpgcgAvBADIlx2WDG57JAYCvF/TqF3jCPP1BQ9mTyuN40a01TJTM/ozAVAGItSI++hbWC6I/tup6QaErRqURrR6lfoytHwBYpQNZnv6gYexTmWEKwJp1CgDxR51ZGcA1/TKAA3CWAeNYa9AoJw8A1ivDtaD7plpQBhCkVgDwz90OyABCFHM/4VGslQFQpbJWBkCVyloZAFUqa2UADTUW0R30srFvN7QygMTKABIrA0isOLMks3orA0isDCCxMoDEygASKxSA0kPqr+fSfkxd9PpTaQF3f738sN/tRlAgANVTwlvQx93jMrx/xPPuwF89QE/Z5zsZQYEA1FGyTupzGdyuz62UwdRpKRBAwKv1SQEvP/rXPlnQDQagekp00MsP++XlD7E1OwpnQX3uN4LSpYA+l/W9HSqE//Y3NxJAQBmg7ozoebu/78+7X/ExuIJrQff7ZOa9c64n/bIgdKtela4RlKgdoGzI1O12fSr0qNkx0VpobgmnVgaQWBlAYmUAiZUBJFYGkFhTA3Cu7yT34rPGOZt7R67geWlaAK4PYfuO80LsbS8b26CW4BloWgDo9innYheVDGBMXe3zTx9bdnPvk2VR7G3uPSoKYHJ9WBQLHEZCDjYQnPKJwzUpANL3TAEcwc/N8tZj2OCM/YPDSwTk/NbjnAKiarPLdzBTARzg3/EGZ2tsdXYoA4gqWwqA31H2hLf6Jsfk4HlrUgBYGXD13lEGkES0+sMydyUL2n0M2/yxLCgDGESiHQCbWl4f7hxBojAVwhSAVG+aqaYFALUEWEsYapgfvHeEDixwNRQ2uJSroQQABKd+5DBNDYBJ889nHMoAEisDSKw5ALjRygASKwNIrAwgsTKAxMoAEisDSKz/BxhfufROnCNhAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb83"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb83-1"><a href="#cb83-1"></a><span class="kw">plot_metric_boot</span>(opt_cut_b_g)</span></code></pre></div>
<pre><code>## Warning: Removed 12 rows containing non-finite values (stat_density).</code></pre>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABpFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QBv8QDvsMDv8QDwMQHvcIHvsIHvsMHv8QHwMUHwcUOwcYPu78Pv8MPv8QPw8cdxMgeur0ftrofu74fx8swyc0zMzM6AAA6ADo6AGY6OgA6OmY6ZmY6ZpA6ZrY6kLY6kNs6ys08vL4+ra8+rq8+vsE/z9JNTU1NTW5NTY5NbqtNjshe0dRhtbVh09ZjoqFkuLhmAABmADpmAGZmOgBmOmZmkJBmkNtmtrZmtttmtv9m2dxuTU1uTW5uTY5ubo5ubqtuq+R11dd8m5l/3+GOTU2OTW6OTY6Obk2OyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rbo6ryKur5OSr5P+2ZgC2Zjq2kDq2tpC2ttu229u22/+2///Ijk3I///bkDrbtmbbtpDb27bb29vb/7bb/9vb///kq27k///r6+vysKzy8vLzpZ/2qKL3j4f4dm34d277raf7urb/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T////LYMq3AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAQ9UlEQVR4nO2djX/cNhnHnWw5J+1gbQoJJexaGNuFQjK2HllKge7WwZamQDNeQiEbsDBg2W7ASFsoHON6I+nV/zR6sWzZlmz5LFl+eX6fJjlbz6mX39d6JFt25Hggq3Jsf4C2CwBYFgCwLABgWQDAsgCAZQEAywIAlqUMYOgQff0DSfnke0fSLZBceQE48xJnh5GCoSwMFJM6AGLp9A/OIK1cvAWSKycAb9Ltoe+fXXecp0ky8l9N91Dj6HnTnzvO3Gv+1nSvM3TmP/A+/ira+QPPO93ujbvO0weGfpOaKm8LeH8OGYh8ROJfMctJlhqwrae6zsKxn7sGCMDzXf9toEC5+wDUAJC/r+EdC8fhKwJo0n3hGH3rUFyokASjRkB2nm47nePp+07H5O9TO+UGwCz2vMO5g/CVD+Cp7/+ZRhMAfkfwtz/+outgAIgTCobugVfePuBj5OSkSzriMQbAXtHyQzZSpQCI4ROSpQgAQmsIOYhXXgD4AJYD8P5ynY5UQwAo8Tz/qz99sg0AxJoNgCQFEf31Z6jDDQFQRrQPwNtBYgIR5U9BfNfLd8IIwth58ZgOlPAWA9A5xoPVhWPUFl6ETjiu3J0wPhFLDkNJORuGYhxkix7xjr/zdPupbsqpdEuVF8DXSAKfBCdiwavT63iQiU/EXjimW3+nnTA5Vfs1ylyoD0C8pBeTWqoSr4b6nTAoIgBgWQDAsgCAZcGMmGUBAMsCAJYFACwLAFgWALAsAGBZAMCyZgbwWfyi2tCfbp9cyjHhgmqRxif+h0ZqVgAJ2063B5KSPLUoljVI2gAEOwBALikDwBMrHd+WyaW7XXKDSliAp97ppZ7JpX0yJ0Nn43tkVnju7uWjIBjPyJCyAf7eQzX69eHNAXsfKYvF6//17UsVwHSvQ758AAfB8RkpwJp08VRlh+akIZ6H6aHXdB4M7/eGC8e0lu4A/8SVdckkGr27iL0Pl8XjDRhgW6oAxtjBMbUlAiBSgEV8Qlv/I/ekXDogAf6MMnmNHJ74DSIAMEi+D5fF4xsoZQAkqfjGRwDwBVjELHIYj8l08ZAEUAeH7O66Q5qwGAC/Uv8bfR9+GY9voIoAwPb0+AKygwE43Z4j7kYAkFliItQz0GYTB8DeRyqMxWv+5asgZQDkHghRCuILsFgEIYNKWY4KgplwahEAYO8jKSgWr+EXrprydsK4S53uzR0Ebkg7YWzepIsj+U4YucugkKN9kAQQvG+QiDdggG3lHYaSAeGNawcoK3ciBVwL2KeRKG/P/WZ7QIah+/PhMHQuSPM4ppPoA9j78P8Qj2+eSroWNA6TOSgi8wDGc/54HiRSCS1g6DR2DKlBcDnasgCAZQEAywIAlgUALAsAWJYqgAciifeWFWm0UqOm8wIA4lCjpvMCAOJQo6bzAgDiUKOm8wIA4lCjpvMCAOJQo6bzAgDiUKOm84LzAMuCFiAONWo6rxYBGI1yVGrUdF4AQFypUdN5tQfA6OFDFQIAQDkSAAAADWoNAOQ/ANAbmR+ACgEAoBwJAACABgEAcaVGTecFAMSVGjWdFwAQV2rUdF5tAUD8BwBaIwEAANCgVABP3rnDXhZ2QH9kCwDc22wYAAUCVQJw8qOfAgDTSgHw5N3f0xT0HFJZn8eURp8SjaI7LX0YXikA7r3VnD7AbwDRFiCcIqtOCzh5836zAYyECak6AO5tYr3lbxV1wEBk0wF4TRqGAgADkcUBiAhUCgCnog4YiJwFAG85ACgYmSM08B8AaIwsCEAySw8AlCMBAADQIAAgrtSo6bzaDUBAAAAoRwIAAKBBbQMQWg4AikYCgNoAGH0KAAxEtgtAreXPR0YmJUeCKUobam0LkN2nUtUWUNAsACBTywEkCQAA5UgAUBcAo4cAAADIBADElRo1nVdbAQhvleMqNWo6LwAgrtSo6bxaB8C3HAAUj1QNRV4DAAAgEwAQV2rUdF4AQFypUdN5AQBxpUZN55UC4NHm5rc/YhsFzaoaANHNupFKy/CeKOUJmR9+5N17hW0VNAsAyJSegjAEqoJmVQgA8VwTAA2rK6YDoC2g5k9J8hOSbBpyFJ+hnE2GAZzcfOk99rrg0WqxBeCDvbYt4PGPGYGCZjUFgL+2fbio776DV2InC5bevXw0ufxLZ/6IX96UD5oBgPche0isoFmVBZAgkAqArW0feOsv3cuW7J10O7EFfkkQZiJbzFEO4NHL91vQAnIC8Ne2D70d0DWQ8Wq9Qwxg4C9eza997QflBYAfFG5kH4A9nxEAW9s+8NZfvp0t2072i1Z/ly8H3vgzYWJ1HIDoXtFopVIjyNr2AEA9VDcA4mXEW5aCWNYZkyXA4ynIT16pAE635auGFzSrIQDY2va4053uzSU7Yew564TDIPVO+FA6XipoVrUAjGZuAWxtezwcvXGNjjA7Hh2G7vsAPH8YmghSAEBqcnoNAkCtjgJ4OHo4K4A0jSXHbvrZWrIPwAjmEwmroFnVAvBQNwCc9qVZJh+AIWkthwmYBc1qOABsmzTL5ACAkxcZLY0TTaCgWU0HMLuio6Bk6mEqaBYAkCkKQHL4ewDAmBoOwB/v1AIA7keoRL1JQbMAQDaAlOsVXoMBxAmkAvinWLoANFDx6UiB8kxKjsQq8gkDAOjwx6dgWM3pA9gpr64WMPqXSHoAZKigWQAAANQBAE1DwrOxgmbZARBcdKsLgMMOntgcNmYYWjcAqAHgS3rNORGrIQA8i9MYAOFl/3IATPeEUylESlOS070evrB92JQUVDYAqcmqAPAcWkcwFwAAlADgackDOoiZvr3vOL0xmVzEE5MDAkA4wGnwMJSbeNQH4N8JRVvAYc8b+jfGdTt4z+m1A3LDSlAGAEymIGz36XePpm8fePgLbyKhPWFZGgBy62lTLkXwM+/lAdgmt01EABxiR8OyFADym1cAgCoAeoRzAPAlZpqCkkd/HECzLkerAogRKDgKQnl+vHDMAcA7/ZsYaVkKgOkeACgKgAyF+BaAJ7qevj4Iy1IAiE/BAIAqgJkUTUGR+YCTm5s1XkuS978uAKLCD2ecvFHbBzTqD+ARfkKyvo8o1RMA6jDowwa+6CNKdXxMVWEyeIZJYePzAQuf0CvSvp68w1YyrF0LiDQAfS3A4KS851+OHoRjoce3A/8BAJHh21JiAE5u3gnLCprVEACuWLoAeEOcgvCcTML/ugGI+q8PwHdE0geAPIHDZnXocrY1HQXVFYBcBc0CAACg6gDYdYhmzAfUEADWcP5IdlG6oFkAQAVAgx7QqAiAtPskGg0g5n9dALAUVP8Hte0AyHEzigQAPQ8QTosVNKsxAL6SEAdA/WYUGQC5CprVGADpLUD5ZpSGA4j7bwNA+s0otQcQXAcTluYDECWgB0DWzSj1BoB8v4r1+tWrYgT2AWTdjFJnAMx+AuAqZpAIsQ8g62aU/AAqIuT+63ElLsjnmI7MOSfZ7otxbnjs8y2AtgK+GSQaQEktoMkAEuZHAVAGrFMGALojBeYnAHAYAIDeSNHBLwPgUwAAGiNl9ssBbCUIaANgeFK+igDk/ssAbPX7/RgBXQAMqOoAUvxPAxAjAABmByD3PxVAlAAAmDUyzX8JAOp/lAAAmDEy1f8sAByBdAARAgBA2f8MAH0AYAVA4D9PAADMFJnhfyaAkAAAmCUyy38AUHkAAYG6AghXMrQAINN/IYCI/wGBmgJ4ZHM11Wz/lQC4NQbw4Uu/s9cCFPxXAdCvM4AgBdl4SjI586ikrVfjcvU+J6ldFe0DVBqAqAXEG4CfhGreArDKBaDkvwBA0n9KAADki1TzXxFAHwDkjUybA0gFIPQfE+AB0EksACCPVPU/AWBLAoC/MIq8p3vcOgDgVB4AVfsFACT+BwR89+NQAAAn5cM/F4BX+zTxSJoFT6DdAJD9slsdsgHI/UcA0hMTAKDuu4LDWhlAiv9iABwBABC557wsAH0AEHe/AIA0/6UAXADg+y87rM0CCAi0G0B04FMqgD4ASAw8ZwSQ6r8cgE+gzQDiA38rADgCbQOQOPEqGUCiCbQMQPLEdzYA6f6nAnDbDEBw3aFsAP2aADAiwSOPumYic8i1OilpswUIL7zN1AIyGkBqCyBNoPotwAAA8YXP8gFgAu0EkOFqeQB4Au0BkPPRx7TQLP8zAPRbCUA28WIFgNs+ALmfPU0JzfQ/C0C/fQDyP/xrFEDfbQyA2CPMksj8z56mhGb7nw2gH/+kRk3npRVA4o8KBBSikTM8ewoAsgHI0gp5mF8hbkYACv4rANhyo7+TUdN5aQOQbmuQjqR/eWNWACr+KwAICNQUQJavr19l+UjJVWUA0pvh8gO46tYXQLav6rbmALC1pei/GgBKoI4AFO5m0w1ga0vdfTUAjEDtAChkFc0AAu9VbM0FoGIt4PHtzZfvsw0JADX7dQLgDny9ABCBagF48s4d794rbEvkv0qfqhdANO00HMDjn3zEPSAg8N51TXStsfsNo5rB1hyRW6NKATh58754LUnyhwnLknEDbCtlNdWXGQCstFGQgkxEGq20HPc9tRaAVZoDAIBJ2gfsxD/s+oOdHbRzZ/nB6uL66s6D1eXlnQc76yTSJfsfJP7ANoo6t75+zl1fdpeXV5dX1113Z/mce2Z1Y31nfXl9Fb1aPOO6Z8ILq4tnzjzrooJzX8Ab3BXXLy1+3l268LlFdwltLNFY/GppCf24+MytC888cwXp1sVbS1cuXFhapFEX0L+LSxcuXrxyZfSN9X+Q32qnUgDwWrbCURAA0KkZzgMAgE7NcCYMAHQKAAAAAJBTu/EdG97uLtq5u+KtLW6s7XprKyu73u4GiXTJfi/x525Q1PmNjfPuxoq7srK2srbhursr592zaxsbuxsrG2vo1eJZ1z3LATh79lkXFZz/IgPwHC34ckEA3/zPtzb+S34r/FX2H+Yp6eZcI79WfSpNEQAoo9IUAYAyKk1RzVZRap4AgGUBAMsCAJYFACzLOAB2RTtyaU9Xpfc2N7m/7lusypubm3fIK62fNFOmAbA/fxyd4tdUqffhHU01kqmnkzfw9JPWT5otwwCCP38cnd7RVOmTd9/LilXVI2w54anzkyqorBQUneDUVClKFixv6BD9fHo/aabKAhCd4tdUKc4Z+loBngL0dH/STNW6BRDp6gce336L1t3MFqA3sxoAcHLTr6ehfUB0il9TpThbPPmtFq8C/zV/0kyVAgB/aT8PwF/oPOAlPckCn1GgDl37J80UnAlbFgCwLABgWQDAsgCAZQEAy2o2gMmlA9sfIUsAwLIAgGWZAzDpOo4zoB6gb5NLd9GOHt47SAR50z3HmT/yaNAg3J+obN9xFo6DePKzw/2nbNv/GcRXV8YAUOO7gwBAFxkxxG4MsXPRoOkecmu4cEyD5o/Y/kRl3fkjHMvi8c/T7RAUK2NfuD4SW12ZA3CZ2hwCGPiecnmBBY0xE+QkK2f7E5X55UF8LMWQ/egb+xn//yoocynokCaDMAUdhBvxoKFD1AvKD6OZJVIZMp7Fj+ejoMY43WBA/k8Cjm8i1ZPJTvh022HZRAbADxr6eZorJ/uFlWEAfjwAyBDLEikASNB4ju6KlMeNC1LU5aNIPCeyH6cg/6fw/6uWjAEgByf63U+3e6g/nBMDYEHTPXTIjsMgtj8eF3bCND7okH1BJ8xpjJL03AEdQd64JmkBLAgPG9HPoJztj8WRYSUbZrL3ZQ1DK+1/zU7Eqp1NZhIAsCwrAMh5bSzJKMVJAKjWV0XVqwU0UADAsgCAZQEAywIAlgUALAsAWNb/AeoDoN1HSnueAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb85"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb85-1"><a href="#cb85-1"></a><span class="kw">plot_precision_recall</span>(opt_cut_b_g)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABF1BMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6OmY6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq+SOTU2OTW6OTY6OyP+QOgCQOjqQZgCQZjqQZmaQkDqQkGaQkLaQtpCQttuQ27aQ2/+rbk2rbm6rbo6ryKur5P+2ZgC2Zjq2kDq2tpC2ttu229u22/+2///Ijk3I///bkDrbtmbbtpDb27bb29vb/7bb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T////PKuBlAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAQqElEQVR4nO2di3/bthHHaTeRpqZdN/cxd9mj2iPd4u7RrXO3Tlu3OFW61a1nRW0chf//3zEcCL5AgDwAB4KS7veJI+kEn3n3JV4kBGU5K6my1Adw7GIAicUAEosBJBYDSCwGkFgMILEYQGKFAlhnUvc+6Smzu7h/bX5hc5e9+WTg74Kb2lXjKDT/259cDYWQVkQAsmxpL+MBIDsdyJsRAByF5n895Ci1ggEUAT7L+tLq7G73j+ysv6AGoD6KIwWQr06vdhezdXb6JP/2YdmEPFtk9y7VSb/7fZadfFDWAChz74l89fkb2cnPdHd35wCg40k8eSOTxY0AiqNo+N9diFoxgDKxSAG8thCn4HYBbcGJSNdGPSmS1WwiNovqrXYDptx9t4K0dT2Vbc1yAEDp/3gAPMtmkMwzODXhPP9yAa9Fyp4JG+Rku/jBtcjorKTxASRTPptdi2ez0l2jDzB6EjWs4abTBOn+j6AJUgmTZzMEu13IM04Erp7lCsBrP/13/UImfAWnKPzO3bkG4ETQMnkS+u/nf1hkHQDNo2j6PxoAbz4pW/dNlYvtQjUs8o0VGN96ohIk39kAAPgd0XuU7iBfzxZvQWINnlSrZANQHkXt/wgA1AH2A8i/fChbFgQA4cPi6e48+/6f/vXVuaUJKv/YUQNonq1nzTeE/vO7bKk3QSYAueqDO54Kk7UPKP/YUTVBOgDRYYr56LcXcK6LBGzPi0Rtsh+Koeg/i5Q3O2EjgLvzohPWPYnMXsMQs/hFK4BmJ3xyGRhhZJEDKFuOs/JZOWSUL1TWGsNQI4BiXGTylDXcWAHU/qF3OIphKKjMx1Y09q99AM/E9Kmae8mJmBjcqIaqnogZAYhMLw2eignWJ2q4bwdQ+c/vHsI4d8Liq6GJxQASiwEkFgNILAaQWAwgsRhAYjGAxGIAicUAEisQwE2p+llb+2enyStaDEC30+QVLQag22nyihYD0O00eUWLAeh2mryixQB0O01e0WIAup0mr2ghALz41Rfy8eWHb7/3TfVQaEKJo7JHS7VZwwCev/2uBPDq08f51z8qH5QmlDgqe8RkmzQI4Ok7nxU14OVvv4DKoB7UuxNKHJU9arq7wjdBL379Tf7yN39XD8LwPaGBX53jFR7KfgoP4Pl7MvPqQb1HdibOifyE2yMm26SAGgAiSwQDsOvFKH3AnMhPsD1isk3CA3j16aNiFPQoyiiIAVgFAOAn7jyAAXiJLhEMwEuEiZgT+Qm10+QVLQag22nyihYD0O00eUWLAeh2mryiNR0ABQEG4CbKRDAAD1EmggF4iDIRDMBDlIlgAB4iTcScyE+YnSavaDEA3U6TV7QYgG6nyStaDEC30+QVrSkBAAIMwE20iWAAzqJNBANwFm0iGEBiHePioEnVAFEFuAa4iTgRDMBVxIloA7i9vSX2zwAG7BWA29si+7f95RkAdSLmeZX7Qrf95RkAZSJE4ueN3Bc2Qv84O01e0ZoIgPK0n3fK35L4d7DT5BWt9ABaTU4XgCLAAMwKC7jd3EvNu0Vvvf172WnyilYqAIbcSzEAN3kELFN/aytvACAJMACz3AJunPYuAIAAAzALHZje5DgBEAQYgFm4wAzNvbW8pQowALNwgRl6WzcAN9Y+gwEgAjONdhwBGH0M/V0/O01e0doXALmFAANABOYEwNINMwCzMIEZU+cMwNIIMYDhwIgAuPrxtdPkFa1BANXHgr9+G/RYPr7r8El51zPXCsCxKfO1R013V0MA2tsDwU4dTx833kYE5tx0GAkcLYDW1hCwScerv/698TYiMDoATvMJb3usTFs0BKC1OQpUBdEkQUOUo/YLErp1PqSe1UHuziavIQDN7YHk44tfNmvB8JnlPny01wBDHTiqGvC82qSj6geGA2MA/XLpA54+Kq14AD6XEEwEqquqDn787JESbdPwKKjaHqhoeKAavPobehhKDUB3ePAA1DxA7hdUtERiHvBONRAaDIwcgObx8AH0aygwr8vIDACvocD8ruMbCDTK31rseP+9dpq8ohUXgOedrH4AN5hbmwxApSoGgCYBBtAbGAMYVFQA3stJugTa5WN+boAmr2jtJ4CYnxugyStaewog4rJ1mryiFROA/5JCBoBVb2AxAcT73ABNXtGaJoAugW75WJ8boMkrWhEBhCwrRwCItWwdF/j29cuwzJXaYwCRlq3jAo8JYLvIQKdXiF/vCSyoiUABiLNsHZe3iAB2FzP8r/cEFtZG6wRIZ9q9dkuk8rRcFokX/21f/zjL7l/n+d15dvLnB1fbB38Up+zuIstmebeQE4C78+XeAIixatqSf5nTxbLK7eL+NZyqd+dn4uf0aruYFecu/NSFgEn/+WyqASQAAoeJSAARVk1bADy4aoCA3C7l4wZa6jUAEK/lC/GfXsgNQOFmXwDQL9q1RLqSjUsNAICIxmINLYx4Ie0b+QKansv6caBFMTVBGb4TtitwCQ9+66DR1gqJxDRO7ngAnGQ9s4IvFcwt9m554kW7vQiWrdyWTVDZ6mxOLg1N0IPeM/kAABCvGTUHKhMtcgqd7u7ipNsJQ87LTrgu5NMJiz4FWqCz/QFAu2bUEulG5OTkshiO/vz9YoQJqYVh6McKQK6GoZ1CbgDWp7J5QxGwBRZ+vd4FgIHAqNeCNpaxPma2Zp0H4MZCtsAIbpjMLXZjecIli4ioG4Jm39rKHBEAwiWLiKibgvba1sr4AaBogiju2boBoFuyiAmbUHE64QQAyFbM0eZ3UFGGoTTLRuYWu6080Yo5mryidUAAiFbM0eQVLQ2A6IGdLkWYA0sEgObvRkmzXTFqANHaTXcAJH2POdD/mRWWPdBhAaAY/poDHRNA0QzhLoYaA6NavTy32Pv8hK8XMgc6JoDVDKYCa9SNSVNgZOv3fQCErxcyBzoiAFEBYG7tPxNOCyB4vZA50HEBwDR4bwGErhcyBzoigN3FGVxhWvk2QYSf4Zpb7AN+wtYLmQMdsw/YLrJZvhpYTaFkCCA9gLD1QuZAxwTgom4AlJ/j9QUQtF7IHGg/gN2F/dpZ6C3Jar+gcqOg2jBdACHrhXwA9CXZDUDnUkRjv6Bif4L2BkLdAEg/yT632If9+K8X8gAAtyUvi7nT7qOPs+xsI68mw43JpQTQM6/C7xWhtkhpbSDUBUC7l4M/AP/1QjYAtx21a8DqLF+rhXGLGVju3r+UC1aq97wA1LulqI2CaoNxvyDaVTohXyxGeySDTRCk++7HV7uPLnP4gZdCwlK/Z3RsACDGofVtznq/ILVRUHMDoSnXAO/1Qr4AzuWyiRaAFTTk9XtYAHIGUBJo7ZgF/UDb0AlgQgA81wv5AijO8AYAuLleNEG2s98MoH1Tvt3kCwD70geYDgblxxMAtPOb+9cNAGBUixiL97wA1PsFqY2CGhsITR6A13ohXwByKNSsAXBv/d7DZf0eEoC2KqLeL0htFNQ/D5gWAJ/1Qj4AAmQaBW38V0UQ7+kWDMDjhDAHOioAB+kBTA6A+8TQHCgD8PbjenHQHOioAETnUXzwYFh6ABME4Hp/whzomABW978qboohpAcwRQCO96jNgY4IQF6PW3reEZskALf1QuZAGUCIn/0CkK+hCfJcHc0AXHXo8wD9qPYCgIP0ABiAq4g/Kc8AXEW8VwQDcJWxE0bNwaS0ACg/rQg6SgAhnw9gAKUG1kL0AXCSFgADKMUAmi+iA3BejNIDYOW0T4oWwCEDmHfUAOC6GMUOANaErr13qqHeOiZkWUpD6mvsWxK9nIuLoRrguBilcRztl3IS4DAT0M6gidaAQvWKKvFCjjOM5YMBYBajWAHISQAsDDpAALVdMIgGALcY5cgB3ESsAbjFKAzgpp1/wiYItRiFAdxYb1X6AAhQB0BW6oBmwmb7JAG4qh0A6dZhoKgALPeKzYEeK4C52U7jnwEg7DDjpPBjsjMApB0YMACz2gFEAnBTVwRa/8ZrROE5ddKeAAC7kQEDaAYQF8CNqSIwgGYA0QGA2gzC/Js2eKLJK1r7B+CmVREYQDOAsQCASIanDCDIDhUh0I9hizOavKK1zwBAgcPTwwJAu48/2m6cIzAA/4T62Oc6hcMBUH8q9cUvYK+IcteaQq3AEgKQUosVart208Xkp7vLYsxsGzQEoN6dBvYngP0iil1rlFqBpQagVK4a6dx2NJWfPIB6Z4LngOHpY7VrjVIrsIkAKO3zgwDQ3ptDPFO71uSG7WpG+0IppCSAoUKpD3roAFu708A+EWrXGvV268yaWA1A9QHdjV5jZtsglxrw8sNHylr1A63AJgfA65u5Y2XaInwfIEZBVffLAMg0PAoqd6dR+Ve71qi3W4HtJYDOVscxs20Qch4gKgGM/6H7VbvWFGoFxgA8dCAzYYv9uAB4bpIX1d65SsQAxrZrCBjA+PahW5jadvc0eUXrCAC0EDCANPa+e8gMYBR78zJ1WwxgJLtEwABS2i1rTNtfeUKTV7SOCoBhXgBiAGPaDQgYwLj2DgEGMLJdJ3AgAMzfGTJFACYCDGBUu0aAAYxubxNgAOPbW4MhBpDC3iawrwAaSr3AxlmNzYgSHvvx1oBmHdjfGtAIbP8A1AQYQCJ7SYABpLJXBBhAIrsiwACS2QsCBwDAnP/pJNpql1MyBpDSPmcAie3z1ke2xxUDAM0ZQGL7nJugydhp8ooWA9DtNHlFiwHodpq8osUAdDtNXtFiALqdJq9oMQDdTpNXtBiAbqfJK1pUACz5n0BCXe00eUWLAeh2mryi5bBfkHpWG3IGEC78fkHqWW0AVYfNADyF3ytCPWtsHpEzgHDhd0tRz2pDZ78glofw+wWpZ60NhHLDGZTqzKWyR064roAaAJpQ4qjskROui6oPSJ84KnvkhOvC7xekntUG0IQSR2WPmu6u8PsF9c8D0ieOyh433x3Rfp9wysRR2WnyihYD0O00eUWLAeh2mryixQB0O01e0aL6hIzrnHjfy5OJAfiVJxMD8CtPJgbgV55MdJ+SZHmJASQWA0gsBpBYDCCxggD0r5joK2/6Rqa+8qog1n9jp/c+/3l1ZwN3/DEUAmBgxURPeeM3MvWUVwXR/kFw77Tfvyij8OCOP4pCAAzcLespb/xGpp7yqiDaf14wHvCfP33ns6I47vijKATAwP3invKg1jcyDZRXBV38w5k84D+vmiDc8UdRCICBFRM95XPTNzL1lFcFHfzLxwH/eQUAd/xRlKoGGL6Rqbe8LOjg/3nVmfb2A/tdA/z7AOM3MvWVLwo69AFPH9W/1hPCi73uAwZWTPSUN34jU095VRDtXzU8A/7zCgDu+KOIYB5gWzHRU974jUx9/lVBrP+yIRnwXwDAH38M8Uw4sRhAYjGAxGIAicUAEosBJNYEAewu5FdhZ2f2ItvXL8W/8Q4poiYJQKZ+u1haizCAmFIA8tXMWoQBxFQTADRHp1fFI/DYLmTbxABiSgHYZEvxVGR9ff8aHu/Ol/InX59eMYCYKjthkesNnP0i62W2v7vOy/aHAUSTrAHbBVSDtRoPSRBSG/HyhAFEVdEEQQsErY80lQDuzk8uuQbEluoDViLpm5Miy2W2NwBkwzUgrqp5wEw8FRkXCYdOGH4AyHbBAOKqHIauxXgTOmSoBeUwdCVe/kV2ygyARSIGkFgMILEYQGIxgMRiAInFABKLASQWA0gsBpBY/wc9/dLW4gSXCAAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb86"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb86-1"><a href="#cb86-1"></a><span class="kw">plot_sensitivity_specificity</span>(opt_cut_b_g)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABU1BMVEUAAAAAADoAAGYAOjoAOmYAOpAAZpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOjpmOmZmOpBmZgBmZjpmZmZmkJBmkLZmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq8huq+SOTU2OTW6OTY6ObquOyP+QOgCQOjqQOmaQZgCQZjqQZmaQkDqQkGaQkLaQtpCQtraQttuQ27aQ2/+rbk2rbm6rbo6rjk2ryKur5P+2ZgC2Zjq2kDq2kGa2tpC2ttu225C229u22/+2/7a2/9u2///Ijk3I///bkDrbtmbbtpDb25Db27bb29vb2//b/7bb/9vb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/5Kv//7b//8j//9v//+T///++oKptAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAXr0lEQVR4nO2d+7/bNhmHfboeEnIK7HbG5XTlNmBNYXRAT7lsjHE2YNkYXbfTpdtgK5wlTZum/v9/Qq8utmRJtm627FjfT0+T2EkcvY/1SpZev8rypKjKYv+AsSsBiKwEILISgMhKACIrAYisBCCyEoDI8gCw++e3suyZnz9oetvp4YN886Nz8kTaany004nLgfP3s+wb1wwOvMywLr+diz80t/qV9nIHsDslP/mwwRC4OMtLKgCw1fxwk+KZxYHX6I3fMDkwBZBl8yoAm19pL3cA6+zwbp4/OkW/uFnqQrgBsDrwWvkuFQCy6T7iOhAAy4MzeNiegGEeXcuy79zF59eH38oOfoE27f6YZQfX8aav4Jw9hnKRwqDPFFuJhbYnx/Rr7yP3Ap/nvim/P8suf1QA4A6MPrZG+/Br9hPI28/Iub5Ah7j0CTZouVV5YGbmxaVzAgC+7/JdUt/Yj2tBPjWg/FmbGVReZBjmHubs2VwEsJnN8UfnxVYCcE2MWniCOfdNZNvlaxP5wNuT52bkwOVPwF6H/BYeALdVfeAKgDX9vv4CQM3bwXN/+Q88253Cqf7ZbAJmmyAXm02QQV5AjR3exLUB5ORihcSFXuD/aKXfnV66m9OPsW/anhy8DR5nIh94e4LetPsgmwg/Adn0PrIZbQPm9MD8VvnApQua0Pdn1wG9ZUtlL59u6Bd/nBEvs5nhU2QJdoVfCyfXZvbMzz6CrWIjDP4Ddpd2ABOVHijP//3hn2bYCuU3wU6AIh14e4Lth0xZ/gT6LBcBiFsVB2aNMK069HAL9LK/AJB2n/0QnVXr4seTMxw3mFD7s+fvVgCAIZhRSNHAxoUjoJ6Enobkm4iz5rqh5YGJGwGq5U8gXi6XAPBbFQemAGhDxt6/7jsA8muVAPLProEDPhcBYMdcVAdcNPSycATIpzz35399edIIgBzYC4BwYN7MwwBAaz+t/6yvxwNA+uIPuLnlrwOWB38Fq3F2WGevFo6AfFPRdOBvIt6D2lo8MHkO3qr8CcYuSDiwDKD3LmgB/iXf3Z/hFu5t6JmzHhw5b19E5/sHtFIsqWsF+7wCZ3SxFZ5+m/NAqOVFHUDWXMM3oUb4Om6T5QOjCvNi0QgXPwF92+aEfoPQCBdb5QNXAfCNcPmmFuQOABWetVqsETjmzlvajWQlpt1QfB0LRS225lDMovDlVS5Xl9ZiN5Q78PbkmRn2c9xPoM/mYg0Qt8oHrgIouqE5e3NL8h0LOngeX/tskL9/5rrggvCF2AvE6+fba9nkK3KBucSncrH1gdjBwZc/by/4uoRbk8ufVMaC8IGRX0KGEn8CvuQ6uF5xQeJWxYGrAPD3XYZvpm9uSX0YDVUPF5ioaBi6PnA49QDAI36w0k5+ADwOHE7RAYBHd3axPgC8DhxO0QGgZvcF5w/7APA6cDhFBzB2JQCRlQBEVgIQWQlAZCUAkeUM4IKpfFZVsSfT7jHeEfQjij0hbWqlOACkDS42c/9IApAAFEouiG6JpQSAboml5ILollhKAOiWWKoF8PhXn+LHJ6+/9P3/FQ9ENsVMLkivOgAPX/oeBvD0nVv5f3/AHqhsipkA6FUD4N53/0FqwJPffgqVgT7QvTbFTC5ILxMX9Pi1/+VPfvMefUAbvolUvGm1sj+IzwDI1EPwafEvukwAPPw+tjx9oPvKc2e1WjWcZ/4uiDOi/BHYJm2uO8rgawBIKIyaQRAAzOqqj4DhpyNwQUZtgAKBZxsgnvAVm5Vn/QgAPH3nJukF3dT0gojtJQQ+ACRfw/ZMifHrjOmyp207a9UIAP4arwOo6SueyN0FyZ4e9qh9/T4DqJOmMDwCVwAK8yu3NXzXOAHw1cDRBUmmxsYf5XVAnSqFEXwPReAEoHqqT4s9xsZ02RPSplYKVgPEFhhXAwcXJJqf9/kJQEWNxURXyOypKQC+U19tcBOAigyKWbQGhi6Iv9xSXPAaG9NlT0ibWikkAMV1AGFgBIBvahVdngSgIkVh1BdiCIGBC5ryl7VWAzt7BGB7YnE7mkUxV40Apo19nVEAwDdXm940YlPMrFo3eCLCCGeuPv3rDrNXACzuG1EXRjcUUSFQAKgO+eQ6+/cJwKO75HFzFOD+VbkNAAQGtyZrCsNbWg+AunrV4L61NTsHEMTuhaoAlpCnocwhopdNMbNqA51fqKZWEJKRA4DbpEmKhOYqYFNM8Dgcgek0Uzuaad9qwOborRlyyZBBZE7MA0kRYMuVN7JLd45IgiSvW2XFXpBFVgRtYVbynoxtZ3MsQ5mU38zwbfUkuwS+aXx5+ABqAL7FGz0hqRR87jYWAZie/nUAFMXMGqa3sKaO1mwXwJz9d3S2Jjlu5gQA2ebvjroBgE2vmiko5Ty/2K4LOiv/oxmFjjEAus3MVmYAiryNZj6trpgrcc80EzZfyC6oecyzDwBoz6QtALmnM+MkRgpNi4MU2zNhd96H+BylBAAsuxYPIKwLspLxeca3udJcWfXCt881gGRfOTiD87TcNpFzeVmpAIC+liXiCdoGgIFLj1N1TgGs2RkA3OWEWrDIJsK2YN1QKzUUc0X3YBNn4nayIVSIw56NBRmrqZgrbmRNBUB9KTZyALvTY2OPZlRMamW+00OuxwJac58ALEgyMe9uKNvD7Cz0OkPHmOwRgNAXYsV5Lgz/Qx0YylBEB2oTQHHhVQ7/0+0JQCHBBdGMvh4TMrzKsWVmb1olVskFlRJ7QevMOFd7YzG5sWXc6+Qa3lUCUKi1bigJ8qG9zmqvJ1tV5W7NNgF8rZarzVRqCwDr62DLNscFJQCBhyKEwf2pQVzQSrej0Zr7AsBWtcUUBvenJoFZCQCuARbD0XXFnPJ7VN1+RTe0EtFubs0EQCpMJcxN0e1PAAqpZsQ82wBxekscjtabJgGwnBHTF0YAQIajV+SuGf5P0srRmnsEwEr6wsgTvEbh6QMEUBdEuLlyTv5xr2oBhOuGOgIo5nAsrRkPAPYYS30QoWjyJgCyituC//sS6BZ+/F5zthQOgGI+gO5ROKHBAcAm3V6FlVTgpN08+wccQLfGp/Dmysdo650rd559kO9uv4Vf/WSO3lThpQcgpgeCTB33bnG7tYXhYs2V8wH0MzKBXgKQxkxWJYBi0c/Fcb4kqz2tDx9sXz7Hr6gL+uT2GbL65/jVGr2hOtKmnxETUkNAko6n777HvVdbmKm8x/AesVUPAdQ3wmuylBBUgpexk0d/8Crn2oAlonNMnm5//OD9ahiLfkZMSI4CVQG5JHBEeSVfUFWKIB/DfEHNiYe6lkEvaHN0htvOgzMKABYDhCWzGIDNs1/dPiNPd7f/+uNqi6GfkOHTA+HHx7/ka4H2bCpqgDwfIH5G5YQGVQPWZK23OTidnDQJtKVFnqgAgBqAZ+mrfPmKNNavB8DXgIdFko6iHdAWZlo8NgCQCQwMALYXeJzFMTU5AFhD+BYHAF3hHrNuqCKQTj8jxrcB926yt5gDsG8DhgYAXwdApBZ6JE4H/y1oLwiWlr1DrU5ene9+L3VF9TNiXHog4nigGjz9W2M31ANAzVRZLwFYayMvG9R4HYDzBRFPhK4Dvlt0hLSFmRb/N7kg2QntOQDVqpThhyKmxX/NAGQC9tYcEACVRABLdClXc23NS1cYbvrX3gWNHcDi8MuTuWdkXAJgp0o3FHpWfoFZ04vKjNhFw3yAeG+3lkACUJGuMFNpRiwBqJN4HQAuyDMyzs8FjRxAiMg4TwDaVmAcACykK8zUZEKm8hn+3u4EwFC6wvgC0BEYB4AlWfjeRLrC+LqgUQMIEJ7Oh+FaAOBiQ0cMIMQNGlOjwTjJAHxwrppAAlCRpjCOAPgdwwFQE5aCx5+z51gkhEFYCjY9GdyO4IKEjygJ9BBAQ1hKxeIGAFhckFFokKYwzgC40MShAGBhKduX32DzMqQRxRPEH6OHt9A7ypeWYSmuABxdUE8ByAsBfS2FpWxPDh+A3ybRKbvTeb4+/LyYEuZeNoalxATA76hdkMZ4RweNMA1LQa5od/uMi07holL4lw1hKUv8XZ5DESHagOEAyElYCkQCLeYsOgU7GS4qpXxZH5YCLTBJgWZGQCl15h+juKCV5nlM1QMowlIQAKgBZXSKpgbUhqVAVBxJhWY2JaY+m6ZmV8Kqc5OPjlZUgR7WABaWsj2ZQCAKjU4Bp7+5codvA9jL2rAU/G24H+pzHeABgN8xDAAsLGV79adcdArr9rDTn71sCkspO7V+AFTFtF/GqmZRMuMdnV0J02hQA9WFpUBVIdPBBmlz2wGw2ncA9WEpkJJ0hsPbjYZD1YXxckF8dLREoMcAvMT1RxbQA92dGq4hoC5MoDZgnADspC5MqDZAtxyHkTFd9qgLOTYAqwTAXMrCTA2vhDWmEaKjlUsOGBnTZU9Im1qpVwAuEgBzKQujSYPrtp5wdc0HY2O67AlpUyv1DMBK+2LvAVjNxuRtuaARAwDRqAj3CzHNWteua8orVwQy/q7hAfCflA/aBiQAtVIWJgAAnRMaAwD/wKwALqiyLrHJR0Lsace6BlJER7uHJk5Nr4SNrTk2ABZSFaaSHtS7DbjgCCQAFakKEwSAxgeNAgAsEldeBxT5gliioHJD3qILqgxBNKZ03SMA4v2RXL4gkp9ATCCkKoy0NCqVextw0ZxRdI8AiJdgZa4ImiJFSCDUVRswKgAwLVyqzJZCEwWVG3T5gnTrgRnmC2Ja1b7cM4nR0fwlWJkviCYK4hMIteqClPkLxlADxKyJQsYsaAfEDarCtNIGjAiAKNHlIwCNbUB1lYAwbcBFQzq/vQVQ5guiiYK4BEJ5ywAUXdFRAFBdB8A5TxMFNV0HTI0H45qsqWgFxgBgd3qMOkLu8wHV9dlCtQEXodf96SsAMD2J8DX4oKIwrbUBYwKwnLjPBwQEICdUHAMASNyKrO98f0BAFzRSAHCTxiIzCw5VFCZsG1BZcMA+j80AAdhIUZgW2wDNog/1HxkbAP3y2CEA5DoCCQB7Ur0MCNkNvSC5tKxunxwfAHMX5ArA6uax8QHQFjOMC7L/iMWekDa10nAAsLPf+MaB4QHYnhjengSSC9OyCyosbxq3PjwA+DYxs8TFMQFcGMYstgBAcaO1r6ouyDhZhFyYtl2QhsCeAahfnYyTVBjJ/m0CuPBagrjPAJZZNjG6V1sqjHQZENoFaSKGWgWAZ0jmxPDov83Rm8RJb0k2ps2VN9DJujvFRpPeZA8Avsk0QFoqjNQEBAdgmckpAABi09m8sO0MknGQjDLbk0vnm9mEhFPBX/kmYGKUgj6v9oKMBqI10sWk5NZhKVpJASptR6wUS0EWtp3jR3yCLgHAnJ6t6L/qm8wOEe7+gPbbAPXqe626oAV2LiWAK+QWIjxiD9lQYDuewMKu56x8NF6ZNiCASC7IYc0Hm0YYZ+LrBEC5oLPbChodAFAPxgWZKagpKbKlYFvmgpjXYUmWRBekTlJZB8B3QecOXFDno6HY0HixQpLGRGqEweasES7f5NoIW0kqTEwAlsl2zWvAOiNLtUF39NWrpIcJpoVu6JsUQE67odKb7AD4LugsXwa04IK0w9HeHVRTe3HSRY/YXa0FqwFyE9AlAOWu9gCwDJNKxQJQU8xgLqg3EzLLmr6KOwCrjKHVwnQDoP489xiotjBZWIn3CRvfpBrLBTU4GsMEK70FkONLP7e4oF4AcN/TgmnNpGgDFo69IH0xu3JBlTcMEsDCsBcaC0BzOkWDm+uNAXB54rIucsYZ+588mgsyymfpMFOgLmTHAHyGIhT2jweAXw3C8DDqQnZdA2xUKYzC/u24oE7DUroE4DkU0TsAljMF6kIOqQZ05YKMa0DNTTV9BeAzIdMdgC4Ds4YEoK6YQV2QBQCLqRp1ITsFIM2IlXelPr4BuSJY1prhALBYG7cHAKrd0DI7DeQngHwRJGtNCAAduiDDmQIXALtTfRCh05SkqDIzwUPAcO8WzVoTHYDlkJvZNKYLgDoj2wOQuqFibg70jGatyVXpamqCgvJwcUFU3eWvqQWAV+chyxfubr+ZZcdrHFYLE5PznC7f09yc6k0hZKeBPBE0aw3dLZ5NqgrQVhtgP+hscPWsA5BJEmsAW75wkkOUHKyWd/WMrRpG9jkD4GvAk9dv0q1FOyAWRtULbc0FGSaV1jcb4VwQXb5wdxsv5caW8yELGtJ9NgCIG2K5OsrsNI9vFM3vMAE0f8YVAFm+UACAB5TLfTYAFhMIeKTVpsxOQ+1Ps9aoAdQaILALcgLQcFOHKwByhnMAoCtJl+5sPPurANBnwZmtq+lqoP8PzS/NWhMfQGd3yDT3gujyhSUAumJhuc8OAMzLu1wJWwLwdEGuAGqywLoCIMsX8i4IXdBevjYv91kA2J0eQ8DLwiE2dBgAau6qcQEQRNWMWRPDlQyTCwqkMMPRSvu3CKCrm/SGA6BjF+QOQDdZqS5kBBfkNiM2HAC6JKTqQnYMwDyoPY/tghyDT/R71IX8Wi0vi1cUJioiAXCWPnl3g4TC2ALwdkE+AJTT9epCdt0GmF2CEQnF7B6AwdIa2j2q6Xp1ITsG4B6W0r0L8gHQWxdkJaEwCYCzggBQ90J764IuFPES6kJ23QbAQNLcLG9rdAB+2VJ6CmBx+CUZkTYQX0y1/dt1QZ2kq+nYBeHh6LnDcHQUACyVa58AGMdClIoDIIQLuqAIXAFUpvaVsnVBngDyJbggh8U8YwHwy5YiTmwqVQ/AIxhFA4As5mlxrypVfVRQ8LggXqtWg4QgLEX+KyPjnINRSoXphjacge20AVRtpqupbwM8glHCAtD0QrtwQbDDPa86P6jdBEDpgpyDUZQAluhjxlkrewRAm9o+aA2oB2AbjKICwBJQ2DfCGvt35ILkBtXlyxoB1Lsg22AUBQBYPSPHyZ6slzDpAQBlJeiyDbAMRlEAwLMxLPtW2wACuyCsNtLV1LugIBIB4JN/oADkSjAsADAdRoaBzAKDuML0wQVhVRAMbCwInf0kD6lZxhquML0B4Bi2i7co1SkA6MmStH9GH+QK0xMXhLVyDFhRF7JLF2Sr8qfrLgOiAOARjAZAYzG7c0FYLjMFIW1qpX0E4DJTENKmVooDoE0XhFW/CnoC0DoAXAkSAPakaxeEtdIvwJoAdAGgZv3PBKB9F0T2mC8AGtKmVtpvAFALfHJFdCB/AFr7R3dBdI8CwTAAlPmC6LNyQz4kAEbjpK0bWic9gDJfEH1WbgAVP90FQHcuiKriiQYBoMwVQZ9xySPyoQG4aBqm68TYKplkS6HPyg1yviDLg4SLC7JRd3mGLGSSL4g+ExII5TVnk7QndhtgsKcTY6vkVANANsXshQuq39OJsVXybwMSAC/V9YJuFr2gm6QXdFPVC0ouyEuN1wFwztdeByQAXgq3kFtNMZML0isBoFtiKQ6A5IIKJQB0Syz5X5NaXRO7f6abj3SvBCCyEoDISgAiK864ZFKhBCCyEoDISgAiKwGILE8AwgCpoR5yazEZ6uk7XN52Mz2+YX2UKPIDIAZKmAnGt20/c+8Wnhe10JPX0S+zPjciyA+AOElmLsvPwGFsjyDMn/ZYfgBci2lZAx6/9ndbFzQSAGKghKke37C15o1b2KIWwi7Itt2IoTg1wPIzLodBjfCv3917AK5tgLgqYvNhfufE2aHp6F6+vaCb1j0aF7d1z94FoSPY/rIoinAdICzGZHwY2049utoYQi80XQnHVgIQWQlAZCUAkZUARFYCEFl9AbCsZtt8dFd6z+borG73MNUPALtTSJO25HJ18cZWqGH3gNQPADRNHZcwMwHoUtwCZtiym6O3Zll2vDl6M8uAye40yyZ4H9kzh1zl9jnGe6leAODOZwrgDB43s0vnkMiR/cHmGQKyvHSeakBQceseiADm+DlOZLrGVmebEoCg0tUAeI7cE07FTLbxu/dDvQDA2gDIgp0AxBDt/jDnLrigK+cso3UC0JrK6wBIng/JY3FCfkUjTAFYLfzaa/UDAE7bS66EoYf56tUztGGCu6GQzZrvhhIAsDv2Tw6jvgBQaX/8TI0SgMhKACKrzwBGoQQgshKAyEoAIisBiKwEILISgMj6P3VuGSg90qcTAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb87"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb87-1"><a href="#cb87-1"></a><span class="kw">plot_roc</span>(opt_cut_b_g)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABDlBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QzMzM6AAA6ADo6AGY6OgA6OmY6ZpA6ZrY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmZmZmkJBmkLZmkNtmtrZmtttmtv9uTU1uTW5uTY5ubqtuq+SOTU2OTW6OTY6OyP+QOgCQOjqQZjqQkDqQkGaQtpCQttuQ27aQ2/+rbk2rbm6rbo6ryKur5P+2ZgC2Zjq2kGa22/+2/9u2///Ijk3I///bkDrbkGbbtmbbtpDb25Db29vb2//b/7bb/9vb///kq27k///r6+vy8vL4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///8S4JhFAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAQVElEQVR4nO2dC3vbthWGYTeWq3hL1cvcZVftkm5Rs2Rzt1Zb51TtuovbzIqyOAr//x8ZDgnxcgiAB+QhQUnnex5H0ieIIb6XAEgJglQiiioVeweOXQIgsgRAZAmAyBIAkSUAIksARJYAiKyuAFYq0w+/TB9u//YDpd752U1SevTLjv/FYYsLgDq91o9ePcwe3EtxbKbZo/e67+bhqjOANPhk+1c10/8u1AMd/asn6ky3gbtL9d6/kuTbh2rOsKOHKiYAOu0ZNIeZcVXp0WZ6Zrqk5JupuncFoMDZLibwt1KnT9KN3F1OsjZkurPjEBOA/y112tuFoaGzPLspHv1zV3gNHdLJVQXAO1N19u10nj47N72WLnI04hwD0kM40/L0uvTIaLvQyX6TkioAZF0XPNav0SX0kP2PKX7lAYsJwMl7N0kjgM3U9FBlAGkzWWkyUN6U2PVrxyCWLuib6QPo5ZsBmNG4DCAdHyB46IHWpjkdUR/EMwasVZaofQz4+kE2qroBwI1mJgDCZXqLZXrGYz8Lurs0JPIuCBpI1mAMAN0H/QkaTI7oeMQEIAvZdR2Qd/36yN7oxqF5/RwKFwA205/A1YIu8Uf9+oWMAWTtxsuVgh7/znolnI8FWQ8zN3fuPSwAaBqmL0s1s/xPByouADpB6D2q7wW9ejKtvhekL8RO0odfT9WD/xRjwA6ghvbwyN49kndDI0sARJYAiCwBEFkCILIEQGQJgMgSAJElACJLAERWRwAvdiruVbV/Pk+uZAkA7PPkSpYAwD5PrmQJAOzz5EqWAMA+T65kCQDs8+RKlgDAPk+uZBEAvP7NV+ntm0/e/+j7/CbTiILj8nuL2q5mAC/f/zAF8Pazx8l3P9rdGI0oOC6/x7BtagTw/IPPsxbw5vdfQWMwN+bZEQXH5fcad130Luj1b79P3vzuC3OjjXe1et67vnWb6fz8PNou0AG8/ChN3tyY50Z05Pr9W6tenKd6sZ8tABQtUI9vjxqXz6PH2+kxbJvoAEY4BjgOa/92zlH0ePs9hm0THcDbzx5lZ0GPYpwFuaIO2Y41+Vr5/rK2iggA/oa6DrBm3W37RfT7eBbkU2AQyPd0IUHb8fj4qBcADVGztSR7V3/sAIrQexwzfF39cQMoH/K9AChFf0hXwj7RK4Z6HF4A9f5GAFT8en/PBcDV3wiAVEqpF9Wup3tAO2XRH8ObcT75KwbfNnKcV3YJrnzUCwBfxSB+xRiQ5dRSAPgqpgEoHgDOU0sB4K2YcjUAekAm+uHe7OPJlax+Adx2CKLa3wgAuxoq1g6Arb8RAHY1VCwUgPtdBAFgV0PF6AB20Q8XtMvnyZWsMQAoH/UCIEz+it2Sguj5vD7U58mVrPgAzh1+aHBcPk+uZPX6FaVbQpl4M3LGodgtwD07IZbPkytZcQG0+ciwb58nV7KiAmB9H5/L58mVrD4BNM3bGebNtVCfJ1eyIgKIfMXr8nlyJSseAPu8BQEQJm/FvAAapwjG8nlyJSsWgPbzdvr2eXIlq0cAt54K9/BhOpfPkytZcQAM+QlXqM+TK1k9AcinflorPIq3nV0+T65k9QXAV+Fzh98tOC6fJ1eyhgdAnacfy+fJlax+ANw6/Bf56U/8oF0+T65kDQ1g+FkOoT5PrmQNAkDl07NG9NGjy+fJlawhAKQr0qePbF8L7TvQUJ8nV7J6AYC+iKHUeW1WZ/ygXT5PrmQNBKA2RTR+0C6fJ1eyhgDwop7/CIJ2+Ty5ktUIIP9a8Hfvgx6ntx/6vylfAzBEcFx+r3HX1QSgujwQrNTx/HHpaWvFbnHFBIBbTQAqS0PAIh1v//xF6WlrxQRAgJoAVBZHgaaguyToiBLPekF4OtCxT/3xqglAeXmg9Pb1r8utwHpk4RYwzg/fXX6vcdcV0gJe5ot05OOAtWICIEAhY8DzRzvXC6D2hWwB4FHzWVC+PFDW8UAzePsX32koBjDS6Scuv8+0LSJeB6TrBWU9kb4O+CA/EbJVTACEqIcrYQEQIn4At6hiY52A5fJ5ciVLAGCfJ1eyBAD2eXIlSwBgnydXstgBoLVRxjsH1OXz5EqWAMA+T65kCQDs8+RKlgDAPk+uZHEDwMtjjXcausvnyZWsOoC7y5Mr8strFRAAgbK1gKVSZze0l9cqcCwANvfpR6lX9i7o7lKpGeXltQogACP+IobLp+XWM4AMwel148trFRAAgbIDWCk10V1Rc0eEK1DrgeIHGuo7arqZKqXmWfD6n839p1lHrY/Uk2cX15uLP+gDdrtIg6sVCgMAW5nDnXVzE8AVOFgAWabTeZ7t9Oxmu5jo/Gf67/R6M51AcpP0rygETCaBAGBzTblTAYz5u2Au3wHg4roEArKdp7fpQboCAHNzxOp/cKFQANTD/5gAwKnhpAwAgOioVtDD6Aepv04fQNdzVdyaPHsCgHTQM4J0b186uPsCsFI7+bsuI3QE1RrACI7oUN+LYF7JdtcF7XqdNVzC1rqgC++R7GwBNKEKHC6ANGidKQy628VJfRCGzHeDcFGozSAcJFSBwwWQrHWncHKVnY7+4uPsDBOihdPQpwZAYk5Da4XoAPThD5dgoBaDsOV9iPiBhvqEWtfgOM71KVdrrC2g3gBGEGioHxYAdPvOXkYAtPEDE1h5zlfaAYARhKxqBY4RQFdZWsBKmbciCKpWoApg5EsSuHzWeJtl74KW2YjfrGoFBEC4nGPAsvNZkACgyNkCaO9EVCpgGQJGEGiozxpvsywAyP1P4gXQ9ScFY/n2iv7Xrta55+J8K0IAtBDndYAAaCHOtyIqAEa/MJPLt1d0/1qAAKCJ8xMxAdBCAgD79ooOBqDLJ2JlAONfG87l2ys6fAugqVwBWwMYQaChvr2ifgDbhXsiYfBHkkj5ekG7hYIKQwDsAPhCDgNQOw0trReUrU9QXUCoXAFbDzSCQEP9FgDgY8mrdNJEsv30qVKzdTq1Fj6YnKcAsucoAGoq1oowS6RUFhA6JgC3NVVbwHKWrMzEuOkEnLuPr9IJK/lzrQAUq6WYhYIKA68XVJ4UdFATgkCNXRDEfffj6+2nVwn8wUMt7RTPWTdsAZB1Q6bJFOsFmYWCygsIHVULaAJwmU6bqABI31QunqMCWE5gsqNpMpUVs2AcqBrlCpQA7MMCrS6/LYDsCC8BgNPJrAtyHf12APp10JGZC7Fql68BkMaAIwQA/fz67KYEAEwziTF7jg4APpdf786CdusFmYWCSgsICYAygPRUqNwC4KL23sN58RwRwHYxg8kuSzNqF+sFmYWCCNcBe7FEsctvA6CDLGOAPn2dUL4dAypVwNoARhBoqG+v6JAAQlSqgABoJQGAfXtFB+6C2nwiZh0CRhBoqG+v6IAAmia0V1SqAACo/UzACAIN9e0VHRBA27ejUwARg+Py7RUdtAW0BrBXi3S7fHtFhxwDQr6eV6qAAGglWxfUblqKAGgl1tNQARAuTgD7tUi3y7dXdFAAK6WyLyA3q1QBAdBKts8Dzv6dvSNNUKkCAqCVHG9Hz8MnZt3u2SLdLt9e0VAADXMhBIDbjw4gWUEXRPyuZKkCxwwgeDKKF0D6rXzaknHHBOC8phKA0MkofgDtdHAzUSpqagGBk1EK8V0H7Nsq6S6/MwDKZBQPgJV+CXnVSgFQA0CbjOIGsFt8IngQPsdLpQwdHJffEQBtMooTwHaRjuNzOBeiXAoXuy0AQiajOAGkn8bsVt4SAFQAHWQDkB78AqCioQDAx2HZ20C0iUH5bp/XlssaOjgu317RoQDA0Z+tQUpbsSbfbQHQUrXT0CWcgcKSf6SX57stAFqK60JMALSUAMB+90yDJACwz5MrWQIA+zy5kiUAsM+TK1kCAPs8uZLFBsCe/wgCDfV5ciVLAGCfJ1eyBAD2eXIlSwBgnydXsgQA9nlyJStguZrXv4K1Inar1mTKd1sAtFQTgGJ1GlifANaLyFatMcp3WwC0FH25mpeA4fljs2qNUb7bAqCl6MvVgPQ9s2pNgparOce/YCWiqQlAZXUaWCfCrFpjns6PG2kBLRXSAt588si4+TiQ77YAaCn6GKDPgvLhVwCwqfksaLc6jcnfrFpjns53WwC0FPE6QDcCOP+H4desWpMp320B0FJyJYx9nlzJEgDY58mVLAGAfZ5cyRIA2OfJlSwBgH2eXMkSANjnyZUsAYB9nlzJEgDY58mVLAGAfZ5cyRIA2OfJlSwBgH2eXMkSANjnyZUsAYB9nlzJEgDY58mVLAGAfZ5cyRIA2OfJlSwBgH2eXMkSANjnyZUsrvWCZF5QS0kLwD5PrmQJAOzz5EqWAMA+T65kCQDs8+RKlgDAPk+uZAkA7PPkSpYAwD5PrmQJAOzz5EqWAMA+T65kCQDs8+RKlgDAPk+uZAkA7PPkSpYAwD5PrmQJAOzz5EqWAMA+T65kCQDs8+RKlgDAPk+uZAkA7PPkSpYAwD5PrmQJAOzz5EpWwHpB5l5hJAKgu+jrBZl7hQHKd1sAtBR9rQhzr7R4RCIAuou+Woq5VxiV9YJuZV5QO9HXCzL3KgsIJZYjKNaRy+X3HDhWhxYAGlFwXH7PgWNxjQHxg+Pyew4ci75ekLlXGKARBcfl9xp3XfT1gvzXAfGD4/L7zbsmvh/zjB0cl8+TK1kCAPs8uZIlALDPkytZAgD7PLmSxfUNmXebixxUeTYJgHbl2SQA2pVnkwBoV55NXABELSUAIksARJYAiCwBEFmdAPhnTPjK236RyVfeFKRuv7TSu2/7Sf7JBm3/+1AXAA0zJjzlrb/I5ClvCpK3D4LPTv3b12UMHtr+96IuABo+LfOUt/4ik6e8KUjefpIxbth+8vyDz7PitP3vRV0ANHxe7CkPqvwiU0N5UzBk+3AkN2w/ybsg2v73oi4AGmZMeMontl9k8pQ3BQO2n942bD/JAdD2vxfFagGWX2Tylk8LBmz/ZT6YeseB/W4B7ccA6y8y+cpnBQPGgOePipd5qvB6r8eAhhkTnvLWX2TylDcFyds3HU/D9pMcAG3/exHDdYBrxoSnvPUXmXzbNwWp2991JA3bzwDQ978PyZVwZAmAyBIAkSUAIksARJYAiKzIAO4+vqp5K6XUSd3Otbl/pcucPLt/VXZefdnLDvauyACW9aRXp9dJslZz38vuLvHTm/seZGNWVAB3l/VDfbtIs12e3XheWI9bALTRelLPbbuYmXub+0+VAg7bhVLQLOB2Av6zqb4DL604s6W+m6wmA1eioyJ3QZYDd62jzJ6bnl5vF5ME/pLV2Q3cQt+jXwMv03/IWWtKpgHtj8YHIBuF4bieztMCkCvkvCtbAEBOyuLiesj9764xAkgg77Ob9DkdKvDQmqUgkjIA5ED3s2890GgAQMqz6hM5ADMeNwPYXPx933qg0QDADmSadkEX1+uTq0pZZxekB4Cf7lsPND4A2aUBDK/FIKybgKawG5Crg7BxsiuDldq3HmiEANLeCLqj9DQUAoWTzYyKOencASg5yTIft/dLI34vqMW11d6dAx0agNWsuczIdEgANlPv+xfj1IgBHIcEQGQJgMgSAJElACJLAESWAIis/wNhIRhVi9dWzAAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<p>All plot functions, except for the standard plot method that returns a composed plot, return <code>ggplot</code> objects than can be further modified. For example, changing labels, title, and the theme can be achieved this way:</p>
<div class="sourceCode" id="cb88"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb88-1"><a href="#cb88-1"></a>p <-<span class="st"> </span><span class="kw">plot_x</span>(opt_cut_b_g)</span>
<span id="cb88-2"><a href="#cb88-2"></a>p <span class="op">+</span><span class="st"> </span><span class="kw">ggtitle</span>(<span class="st">"Distribution of dsi"</span>) <span class="op">+</span><span class="st"> </span><span class="kw">theme_minimal</span>() <span class="op">+</span><span class="st"> </span><span class="kw">xlab</span>(<span class="st">"Depression score"</span>)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABj1BMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYAv8QZGUgZGXEZSJcZcboaGho6AAA6ADo6AGY6OgA6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtIGRlIGUhIGXFISJdIcbpIl91NTU1NTW5NTY5NbqtNjshe0dRhtbVkuLhmAABmADpmAGZmOgBmOmZmkJBmkNtmtrZmtttmtv9m2dxuTU1uTW5uTY5ubo5ubqtuq8huq+RxGRlxGUhxGXFxSJdxcXFxuv+OTU2OTW6OTY6Obk2ObquOyP+QOgCQOjqQZgCQZjqQkDqQkGaQtpCQttuQ27aQ2/+XSEiXSHGXcRmX3bqX3f+rbk2rbm6rjk2ryKur5OSr5P+2ZgC2Zjq2kDq2tpC225C229u22/+2/9u2//+6cRm6cUi6///Ijk3I///bkDrbkGbbtmbbtpDb27bb/9vb///dl0jd///kq27k///r6+vzpZ/4dm37raf/tmb/unH/yI7/25D/27b/29v/3Zf/5Kv//7b//7r//8j//9v//93//+T///9btEPsAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAQ/klEQVR4nO2djX/bRhnH5axxeKkbXgbmbSmDNaPsJe2gFBiMsWSwFDxg3WBmsM6jnQcsW7amXl3mtF7vD+eee5d00smyTifZz++TWLLudPfo+epepDtLEUEFVRTagHUXAggsBBBYCCCwEEBgIYDAQgCBhQACqySAccT01Rt0fX6weaRDpj8eiTXYfrrbTe8MUeI75ejvUdTVMecHRnqF02iylgMQRXtJP4w3nAAgSlHnTWgeMZ8jACbu5vn7vc7AGiKVCaCoJoBYKwZgJbQUAPBPn5+I899HUedZuk7P2D510zjaeJuXgEkvOnNDnq40REaBr/cuRjLwH1+JOk+r9GXAkMaVuP5NE3obAIi8sAQQ7gP+zysk6d1HetHmBwwAXaPuGlgBTCCMB6oKjUkFGABYrXfmYldG3kMAoOHGCPww7X3jiEx7XVnB04Ih2oDom0fzt6KuBqDbABrtWXAsW6Mt7VjW9jpAV0Gnu50/kHs0HlF5IQBiAHjkqbdVyPwAgjkA7nYeKwGAAaNJdAZ8B9Vg6AANYNrrixCVFwIgCgCrK6JHb5hdHA6gz6KDjy0AmHMnsUCQDtAA+AqLIvNCALoNIOQ/F3l1XQMAmRcCMHpBoP/+jjopCQAcxmqYoaplbFVQAkBWFaRqKcgLAdDrAOHLCW1ryfwt6rGx0eNhjTAlBI0wrTiehta1S1QUsxGOAchohJ/l+6u81hpA4kpYdA03oSsjO/kcwNd7oic5kd1IHcXohsYA6ADjQmyS6IZKbC3XUgA6j6p7QeziiHYPyenFqPuB2QZQZ34NopH3e9GjH4CPdZSpvhCLAVAB5pUwrfnPvKMuxGhe6wwAVZUQQGAhgMBCAIGFAAILAQQWAggsBBBYCCCwEEBgIYDAQgCBVRGAezfI9FxyhgpX1vZ7N3KjWoKVJsa8Frpu241uS2WcbWSWjflBlagaAHlWLn7MzuAEgIzdkkmU8zICsAgBsEGRLlh3CCMj014U9Vmhv8bWemygBhZRXx2B2mUAHzwi390M4yF7LEW2m0rlGk/1dDfqHAqni3VIkkXju023X4k2/glJ8dTjOQ50ZiJFoo6DDGFAYtxN2itsEJnoZRUqBwCGTuB/2oOh8S63kh5yjw2JwcfG6HR3jw1dCgAQDTYJdwz4gfHdVXIyCV2vq1R6MtU+3cYByHVZ3097ezxVwbIHg2bdeI7nBtp2niKkJKNO2AjrXsLe2JGITOTEgaVVDgAr99D8gRX62PhXse1/MFql20Lp0TgAHlUlJ5PQLahKxYwq3CbXIXh7pFJXSZhJaSOTtrPdxDrzOE/KsFfZIDPZLj631amSANh4ojg2arUEIA5VfEzYuK44EllXJ9zBdjeSM4NFXjoV+jFmUbkH5DqEDfkkapWq9JcwzgRg2i4BiKhQ/YgaKGbvRE2U5NM1YjO2l5M/AKe7nYHhypIAYqnkAGANgmgOlgAw3X6H10CmvdIGmYmxXF4lAbAJC7IYb49sANiBTjruKmh7pJJLA4ilYlQgLEzXXCyuPhNSVY8JQGemAcj1+cFTRmUmltIGmUlsuayqaYRFaxUHAKdNTwGQu0DLOT/o8OrW0gir8iNORDMVtr1rNMJd2QgzFrISlw4XLWssR6MRNgGIqMScIKyiSRtkJnK5tPNBFXRDweJh1E21AbSm7PxR1AB6F9aHe+bJAd9F7G52Q9Xe3BNmKvCR0Q0VtbQwRCR1aMlxkOgPixIgohqdGyOasEFlIpdVaLkLsSVPg6rOogpVaQ+niBBAXON+zRkiAFPs4qxe4e3owEIAgYUAAgsBBBYCCCwEEFgIILAQQGAhgMBCAIGFAAILAQQWAggsBBBYCCCwEEBgIYDA8g/gs+/++ezZXxNy/+dnv3jTe25a7/6aZv2RyPXTs/VmXlw1APjWY+TTL31E3uWL2vTpY/BHc/3wy+T+L2/CoomqAcB3bsI/+OD+L17znp3S/ec/+tdrkCPNudaMF1NtAGDx+as1+uHzV//yPNRAZ89+4TVaDOGziVrdEkA+/NVjrO4RqrX+K67aANTeBpDPvk1x81wh47UHUHcviHz+p5us7wV1z7tr3AsKps++F9qCIlpdAB82tNVNaHUBtEQ5AO7u7Hz/PUIevLDz+MdqgapY2QBmP3mPHP+APLy+byxQVSu/CqIQHrz4nrGoyao1Uj4Aes7PrnxMHvzmTbEQ2z/J0IlYbmVFWEx5pn2SWD8R61v2OMnojvXcrCtVHoDZ5fNvkruPM8+LRSJGAS8sceRrD4AYp368BEghgKXl6Ibe2s9tAxDA0soGICqdh9ev8l7QVUsvCAEsrZwScLyzQ9uA3OuAmgDMLrMrkkWzbjuAAqoHALQ9x0n6CABUDwDWAXgxUQQQADkREl9rKgFlLkGkpRVdglSqZQDcuXP7zp07/gFY259FSoDF0gXM8Ks2AJg9R68Hk60wAqgPgPUqHAFgCahEbQAAIxPnS9wFQQBy8wLrWVFSQgAIoBIhAKcZfoUAnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4VTkAfJzpNpM5zlTDzLisuLkjYmlLy2TtRVgCnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4FQJwmuFXCMBphl+1AsDD6zg516L6ANzaZ78RiAkB1Acg9QM9W0QE4A/A7MobugoqMiSZ/FEeDkkWX7cCuLzPfqoaU14JoJaBTrAEkIpKwIK/EUMAFbcBLyEAm2rtBa1jFTS7vLOzn/Owjlp/qL3QwzpWAwAU+tlzb2Y/tK+5V8KrAeAuuDvvgU1rCWAcRVHHeBH70m9lL/fIsqzedUMe2ucPwHhjRMgk2lMb/AKAx2RlP7RvDUvA/IC5frh5JLd4BfDghavE/thKprUE0BdrzPH0Y3ruMIqAx+lu1Lm2PZpuvxJtjOYHUdS1RLIotxdE+0AE2wBTkyjiCJRve5tH84Mu9X+f/m+Mpr0ucOqyfx0JmHStKeY8upj5n2Q/tG8dAfBW2Dy5e3tsOYHGYQwA6Hf2hX4kI9nSywZwvAPab8J1gE0Bu6Gnu5tHyrfbI9iwN4Yahn5h2yfsC1Q9A72kkWyJteJK2KaQ1wGmbxGAJWtvAEQ1kvStrIJkrTOBK4VUFQSo0kIATjNMDcG1stGdH3TSjfC5gW6EdaQyjbBT6wiAtcGsIzTtRdEzT/IeJrgWuqGHAgAR3dBUJIsQgNOM4ppk9PXzrtZaByDrtkVsSDJxk6SGIUmo9rNqmcIARDvN2pMCwhJgSlwfWIUAWnE7mjUvTFkgE0IAVchSAooKAVSh1jXCWRFXAgB0W6mwDQgEILMbZRcCqELYBjjNMHRiV56dLsVLAALIz1oknFBlAApfAShrEEClAE53I2yEc7P2XQIWEwJAABlZtxUAVkGurGspAadPuqcaZd3kXfWZcfkA5gdR375f5oCktQrKGlawWFNbCYC5wXE1EECmk/PCbAAaWAUd7zQfAAxKDugHdd/85cMo6vNZXHB/Z48B4GEJWQAMm1cCZj/9bfMBMCcP+2QspsX1urAFanSYrqLCErI0wh13G5B1WJ4APHz9b6oKymovbEOSSlUNSToBgLtPfzSavzwg8C/aU7pFhyWSbEU39PhquTbAWNZUAtg53BnEAAyhX6nDEkm2AcDsysetAcDPcAMA3OHkVVD67AfFAahZL0VUGwA+S/VqPPdmAoB6nvYiDQCwUUxh5GGJJGMA2O8/YDpXITW/GxoAAOsKmSUATuozF/d0WEItmRXRCgCl1BIAaa0igAZXQWmtJICGNsJWrSaAhYQAEEBG1v4A+B2UZ7/CLD43ZQ0BeFAMwBB8X5jAGgLYsivPTpewG+o0w9DWJZsQwKoAwOsAV9a+AcAv8fE6ICdr7wAWEgJYSwD24TD9oNDYiFhymXie6MIjYnUDYI9HacOzIqgpJ8KUvBIgvy9mhqFFAeTNkxDKA3AXHpXXimfGrSaAW+f/SktAK54X1AwAC0xG0XJWQa14ZlxDABSfjKLlBNCKZ8Y1BEDxySha5UoAEwLIA5A/GUXLCQDbAEOFAbgmo2g5AbTimXGNA+CajKK1xHVA4mYsAlhgMorWElfCWxcuPHHhwgUEEOpWBAJAAHIdARAEUFIIwGmGIQSQEbE2AH4H5RdUfQDEy1RiCgLAg9oAQLxMJZ47AqgPgHiZSjx3BFBvG6BuBFrvW5/cvn0i7ozDnXBZN2cNTS55J7xStQQA3IqKK68EUMtAW1gCSEUA2MtU4kIAtfaCkn0gBEBqBGDzPwKoEYB8mUpMCKDBV8IIAAEUFgJwmuFXCMBphl8hAKcZflUOAJ/+9gSTOQuuITPjVh8AE5aAKoQAnGb4FQJwmuFXCMBphl8hAKcZfoUAnGb4FQJwmuFXFQAwnhlS168kk/NCxJCk+i6uUOQQZGKIUu/GlUytbUOSxnlVVwmQWRunOlgB86bo/1ayBCQsldOrtu6wknEivm9t6XlWCCC5bs8aASCAZYUAEAACKCkEUIUQwMoCsDxSEAGk5RFA4ctOBFBOygv6ZwoIYGFVAUAcLkEAJYQAEAACKKkMAOz+onGP0dgDAaRVGIDlWRFJLwjBIfKjjm0m5QGk8l5DAJZnxiW9ADfYzdvqerPtN50LAEjnvYYALM8LSnqB/p+Ir1smF7m8dOmHl/SvmhcAkM57DQFYnhln+cVyanDJooxZcLZHetryto+I2Ye2ThLbk9HaNCJmeWYcSZ+vnn6masu7QNZFGvcCZvhVuRIgVQ8Aa97rBsDSBpC6AFjzXjcAlmfGkboAWPNeNwCW6wBSFwBr3msHwKqaAJTMGgHYojjWEQCqXiGAwEIAgYUAAgsBBBYCCCwEEFgIILAQQGAtAcB6d4jA26/g9VcpPby+cz55Oxs0u2yLzu5+2h7YGjrrqlUegGWUmAnst22/tc9GVpJ68AJNJrWdvcLM+sDWwFlXrvIA7CMEXJbtEN0a1Tbawl9hZn1ga+CsK1d5APYxMi7LaTi78oa1HshIRvrRmkPArCtXeQD2UWLQ7LLlcOHRh3DESbF6IB1feCH9wNbAWVcuPyXAdl5lRafN3c9ez/CC5YGtgbOuXH7aAEv1+eClHKelK2nRFbHXwgGzrlzL9IJso8TZ9cOtjHqARrXV2/ASvywnBMy6cnm4DjjesTZ5NLqtjw59d0syrEdpe2Br6KyrFl4JBxYCCCwEEFgIILAQQGAhgMCqFcD8IALlvGHdrum5nPfBtlw1A+izz5z3G6+dAgAg8L5vlFAIAGTc5dXRBrz3+zCKaImYbr9Cv4qNZNqjNdWeWkIVBEFdqI2u9VgQSEYTYSpOLK2GKwiAyebR/IB6arx5NO2Bq7rUlwyK2Ah1/rS3ZywhiMejtMbcsTIYtkOp0nGMtOo8vjIKA2BjNAEfUq9R/zFPsqXauM3PXLU8N2BB9EPGTwTLZHUcmVadx1dGoUrAmPWHoj5zHvgclnIjGYquklgCADiXARQ764XHRfBE1DSxOCqthisIgGFXVQ5xALrGON3lFThbZgEQwXYAza99mEL1giYd7kNepWyPmNPkRiJjiSUAgKDJxigBQMMjYncZJ5ZWgxXqOoB9UCepRhicJjeyU1pW/PyUVg2sAUAGy7BYHJlWncdXRiGuhPtynbqHdUO70qdiI5lEsWWsG2qUABkt2Q2NpdVwhb4XtMp3GQoJAQQWAgis0ADWXgggsBBAYCGAwEIAgYUAAgsBBNb/AUCisGbKaXOxAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<div id="flexible-plotting-function" class="section level2">
<h2>Flexible plotting function</h2>
<p>Using <code>plot_cutpointr</code> any metric can be chosen to be plotted on the x- or y-axis and results of <code>cutpointr()</code> as well as <code>roc()</code> can be plotted. If a <code>cutpointr</code> object is to be plotted, it is thus irrelevant which <code>metric</code> function was chosen for cutpoint estimation. Any metric that can be calculated based on the ROC curve can be subsequently plotted as only the true / false positives / negatives over all cutpoints are needed. That way, not only the above plots can be produced, but also any combination of two metrics (or metric functions) and / or cutpoints. The built-in metric functions as well as user-defined functions or anonymous functions can be supplied to <code>xvar</code> and <code>yvar</code>. If bootstrapping was run, confidence intervals can be plotted around the y-variable. This is especially useful if the cutpoints, available in the <code>cutpoints</code> function, are placed on the x-axis. Note that confidence intervals can only be correctly plotted if the values of <code>xvar</code> are constant across bootstrap samples. For example, confidence intervals for TPR by FPR (a ROC curve) cannot be plotted easily, as the values of the false positive rate vary per bootstrap sample.</p>
<div class="sourceCode" id="cb89"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb89-1"><a href="#cb89-1"></a><span class="kw">plot_cutpointr</span>(opt_cut_b, <span class="dt">xvar =</span> cutpoints, <span class="dt">yvar =</span> sum_sens_spec, <span class="dt">conf_lvl =</span> <span class="fl">0.9</span>)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABIFBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYKCgozMzM6AAA6ADo6AGY6OgA6Ojo6OmY6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmAGZmOgBmOmZmkJBmkNtmtrZmtttmtv9uTU1uTW5uTY5ubo5ubqtuq8huq+SOTU2OTW6OTY6Obk2ObquOyP+QOgCQZgCQZjqQkDqQkGaQtpCQttuQ27aQ2/+rbk2rbm6rjk2ryKur5OSr5P+2ZgC2kDq2tpC229u22/+2///GxsbIjk3I///W1tbbkDrbkGbbtmbbtpDb27bb/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///90IyxrAAAACXBIWXMAAA7DAAAOwwHHb6hkAAASUUlEQVR4nO2di3vbthHA6aaVJnuvZnaausu21OmWNesab2s3td2ibolae+k6J5aVWXb5//8XI0CCDzwPLwKQcN+XRCFPB+h+PIAEeEBRZgkqRegK7LpkAIElAwgsGUBgyQACSwYQWDKAwJIBBJZUAWyOJ2Dd9a9ecI/fnr5zAVDzK7sA4PwtEACRml/ZZQBmao7FAMDtn4ti70NyAd2eTsr17PDlbO835WpWvHvBUy3LNw+K4qfP8Zf++eOi0u2fZPQ3x4eVrbfn/W+W5UtyqMQAkEZ1fFWc4P8fNmcarbZ2t6dFURzSFt+u63LR1qdW41XKr+gDwDUtql/dB/DzWVXvz6u/iglPtdJAH/bm5BD+OjnJ6m+OkUGk332z8nTzAcnm+E6jUcfCipxYtQVRAFqLq1lPpa1PrcaplGfRB7Ce3b2o/poMABQfli/RBbSe9ZvVniq6qr7Dn4rJRXleTLqTHP3NcaV1+3Ux/Gbls5dFc6FXGr+oNcoFajoWTbmtVlc73Lb0LVaVrWqAr/6Lrj5YjVMpz2IC4M6v/4U+9AFMkEuq/w77tVa1aqPQP9VPvD1F7kJXbXuSo49tYdd232w+NVJrYGsrHDKHxETzgQHQWcTuXezNawCkPg0AplKexaAPWKAgfff5sA8g3eJicGNBVFd1nBf1j278Qk5y9Jsu9nxv3n1zPes3DI3Hz5s2qG2BWi0GALHYaKwIgIEar1J+xeQu6LuqEyvQxawEQFS5AMhJjr4WAHRZt8XaAuBVyqsY3ob+509V3C+61kQEoFHtnNcHQOyw+l0D032TboImZdMEVfH1sL0H6rQWg7alb3HYBFEA+JXyJ/oAVlXvV95+XV16iwLfvYkBtKpV1/jX6vbvtB82nR1Wv+ti+9+sVNfHjf1Ko+pocSdc6fyktdJptbVr2qnO4rAT7gBUapxKeRbj29Cq3rh5ePuBGAClWgxuTrqTrH5zk1lf3s03m0/NtdncVtYX7XmvzWi12tpVp9FtaGdxeBvaAehuQ5kw9iemD2J3URX/XT14/VfWB3Sq66ppvTN4euudZPTrx6y6LyTfxI9Y7TNS/WD1s7q3HNw3tlqkduXmQTH5nrLYexAj9UFqF5xK+ZU4hyJ0BhpK8jDs0uJ4sg0A3jxQNxk7BKAeO+gPG+jr67ir6l6LQ4BWBqChr+Ouqt+8q9baIQBZtCQDCCwZQGDJAAJLBhBYMoDAkgEElgwgsBhMSe4bjNXqfAnpvhlxTiqsjBQBmgCMIKcpGUBgMWqC1vvoFaBurrDo3v05LOuTh2ussd7/DE9vII+iyY5m4mR98Bc8qVwfaAxgr7fWD1u72y1mANDbP2QKtfbb7GRzfNK8PoJOFrXGeob8PEFK9axTPXCMZ1DIAWKgA4AjgBx2+3ujE0MAJ12rsj6oQfzvon+y0Wj/3Z+v6kly7FB8mBwgBmgAByFe1RxdDJugea9ZX5D3EVf16yPtSaKJnLw/P28nd5vT7YHGAAWgs7vVYgXgvHHo5hi/y7M3H9KhAPSmrWoA3QFsgAbQHHbwI2MWBxGApPLxCvlzRUcAboIOkHNXvfkZfLp/oIFEAWibrC0WewC4KW8cvJ4xAHqdMHo9pXY6Pk0OEAPoZTf0Wg/+eNLadft7oxMHEbBqZhOrNnvvb92F3Gh+hltychvaXPT1d8kBYgDddz78AFtfFJP28HaL5wex7b+CbSUDCCwZQGDJw9GBJQMILBlAYMkAAksGEFgygMCiD+ByKPT/RQLVc68Yp8EMILDBDCCwwQwgsMEMILDBDCCwwQwgsMEMILDBDCCwwQwgsMGgAJZYDGptXXI8BgMCWHaiW2vLkmMyGAzAkhadWluVHJdBEIDr335b//vo6L1vnQBg3M8yiNNfzg1CAFw1br/5w7Py1f3X9gD47qcYxOkv5wYBAL659/c6Aq4fvy5vPiEhYFxrifv7EOL0l3ODOk1QFwE/qkT2BYmcgcTQeMIC6gNunhy1LZBZBCyXZ8oAaESmaHQhph8B1x89K6/es2iCVH4FAxgQ2B0AV9Xlj5ohQwAAv4IB9AnsDgCrCID5FQzA4HYpcQDoz9XR0T0SAHoAwH6FK2r7IWEAXIEXouVXuKKuHzIAxwCW6pLBbjBS3HkAmk9sGYBzAEstP2QArF+n06kVgKWOHzIAyq+194UIYKGi4YedBcD3a+d4AQJgW5UBKAvh+JVqergIoJ1FBqAqhPYrr+HnHAP31hmAohDa1QI/0ifgt0sO3GCkmCCA6kKX+HWIQON+1d4NRoqpAaibGalf+wh0Hhis3WCkmAgAyrcKv3adgdYTm60bjBQTAtDrY9V+bXT1Hpkt3WCkmAyAQdsO8au6rWIAAAhsGwConJ1NYRPxfZnqf8fbDxhHvEYAe71ComAgyghQx8C2RQC0kKH/TUdDJThaxTH85dygdwBLNwCGIoipEfzl3GCaAJaCbt2/v5wb5AK4PT3sNjpNCYCUQEIAFvXepHICwEL8AFiKHpl9+8u5QR6AZpHOlXyxVGAhS+MxNvnpzmwGIC3EF4Cl6Nnar7+cG+QBqBdE37S7U9sAoFsgh5PyokEjr/5ybpALoN4EWbEzJqwQCQCi4BqACEFKACACK4QPgFIxA7AUj6/685dzg+MD4CoZAVhKhk013WCk6BHAeVGcnCt2R4YVMvC/WM8MQG2cr6jnBiNFj88B73x/fOLmOaAFoKqMEYClZNxayw1Gij5vQ9t14y0BYBcBK+MYAIMgA1BWRh8AMi9WhLvBSNHjcwBqgnrPASRJ74cvdBM09AAMEAAfGKbgbL6EAFDPASRRu/zmKc4T0wEwpS9EVa11ASxlL7oY5ZLFAGAgbaJ2l6TtEUCLwA0Ag1Sm6AB0SXqPv2qaIHCiNpoPBikOv6U3f7wVk8XMc0B/90AC4NFTvFxBLRDKbBcAvGw0ImB5Jpsw7sVAQhFAD8Zd99aK0MoTNgZQfVVj1E5FQC+TJjwAZjiarBXxqTYApguA11pn2BREIH0A6C5IswmyADBUtASgk8gRHkC9zRrVBKE/N080F2xiWyAntWYAgAikA2BzXBCRPAwDCuF0AW5qzQCAEEgHAEwAhXgDMCBQdxZqAtBEjgwAoMgCUBNICAAejVPtIAsohNMHu2qJTQAAM2kiALCYoEeBc+v5AE4AOOsKGQBKAukA2NSzMfbD0eMCUBGApjJFAQDdg1oD4HUB7m4GWQAKAukAuD09RE8CC9smyC8A3hpcUgLQXLLwANCOypNyYT0pz/N/BkCf4wEAiboQzwB4EwfKTBo3JevpWQDY/NLmSdg3AN46jBICGYC7WpMCWADipRDA2ZRbA4DbB7sdEuMAEAfB7gHg+d/xmCQPgGIBIkcla+jtHABBEGwbAKWYTcjrFiKYqJfM1Huvk5EkGgHi9djZIIAmdKcSAcpCwgJgewJoQndMAGSiKgRdhL5qPShHOHsvys1xVXLk8wH8AHAPQPL6hGgJLkclRz4fEAEAUUK3o5Ijnw8YDUApAdAPAuiSBlEAcDAfMB4A+b5MvHxiNyVHPh/A74PHB8Bdrc5JyXHPBwgCwAsA1dZkbDalk5Ljvg0dFYCSAJNL5qLkDKBTVABgc8lclOy1CVK8lwgC4K/WrKKaAJVJ46Bkj52wYq2mCAGoCVAPxg5K9nkbag9A0AIFBHAGJRAcwO2pPQBRF+ANgJpAOgBUj2BxAlASoFOZrEv22ARRyQEkQwalaj8FA/BYa67i9gCg5arLi3l1FC8AFQEmlcm25LEAtInaVSj87o8RA1AQSAkAtV5Qu1bEl/+omyB1ovYY88GM6CZ1j19DsUjXCyIAXn0M7QOEfbDXCJCHAJvMZ1my3+HoEzZN9frxaw0APmstVFQBABGIFsCrIyQfZwA2NWTO8Zog0XpB8NvQUAAUCd0wAuEB0OuGkkRtOABhF5AB0Oe4ACAiK0TcB/sGoF4JVk1gOwB4rbVcUQ5gSMCmZH8ArN8LCgtAQICfzmpRsj8A1u8FBQbAR8BPZ7Uo2edtqOV7QeIuYCQA0qV4VSEQAwDL94LEATAWAOlSvIoQCA7A+r0gSQs0HgDJUryKEAgOwPq9oEgACDZTV4ZAeAAgkRQSCwDusjbqEIgIgGGCRjwARGshS0MgfQAS/48NQLAWsjQEMgDHihwA0hBIHoCsBQoBQL2kgVnJGQBckQEgC4EMwIMiA0ASA4kAEMs0qtluIvL5+eAz9C4jQBYAoSLgkk3mo/OItQ1G+yAWKQA6mY/Npdc16KKGXAB2+QHSLiAkAHpwSEQgOADL/ICEAeitc+9zONoGgNT/QQFACGisc+8xArYVwCUAAHyde399gF1+QDoABASgq3zH2gnHDAAUAtBFpj02QYq9nKUAllP/tTZSrPUABKArvEbaCScFgEsAurxipJ2wvAUKDgAQAtDV/Tz2AQcWnXDsANQEoEtreWyClDv47AYAR8vajDwWJO8CIgCgJABdViVOAIo+OAYAQwJyAN5HVXgArJogRQsUHwCWAD1u7bWG4gjYfDAnH0mGzPWjI2WecAoAFCHAWQXQXw3FAMoVeTWOJGqjjTyvP3q2BQDkBPjLMHqqoQxA0wS1idpX75d4Q085AIvK+FW0BMAy8A2gezm0S9JrtrOVJGpPBccjE+n0sES8VYjTCe8xfQBK0iNZqoIIULVAkUTAMAZEaxwr4mC829AWwM2T1v/bBIAioN5K2mENtQCgTeW3BQBFgL/GsZyBPwB0kh5ZqqDvfxEAm8r4VaT1hg7tEYBupu4PAJ2k1yRq10sVyO+CkgXQCwIoACd7I/IAWCTpqQYiYgLAZJHx1jhWALCfuREAME3SU3YBMQFg8/jYNY6VACAM9CdkjJP0lAEQNwDOGscAAFYTBzwAFkl6aQHgZRPTaxyDACgYjDgcnRgALoGpEQDThUgzAB4CIwBiBuMBUPfBkQEQLCoBI8CLFO0aOgZg6IZRFPl6fMeCgkDQVOnVMAMQOBZAADpsnQFI9USOVQeBpLOG1zADEC+rokIAHbYeD0Ax1vt8RoraAFTtEHTYejwAgACIEABDoL+qhwwB7HlBUcMM4FKysBBGoNikHiB+AHAkkflgRlQrfmOBzxsbzSXvdASIV3YahsJUd+pyqMivVQZwCQPAUtAFwGXgEkDh0l2jArgUruwkpKAxaiodNnUJABIA0QK47BjAHMs2SRmAvZ4GgGbMCAIhA9DR056QUYfCWACS7gP6inoAWgqhAYD8nwSAS8jOiGykCEMhAzAyqA2gpRAIAKgLSAjAJWBjMr7QoZABWBg0AdBR4LxoRBfmDsC2NUED0QfQwzAOAJj/UwXQiDaAjkIG4NCgJoCzlgK7s0UGYK5oMhY0pXtKdwBgffA2AcB6mgByBPgwCAXANQIBQFKUbp4c3X+dAfD15ADERgAASKI22sfw1fsCAED/by8AJDzvKwwCALSJ2jeffNvLWKUMZQBE2ObeelK+209YnaidxUwgAK7uEwBI3F42IyjGaVAHQBcBGYAzgzoApH1ABmCmqAMALRQhvAvKAMwUwQDQH+lzwJi1jqPk8QBwJWSt4yg5AwhccgYQuOTAAChx/mC2YwYzgMAGM4DABjOAwAadZMhkMZcMILBkAIElAwgsGUBgsQMwGKVzIVdHzTy0K/nhi6N7z1wavH7ktoZWAIaz9Q4EDb06NYjWHL9yeY3cPKl+skuDVgCGMzWOxKlBVEOnMpwddCBWAJzXBonbkHr8ldsmKC4Aw9l6J3L9yK2/Hj3FPnMmuAlyWcX4IsCpQfc1rDrh33/p0GB8fUC3V4cDufnUxyXismOxvAv62G2T7b5N+8Z1E1TVzulPjuw54NWR49v2qoZuHyyqJxWnPzk/CQeWDCCwZACBJQMILBlAYMkAAsvWAHjznDm03p/LTsch2wKg72z90wElAwgsyQK4PS2KSePZ9f7ns6I4XO9/VhRoB5zuXH3mZI1Ooy1yqs+hK05JqgDQhmeb4xMCYI7+Xc/eeoGOkz/o8KwCcv7WC3waq84iI5AqANKmDAGc4M94I7QV9jo5hE8fSPdHCySpAiC7zQ0AoM9VWOAtwetj/dPlAjdMkckuAcA7Jsu3CRxfUgUgboIOXqDtAEkTNASA+QSsNUdSBUA6WrT55e3p3hx3yJxOuAGATuOgie5+NFUA5FYT31s+/GBeNfATfBs6KYe3oTUAdLpcDfZrj0SSBcCR6K5uiGQAgSUDCCzbBCBJyQACSwYQWDKAwJIBBJYMILBkAIHl/2uc1aCVlHOaAAAAAElFTkSuQmCC" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb90"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb90-1"><a href="#cb90-1"></a><span class="kw">plot_cutpointr</span>(opt_cut_b, <span class="dt">xvar =</span> fpr, <span class="dt">yvar =</span> tpr, <span class="dt">aspect_ratio =</span> <span class="dv">1</span>, <span class="dt">conf_lvl =</span> <span class="dv">0</span>)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAAA3lBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYzMzM6AAA6ADo6AGY6Ojo6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmOgBmOmZmtrZmtv9uTU1uTW5uTY5ubqtuq+SOTU2OTW6OTY6OyP+QOgCQkDqQkGaQtpCQ27aQ2/+rbk2rbm6rbo6ryKur5P+2ZgC2Zjq2kDq22/+2/7a2///Ijk3I///bkDrb/7bb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///+wzq2RAAAACXBIWXMAAA7DAAAOwwHHb6hkAAALM0lEQVR4nO2dDXfjxBWGldA1Tgot2YW6QGgbYMNS00LKkuAu1DGNN6v//4fQfOiOJI8+rMzolez3OWePnNFIq3sfaUaypHGSEigJegOOHQoAQwFgKAAMBYChADAUAIYCwCAFbD679c/YLmY1i/wnSWZvom0QAqSA1em+AtZJJiDeBiGYmoDLeJuDASfg8TrbnS+2i4v1PPnDUv09WyWnr9WsTIAqfJ1nPKukF7nJljj9ybfEdMEL+HCeTU+W2d/vzZNnuoHfLt6zheZYWGefFFaAZ4npgm6CtousV338MWvZMyEXdkZW+JEpTG9UM3WTZ1kdEN4lpgtegE5ulufHa+kSTKEuMCnP82z+8iwxXfACdH+7Olk+XktrYjO+sm1Q3gLlAjxLTJexC1B7urRAFBAW1wSpxqQkQOXYtDDr5HNpgQpNUGWJ6QIVoJuYvL8tCch6V9MJZ8V/lBYo74R3l5guUAH6NFSfcWb7eknAh7ZQ13JdrRHgWWK6IAVsP01m/9XXXH96nZYFqGutD8wl1mbuLottH7C7xHRBfxta/71bTvXrh/YlJsXoBfz2aWU3p4CgtKQz63GrV7sUEJSWdD5eJ3/eb4mpgRZw9FAAGAoAQwFgKAAMBYChADAUAAYoYHO2bK/0lIVU3d/G/tDE1I6APQX0kjwoFAAG2wRtzr6dJ/J180bdaLm004vUzLzY6Bqbs1eJegZIZVQ9UWRv0mzOv0nUvUlTYFegsy5rv5D1jhKwgHmW0/wJRZO3+eV2canvF5uZiamxmas8z1QlNUlX5ktqfbcmL8hX4AToIyAvxkXaBFrApWtVNudGxP/fFGfaGjI9W66VMG3Jzs4L8hVUBZyP+ekhdBO0LDTrN/mjz2v96KGbmddUST5brhLNhV2HvrdsCuwKKgLcesfIOASsbEK3i6wt3y5OlmU7FQGFW2RGgCvQK6gKsMXDR9iFcQhwhVmO1yqf6+oRoJugc5Vc96CcnV0ssJIqAqTJGh8jEqCbcpvgzXxHQKETzgzZpOvZeUG+AvVg3eO1XoHKe16Mi7SJEQmwTb9us0/+6XZkW/OVbsnz01C705tl84J8Beq88/O/6rXfZAvlxaNkKhdio92DnwoFgKEAMFMRcLBQABgKAEMBYCgADAWAAQr4n0U+lPCWDli1eflwWaAACqAAEICsUkARQFYpoAggq8cn4OHvP+vp2y+ff/KrTAyArB6dgPvnH2sB7757mf7yl3xiAWT12ATcvfjeHAFvv/5ZHQx2YucCsnpsAqQJevjHr+nbr36wk6zg/Yy4/3NozKMvEdYbfpVFrID7T3Tm7cTOA+zW+x0BSZFy1XAZAh0BCkBWW6t6cu6tGi5DwwgYcx9Qk/PD6gPefXdlzoKuRnEWVMn5oXfC6h/8OqCpbTlkAU08PavNVX05P7rT0CZ6ZbW5tK0PpYAiXbPaXFiXc16ItdI7VdWc80q4H3unqq5toYB+7Jmq2hNGCujJXqky+z0FhGSPVFW/inlyAp+6fLgsTEGAa/UpICQdU9X9C5o9EvjU5cNlYewCyuc8FBCSDqFWTzkpICStoXY846eAnrSF6vkyhwJC0hYqBWAZ75YFZbxHQNMt2SfvwU9dPlwWKIACKABEc6jdb2hRQE+aQ6WA6DSHSgHRaQw1iZJVCijSGCoFxKcxVAqIT2OoFBCfplCTOFmlgCJNoVLAADSE2u+R5aBVj1pAtMfdKKBIfagUMAi1odY/g0UBIakLNYmXVQooUhcqBQRD3kj65bnipZ5+3PKSXtNTiBSwF+WRCdRLwncvC7NrQqWAYJTeSlXvB7/71w+F2TWhUkAwSu9lq0Mha5JUQ5Q2DlVwJE9D5MQMtzgygZ4+/K14FPj3tf3fZ+QRUEfxCLiX94OlH/CHSgHhKPYBd1d5KQWUiHsWJCMTmIZHHQbv/t18GkoBATHXAXqoAtMSZdcBL+REyBtq88swFBASb6gUMBzeUClgOLyhUsBw+EJNdkINmlUKKOILlQIGxBcqBQyIL1QKGBBPqG3vZFNASDyhUsCQeEKlgCHZDbV1yDEKCMluqBQwKLuhUsCg7IZKAYOyE2r7yDQUEJUxbctg8Ag43iNgJ1QKGJZqUIk31KBZpYAi1aAoYGCqQVHAwFSDooCBqQTVZYBECghJJSgKGJpKUBQwNJWgKGBoykEF+GmjoFUpIEZWKaBIOSgKCMHj9WX3yuWgKCAE20VfAcP8Ms/BC0hXz950rlsKigKCsF2YH5o6ve1QuRQUBQxPKSgKiIH78VQ7RkH9r6lSQCBWqgWyPXFhqALzamR57IJiULvDtFJAL1aq9d8uLvQf7jVV+3Z2/S9qU0AY7Gno2nTC7kVtO0aBK6gOVXCUD0RoYgpwQxXYMQqKYxfwCDAE3vXWJ0vXBJUG61D9QLmgEFTXn4uhgBby6wBzKVBu8jMBtX0ABUTBDVVgxygojF2QUoAhZh9QGKrAjlFQdx1AAYGoCGimEBQFBGElHcCsS3UXVOff66GAFnp+HU0BEFxQFADBBUUBEFxQFADBBUUBEFxQFADBBUUBEFxQFADBBUUBEFxQFADBBUUBEFxQFADBBUUBYEaxESB4BBzvEeCCogAILigKgOCCogAILigKgOCCogAILigKgOCCogAILigKgCBB+fJPAfGRoCgAgwRFARgkKArAIEFRAAYJigIwSFAUgEGCogAMEhQFxMK9kfTwhf4tcztigUGCooBIuJEJ1Lup6l1h+TFnhQRFAZFwb6XeKw13L93vySskKAqIRPm97OyTHbEgrQxVcMwPRUQNvjQygXpH2I5YYGfLXsUjIBLFI+Dtl1e2VPoBCYoCIlEYmeDhC+l+KaDEMEMV2PzbEQvsbAmKAmIhQxWo83/V/doRCwwSFAVgkKAoAIMERQEYJCgKwCBBUQAGCYoCMEhQFIBBgqIADBIUBWCQoCgAgwRFARgkKArAIEFRAAYJigIwSFAUgEGCogAwY9gGGDwCjvcIkKAoAIMERQEYJCgKwCBBUQAGCYoCMEhQFIBBgqIADBIUBWCQoCgAgwRFARgkKArAIEFRAAYJigIw5EF5808B8cmDogAQeVAUEA33prz95P0xTwqIhXtT3n5yBYo8KAqIhXtL0n7y/6AzBcTCvSdsP7mC0pvyR80wb8rbT6VX59Nu+1rQ3ZpHAAVUGUEfQAGxcG/K20+uQAHI6pEJcG/KN14HUAAIQFYpoAggqxRQBJBVCvCxxxXZlKp2hAIoAJ4qCjjIqh0ZgYDjhgLAUAAYCgBDAWBQAprv19dU9Q2CX1PV1umw1sKQmjVrTeU79NZt3R+QgJb79f6q3kHw/VVtnS5rVahbdbVrzWZbM63b2gOQgJZ7Nf6q3kHw/VVtnS5rTY3Z+rWmdy++NzVbt7UHIAEtdyv9VRWlQfDrq9o6Hdeqduf6tabSBLVuaw9AAlru1/urpr5B8P1VbZ1ua9XT+rWmIqB1W3swrSPAMwh+XVVdp9ta76VHresHDu8I6NUHeAfBr6lq6nTrA+6u3BL+zX04uD6g5X69v6p3EHx/VVuny1ptw1O/1lQEtG5rD7DXAXX36/1VvYPg16zV1umw1rw1qV+rEdBpW/eHV8JgKAAMBYChADAUAIYCwBycgFVyskRvwz4cmoDt4hK9CftxaAI2Z5Pa/w9OwGaeJLPN2askefYm3Zx/k5zeojephQMToI+Azfz09vF6lk1n6M1p5zAFXOoPejp2DlKA6gey3ngS/QEFgDlIAboJOr+lAAClTpgChsc0Qdlp6Gwi1wSHJkAxicTnUAAYCgBziAImBQWAoQAwFACGAsBQABgKAPM7UkUUxKEKFy4AAAAASUVORK5CYII=" style="display: block; margin: auto;" /></p>
<div class="sourceCode" id="cb91"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb91-1"><a href="#cb91-1"></a><span class="kw">plot_cutpointr</span>(opt_cut_b, <span class="dt">xvar =</span> cutpoint, <span class="dt">yvar =</span> tp, <span class="dt">conf_lvl =</span> <span class="fl">0.9</span>) <span class="op">+</span><span class="st"> </span><span class="kw">geom_point</span>()</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABBVBMVEUAAAAAADoAAGYAOjoAOmYAOpAAZrYKCgozMzM6AAA6ADo6AGY6OgA6Ojo6OpA6ZmY6ZpA6ZrY6kLY6kNtNTU1NTW5NTY5NbqtNjshmAABmADpmOgBmOmZmkNtmtrZmtttmtv9uTU1uTW5uTY5ubo5ubqtuq8huq+SOTU2OTW6OTY6Obk2ObquOyP+QOgCQkDqQkGaQtpCQ27aQ2/+rbk2rbm6rjk2ryKur5OSr5P+2ZgC2kDq2tpC22/+2///GxsbIjk3I///W1tbbkDrbkGbbtmbb/7bb/9vb///kq27k///r6+v/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///9Qcb2vAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAOg0lEQVR4nO2dDXfbthWG6ayRJmffVta62VJ5W+fsw146pdviLVYSr5tdK8osO/z/P2UEQFISCQKX0AUuSN73nBxLxEsEug8BkABBJimLVAl1AYYuBkAsBkAsBkAsBkAsBkAsBkAsBkCstgBWv7qsblofj/bZXerh/PE1wNZDtQVw9WgvAJrdpSoAmmw9VCQA3Gx9UDsAD+dJkhytj4+Wk+Szeb4xAyC+vknTZTKT34/ylO+USx3eD+ej+u4fnyVyT2HJ/v3rR8nBr/P/Be83Ri03AD+dZH8PcgLr4x/kX1VdWBYJy0RtrgEod19OtiwyNdOMARgkGof1cTK6fvhnkrc82ddf5F8vRNNxkTfnD+dZZL/LIlkCqO6eRfqrbFsij34BYHSdfRtxE2SQiqAM8UUeJfX14Tz7KtqgsgVaTfIPNQDF7quJZHhxMFcARIayFjGARqkIyrhd5U1NHvGrvA0qW6DVZKY+1AAUu+eOZQFgxzYQ4QIQh3XRAjEAkNybINVipMVpqPq6TJ6X50BlE6Qaq7Jt2exebYIYgF2qoSl7XaHsa9bRqq8P5z8uWiDVCa9EtC8SeW45qu5e7YQ3AMpMeq/WAOR5pDzvLI7S/LRSfb1KNgfvUp1V5h8+ezaq7V45Dd0A4NPQJq2fJaP/yCuvn70pNskLq5+or3mropRdiB18JT78O7P/V0S2uvtq90KsACBs1+kg5DIaahp7UBfDzrsPT8gAPj57bD1yGcC2UAFk3Sug7WYA20IFkJ3V/HyP3QcpnhEjFgMgFgMgFgMgFgMgFgMgFgMgFgMgVksAq0OHceI2Ownvxzd2X28Uoga0BOAEubNiAMRq3wStDr+ZJOWg82qi5lzk36NUJR6tpGN1+DJJHl9LAOJen3yqZvXkL9nHYkOegYx6mftRmW/v5QBgksW0mLRVcZvM1sczOZOrEhPlWE1EnEfCpOa51FC1nLMpNhQZbADIGlBsRv+90ckFwGzTqqyeKBD/u95OzB3l38P5Uk3Ly4DKzcWGIoMqgCdDmZV3aYLmW836RZLPzS/l5O4msXCKIB/Or9Rdh0d5HnLWV23IM6gA2OTbd7kDuMoDuj7O2vL18cF8l04FwNZEmQKw2SAzqALIN+P8yJi1bw0QymK8FPFcVmuAbIKeiOAut+4zkcnbG3JIFQBlk9Vv7QlANuV5gFeTGoCtTjgjlAddJhcbigzE7XXiRiL5cVbmi/57o9O+NWCZ32ietdkHf90cyLnzpWzJi9PQ/KBX+xYbigzEeefzL2XuF9lOxebey+eF2CCO4H3FAIjFAIjFw9HEYgDEYgDEYgDEYgDEYgDEagngdlfV741iYy2NAdAaGQCxkQEQGxkAsZEBEBsZALGRARAbGQCxkQEQG3EAvF8sApa5V0Y0ADAEkUaB0IgIAMIg0igQGh0BVPS+1F7ZDFk4NcBaDSI9DAmNHgCYEEQaBUKjFwDNDCKNAqHRF4AGBJFGgdDoD4CWQaRRIDR6BVBHEGkUCI2eAVQRRBoFQqN3ALsQIo0CoTEIgA2DSKNAaAwFIIcQaRQIjSEBZIo0CoTGwADeL4AD15GGC99IAADEINJw4RtpANghRBoufCMIwKdXZ2l6/2L6+feIAMwMIg0XvhEE4GZ6JiHcfIELwIAg0nDhGyEA7n73p7P0/o/v0rvfvkMGwMOmAACfvv1HdvTfff19ev+H19n3H2aqWN7vI1PdG44MAG5ORfPz4fMCgFAF4x41oKEWRHq84hvtALJD/9NODcAHMORxazuAm6nQqa8+oAFBpOHCNwKaIHUa+unVqYezoEYEkYYL3wgG4OM6wIAg0nDhG0EA6qrkggNgkBMHcQHYEIg0XPjGyAAMb+YmOgA5gkjDhW+MEMCwps6iBJAhiDRc+MZIAQxn7hIFQJIk2ABKY5AoEBoxAMiHD/sCUMpnFAiN3QHQQCLSuIKNaADG43EgALsEIo0r2IgBoOgDAAx40KiWhgFgcxZkY4B0ukQWLnyjI4CKtmcYMwbjvaYohzmNiXsd0FgReNSuluYFwIZB5fwI74qNJlz4Rm8AFIPqGSoPm9bSPAJYyArgC8CCIlz4xhAAtnoEHreupfkFsNhcIowZgDbNM4AtyfNTTACL8OHCNwYEUEDAA9CHmZvAAN5vQzAN4QFzjDauYCMBgEUOwTiIOpipMyIAQigAOj91Rg2gqUeA5xg0XPhGQgCqD2jolVvkGDJc+EZKAIV0CNrkGDBc+MYYAOiqwWBWHgMAfJhOf/nO+93RFQQMoJRYlXHzhadVko0I2uUYLFz4RlgTlEHwvEImR+A6ahcqXPhGGIDs0Pe3SnJHrtOZxuJ3QcZ1widPX3tdJVmvBu3vtQtzvOIbIQBSEXmvqySrDFxu9QoSLnwjDED69ixEH1DKeJHclGOIcOEb7QDytsf7KskaADliZx2+7vrzVwA14GY6zfqAAKskdwmUnysYDPdZ+A8XvhHYBFVVyQV/fUDNWFQH4zS/93DhGzsDoODAAGgB2G508RwufGPnACwq50ddfwRU9wAIbRHo+lKabgLYIqAxegwXvrGjACxGb+HCN3YVgGXY1Fe48I1dBVASaDL6CRe+sbMACgKNRtq4go04AMrvAQHkBLq+nhUZQKkAABbW+63xw4VvdAQAF95kWV322TNfvwpfXl9n66kGyDpgMSIfr/jGIABuSwbYvfW486v59ACuxIjXDBOAkAcAC8CSD8xw4Ru1AK4eXabp+vgIGcAt1SUzXrjwjToA62N58C8FBmQAwogLADRzjBYufCMBgFsIA3gNgM3ddwpAujyYe2qCtoxYAGB1INqlNPoakBRqqgQ4RcEBACTQIQAA4RUFAQCMQKRLafQ1wHcfUNG+AEAEIl1KEwUAob0AQAh0BsBV2QGMQjRB23IHACAQ6VomQw0wyV+ZHQHYCUS6lEYHACC/ZXYBADIilhHLGCWAW5cxC1sViHQtEwDA3cl0anmFibcytwBgIxDpWiY7ALEq4+43r42L9HyXGQRA3jfaQwAfRNQtCzQCldkCwLyuJjf6LmNbI6AJSitLlDSL9IKqcf5RAujqLKXlNVan5kV6gcdZ9qkBsa3mgwC4f3Gaml9lGHqgSw9godY1WQBEtpoPAODu5ExQiKAP2EgPQKkBQaSr+ewAVPzNrzIMP9RrANCAoLMA1Ms8z2iuA5plAqBFEOlySkATpBNpmZWMADQIIl1O2V0At2YAtStjBoButACoVIJIl1N2GcCtBcAugl1jsDLajJ0GABi3bnoCUbgyWozdBnALWUqjX8kRroy9BnALewKRBkAsC4q7DgD64Pb6I6AClrHXAFIIAO1AXcAymtI6DwB2s69upDRgGQ1p3QcAIsAAfBqBBGrjQyHL2JjWBwAgArrbqEOWsSnNDUBsclxVSV3sLXW7BkDvXwE1Qp2oAaRl1hpBAEAEGICbEQSgRiBsGXVpvQFgI9BwG3XYMmrS+gMAturM3ggxAGcjBIC9CjAAdyMEgJUAA9jLaAdgI8AA9jXaAFQJUJRxk9ZDANbbqM1VgAGgGE0AzFWAASAZDQCMdYAB4BkbAZguBxgAprEJgGFsOkIA8rb0yG7OhRv1AHRr+ijKCAHwQbzKkHSR3p5GKACKR2sBALx9+vesBsS1QKO1UfdSFA2B8G+pBzdB8SzScxRggoxylswKIKZFeo7G2uHesJYpaBnBAKJapOdohBEI+1gVMICO9wHKCAUQ8KkeYACRLdJzNEIIhH2mARhAd68DdowAAmFXdIMA1OWjKGGMdgJh17MODoCdQNjVfMMDYCUQdjXfAAFUCVQfM9Sw5MNTGYcIoELA8nbKZhIMwNm4F4BSDMDdWANgfDtlg1Cu2AYKoEpAvjDaBQCIAQPQGDUBHVvfTKYFYIfAAHRGXUhlNXB8ZqxjGYcLoOH+rTHgtUBaAAYGDEBvbIhr41PnrACaIDCABmNTXMcwBigvcnUE0BMZ1vPZX5NoUvuiDLIGGG8ildUA8DDkRsHKOHAA5renjGGPgrVCYAAGozGu1Yvk1gAUBAZgMloBiMZorOuXW5yvGoo1eAA1AtWHIReqcjD2D5ocG4rFAFq+wKngYO4fGnLUFIsBVAkA3yVt7h+ac6wWiwGUagXA0j8wAHcjtG+t9w/aJAbgaLQTqN9v3dA9MAB3YysABQfjHFv1f2MAEGMrALXugQFgGYEALN3D7v/GANobrQDqVWWne9j538AA+nFzLqbRYeps7A6gy4v0fBshAHKrO4BeLNDwbazHvmZ07gM6v0gvclkB9GCRXpRGMIA+LNKL0QgGwH2AHyMYQD8W6cVnBAPg6wA/RjiAHZGWuVdGBkBsZADERkcAFeFfmA0uRwZAnCMDIM6RARDnOMzb0yMSAyAWAyAWAyAWAyDWHgB2RulQ9GEqHhWLqU+vpk9f220tdHeCW0Z3ALuz9RgS0w64OaZvz+SUHp7uX2S/GjNHdwC7MzVYws1RlBFXuxOECHIHgF4UKeQ69fXfkJugiADsztbj6O4EOVwnZzJkeJJNEGYhY6sBuDl6KGPWCf/+2ygA+OkDRLeJp/s/ezlIMHuWfc6CTpFbbA+N2lv0JigrH+qvjuo64GaKfdaelRH5yiK7VkH91XwlTCwGQCwGQCwGQCwGQCwGQKweAfj4prZpdTg3Jceg/gDYDnb7ZDIxAGJ1GMDDeZKM8siuDr+ZJMnR6vBlkjy+3k5TKbOVSKYusU7dBfBwPkrXx7MCwFz8XU0eXYrtxT+xeZIBuXp0yTUAW0VEdwHM5Oflo8s0XcqoF5sYALZkkNMKAPE5qxZL0QypbdvJMYoBEKu7AJqboCeXy4N52QQxAE8qOtr18VH292AuO2RNJ5wDEMkxqrsAilPNVJxhPv9ynl4kI3kaOkp3T0MVAJFMXWKdOgxAo1jbGYMYALEYALH6BaCDYgDEYgDEYgDEYgDEYgDEYgDE+j8n5sOcCm+U+gAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
</div>
<div id="manual-plotting" class="section level2">
<h2>Manual plotting</h2>
<p>Since <code>cutpointr</code> returns a <code>data.frame</code> with the original data, bootstrap results, and the ROC curve in nested tibbles, these data can be conveniently extracted and plotted manually. The relevant nested tibbles are in the columns <code>data</code>, <code>roc_curve</code> and <code>boot</code>. The following is an example of accessing and plotting the grouped data.</p>
<div class="sourceCode" id="cb92"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb92-1"><a href="#cb92-1"></a>opt_cut_b_g <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb92-2"><a href="#cb92-2"></a><span class="st"> </span><span class="kw">select</span>(data, subgroup) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb92-3"><a href="#cb92-3"></a><span class="st"> </span><span class="kw">unnest</span>(<span class="dt">cols =</span> data) <span class="op">%>%</span><span class="st"> </span></span>
<span id="cb92-4"><a href="#cb92-4"></a><span class="st"> </span><span class="kw">ggplot</span>(<span class="kw">aes</span>(<span class="dt">x =</span> suicide, <span class="dt">y =</span> dsi)) <span class="op">+</span><span class="st"> </span></span>
<span id="cb92-5"><a href="#cb92-5"></a><span class="st"> </span><span class="kw">geom_boxplot</span>(<span class="dt">alpha =</span> <span class="fl">0.3</span>) <span class="op">+</span><span class="st"> </span><span class="kw">facet_grid</span>(<span class="op">~</span>subgroup)</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAMAAACKBVRjAAABMlBMVEUAAAAAADoAAGYAOpAAZrYZGT8ZGWIZP2IZP4EZYp8aGhozMzM0NDQ2NjY6AAA6ADo6AGY6kNs+Pj4/GRk/GT8/GWI/Pz8/gb1CQkJERERJSUlLS0tNTU1NTW5NTY5NbqtNjshgYGBiGRliGT9iGWJiPxlin9lkZGRmAABmADpmtrZmtv9uTU1uTW5uTY5ubqtujshuq8huq+SBPxmBPz+BgWKBvdmOTU2OTW6OTY6ObquOjo6OyP+QOgCQkGaQ2/+Xl5efYhmf2dmrbk2rbm6rbo6rjk2r5Mir5P+0tLS2ZgC2//+9gT+92dnCwsLIjk3Ijm7I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrb///kq27k///r6+vx8fH/tmb/yI7/25D/5Kv//7b//8j//9v//+T////b8gMVAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAJsklEQVR4nO2dDV8bRRDGA8WXpqEKKq012he1oMWitVJtQOiLtZQK0lIoKDS83Pf/Ct5esg1XyeRmM3uzezzPryU0mTwzk39295pLNrUEUlVNu4CzLgBQFgAoCwCUBQDKAgBlFQWwFYMirBUAtAQAygIAZQGAsgBAWQCgLABQFgAoq2QAr36uX+p328vPHg7pbuUXgFydRiUDoIoHAP8ANm/U35tPf7z/cOvV3V/q9Usv6mZEvPykXr+ZNda5TaipoX2MfNZppDACHlzaen4+nYzOpw2dN9dsfj1vLnq3CTU1tI+RzzqNygdg2tj85uGru/Nb5q/5Z6r0mt5tQk0Na5PJZ51GCgBu1OvpRJRr7EE9HdG924SaGtYmk886jRQAdJ45JxrbvHGzO7Rln1USVl7rNCofgJk/X3z414nGzJUvP53v3SbU1LA2mXzWaaQAIDsUyg3t5/X6B9/e7N0m1NTQPkY+6zQqGUA5irBWANASACgLAJQFAMriAviHI160nFWEtQIAAAhaRVgrAACAoFWEtQIAAAhaRVgrAACAoFWEtQIAAAhaKdd6gZUXABwS0AIAALDmrGg5KwCw5qxoOSsAsOasaDkrALDmrGg5KwCw5qxoOSsAsOasaDmrKgOABuuCy50wAljRpDAFAYA1Z0XLWQGANWdFy1kBgDVnRctZAYA1Z0XLWQFAR63lFqtwsk5WMAAYtVbWVlq80ok6WcEAYAQAhfICgEMCWqEAwBpQKK9HADgKKpIXABwS0AIAALDmrGg5KwCw5qxoOSsAsOasaDkrALDmrGg5KwCw5qxoOSsAsOasaDkrALDmrGg5KwCw5qxoOauKAWjPNr/aAIDiEgZweG8h2bm6CwCFJQygPbebHP76FAAKyxuAC6n6hnmWWmIHCb8zLpuCvtQeAS7PKlaCcEeAWYR/egwAxeXhMPTwN3sYxKoEAArkHTwFpfP/zjX7L1YlAFAg7+ARsN9svj0KjaopVoKAAeTEqgQACuQFgO59WNGkACDCWgEAAGgBQIRNsRIAAC0AiLApVgIAoAUAETbFSgAAtAAgwqZYCQCAVmgALriqT60AwAXwb38lxG0A0Pc+rGgAoPMCgFNTlAAAAHICADIvADg1RQkAACAnNQBaCu2tia719L0fRgArGlMQnRcAnJqiBAAAkBMAkHkBwKkpSgAAADkBAJkXAJyaogQAAJATAJB5AcCpKUoAAAA5AQCZFwCcmqIEAACQEwCQeQHAqSlKAAAAOQEAmRcAnJqiBAAAkFOZAI4fNd/u1wQAXZUJYH0h2XfatE/yGzR8A5icmuTVUx6A3mZNzKZEv0PGM4DJ69PXWQRKBNCee9Kdgpib9i2vra0tM+IT54/9cJKcrqnp6ekpVq2OiQa/M+5g5tbBTC3T6Kq5oj27kO2c2BHjScIeAdLPKkbqsEeA676h3DVAEUDQa8D9sjZu1QSgXWseQGca6sxA2VGQ0xSk3lRMteYBPBtLtkdXt8e6Q+BeSZt3A0BvABwtjiV7dgicVExNxVTruwAOZiYAoMxacwCOFif2RpbMRAQAZdWaXwPeXKyNJc/OvQaA0moN4tVQAAAAtVpPvhRRs8IiXF6t+RGwbR769FgII6C0Wt89DDUXOAwtsVYACAmAnYImhpyCxsfHVZvi5A7s1dA9swaftgRwmhq/8sUVFgHpfdgYqUM+HwAAp9ZaSQDCTTFSVxIA1gBtANpHFjHVCgAAAAAAoFgrAMQCwKfE323mUZXcMw4jAADUagUAAAAAAFCsFQCqCKDRaKg2FVOtPgA0GsyuFAE0JqcmebUKn7sIA4DeCRkAGNgwJ7gMABFMQex5lWyYE1xCrVEAwAe1AYBxH1Y0ANB5AcCpKUoAAAA5AQCZFwCcmqIEAACQEwCQeQHAqSlKAAAAOQEAmbdaAPabJW3WQekMA2jf3kh2rrkAiOnV0MDfnm4gsJtiv8ZONswJ5tca+gc0OiPAnNAhw/JqTE1NNRjxpDy//ZC/aZ+r+hlSANqzThu3VnoEUHKpdcAIcNszrsprACUPAMy+cfymqnwUREkYgNm2uJxdEymdYQDJThObd7PkYwp6K1YlAFAgLwB078OKJgUAEdYKAABACwAibIqVAABoAUCETbESAAAtAHBoKqYX46oIoHH588uxvBwNAIPkHQArgaAVALgkELTyCCCuNaCKALTnVX8JBK0AwCWBoBUAuCQQtAIAlwSCVgDgkkDQCgBcEghacQFoyfvGfIIJnKwwAljRclYA4JJA0AoAXBIIWgGASwJBKwBwSSBoBQAuCQStfALAq6EF5BFATFuWAcDAOlnBfWoV22mPFgD0BSD1yTpaoQAQXQMAwAGA5FEQAABAYQEAAFhzVrScFQBYc1a0nBUAWHNWtJwVAFhzVrScFQBYc1a0nBUAWHNWtJwVAFhzVrScFQBYc1a0nFXFABzea17dBYDiEgZw/GjBcc84QQCt5RYnb6UAHP62Qe0ZV8pL7K2VtZUWuymxWnUBtOfsfkGmlP/d7NxU34SnaHltbW2Zc4fTVUqtjgp7wyahEUDeh1eRmNVgAL0RoNiUzBpA3ocVLWc1GMCANUCsEkGrCGslj4Lu6B8FnWUAFfp/gFgCQasCAHLyV4mgVYS1AgAACFpFWCsAxAKAJe3PXSkl8PoZMZa0m1JKAADKCQBAOUE4AKDiAgBlAYCyAEBZAKAsAFAWAChLFkB77kmzuZCdSuh9DaWD1hfMGdGuS+47LeUUSK3CAGbvZCfz1zsXztq/Y/6sZ6fkzKnR3pk5OQVSqzCA29l5ZFNH73y+gw7v7/791DikTkMZEQqkVi8AzMXx4yGaOn78530zqrMv0mrPnvg6LTkFUmuYIyDZad3JxnNXQ80RfRRIrV4ADD2vJu0f0oek42KMPALQrtUPgGGPLJLjPzay4xMzntc9HQWFUWug/w9o/6hdQXENV2uYAHZ8rLqeNGStYQI4QwIAZQGAsgBAWdUB8OajpXd/3RtdVSunqKoD4BQBgLIAwKveXKzVarc68036w1weLdZqY50p6GCmNvJ7CsBcFTKHeAF0HviLt04AOFocSx/4jMnBzET66+iquSrZPvdau9q+ihjAx53n9QkAdvFNL7PZZ3t0Nbs0UEJVvACSZ9l0cxKAnfPTX7Mnfcpou5ZpQrdUQhEDMM/sdHofACDg2SdT1ACyyYWYgtIfeyNLAyyUFS+A7CHuLrdHiyN2ETZ/s2vHuotwOgRCphAvgGQvndtHljqHo99/Rx2GBvz4xwygGgIAZQGAsgBAWQCgLABQFgAoCwCUBQDKAgBl/QeXRv3onuc2twAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
</div>
</div>
<div id="benchmarks" class="section level1">
<h1>Benchmarks</h1>
<p>To offer a comparison to established solutions, <strong>cutpointr</strong> will be benchmarked against <code>optimal.cutpoints</code> from the <strong>OptimalCutpoints</strong> package, <strong>ThresholdROC</strong> and custom functions based on the <strong>ROCR</strong> and <strong>pROC</strong> packages. By generating data of different sizes the benchmarks will offer a comparison of the scalability of the different solutions.</p>
<p>Using <code>prediction</code> and <code>performance</code> from the <strong>ROCR</strong> package and <code>roc</code> from the <strong>pROC</strong> package, we can write functions for computing the cutpoint that maximizes the sum of sensitivity and specificity. <strong>pROC</strong> has a built-in function to optimize a few metrics:</p>
<div class="sourceCode" id="cb93"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb93-1"><a href="#cb93-1"></a><span class="co"># Return cutpoint that maximizes the sum of sensitivity and specificiy</span></span>
<span id="cb93-2"><a href="#cb93-2"></a><span class="co"># ROCR package</span></span>
<span id="cb93-3"><a href="#cb93-3"></a>rocr_sensspec <-<span class="st"> </span><span class="cf">function</span>(x, class) {</span>
<span id="cb93-4"><a href="#cb93-4"></a> pred <-<span class="st"> </span>ROCR<span class="op">::</span><span class="kw">prediction</span>(x, class)</span>
<span id="cb93-5"><a href="#cb93-5"></a> perf <-<span class="st"> </span>ROCR<span class="op">::</span><span class="kw">performance</span>(pred, <span class="st">"sens"</span>, <span class="st">"spec"</span>)</span>
<span id="cb93-6"><a href="#cb93-6"></a> sens <-<span class="st"> </span><span class="kw">slot</span>(perf, <span class="st">"y.values"</span>)[[<span class="dv">1</span>]]</span>
<span id="cb93-7"><a href="#cb93-7"></a> spec <-<span class="st"> </span><span class="kw">slot</span>(perf, <span class="st">"x.values"</span>)[[<span class="dv">1</span>]]</span>
<span id="cb93-8"><a href="#cb93-8"></a> cut <-<span class="st"> </span><span class="kw">slot</span>(perf, <span class="st">"alpha.values"</span>)[[<span class="dv">1</span>]]</span>
<span id="cb93-9"><a href="#cb93-9"></a> cut[<span class="kw">which.max</span>(sens <span class="op">+</span><span class="st"> </span>spec)]</span>
<span id="cb93-10"><a href="#cb93-10"></a>}</span>
<span id="cb93-11"><a href="#cb93-11"></a></span>
<span id="cb93-12"><a href="#cb93-12"></a><span class="co"># pROC package</span></span>
<span id="cb93-13"><a href="#cb93-13"></a>proc_sensspec <-<span class="st"> </span><span class="cf">function</span>(x, class) {</span>
<span id="cb93-14"><a href="#cb93-14"></a> r <-<span class="st"> </span>pROC<span class="op">::</span><span class="kw">roc</span>(class, x, <span class="dt">algorithm =</span> <span class="dv">2</span>, <span class="dt">levels =</span> <span class="kw">c</span>(<span class="dv">0</span>, <span class="dv">1</span>), <span class="dt">direction =</span> <span class="st">"<"</span>)</span>
<span id="cb93-15"><a href="#cb93-15"></a> pROC<span class="op">::</span><span class="kw">coords</span>(r, <span class="st">"best"</span>, <span class="dt">ret=</span><span class="st">"threshold"</span>, <span class="dt">transpose =</span> <span class="ot">FALSE</span>)[<span class="dv">1</span>]</span>
<span id="cb93-16"><a href="#cb93-16"></a>}</span></code></pre></div>
<p>The benchmarking will be carried out using the <strong>microbenchmark</strong> package and randomly generated data. The values of the <code>x</code> predictor variable are drawn from a normal distribution which leads to a lot more unique values than were encountered before in the <code>suicide</code> data. Accordingly, the search for an optimal cutpoint is much more demanding, if all possible cutpoints are evaluated.</p>
<p>Benchmarks are run for sample sizes of 100, 1000, 1e4, 1e5, 1e6, and 1e7. For low sample sizes <strong>cutpointr</strong> is slower than the other solutions. While this should be of low practical importance, <strong>cutpointr</strong> scales more favorably with increasing sample size. The speed disadvantage in small samples that leads to the lower limit of around 25ms is mainly due to the nesting of the original data and the results that makes the compact output of <code>cutpointr</code> possible. This observation is emphasized by the fact that <code>cutpointr::roc</code> is quite fast also in small samples. For sample sizes > 1e5 <strong>cutpointr</strong> is a little faster than the function based on <strong>ROCR</strong> and <strong>pROC</strong>. Both of these solutions are generally faster than <strong>OptimalCutpoints</strong> and <strong>ThresholdROC</strong> with the exception of small samples. <strong>OptimalCutpoints</strong> and <strong>ThresholdROC</strong> had to be excluded from benchmarks with more than 1e4 observations due to high memory requirements and/or excessive run times, rendering the use of these packages in larger samples impractical.</p>
<div class="sourceCode" id="cb94"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb94-1"><a href="#cb94-1"></a><span class="co"># ROCR package</span></span>
<span id="cb94-2"><a href="#cb94-2"></a>rocr_roc <-<span class="st"> </span><span class="cf">function</span>(x, class) {</span>
<span id="cb94-3"><a href="#cb94-3"></a> pred <-<span class="st"> </span>ROCR<span class="op">::</span><span class="kw">prediction</span>(x, class)</span>
<span id="cb94-4"><a href="#cb94-4"></a> perf <-<span class="st"> </span>ROCR<span class="op">::</span><span class="kw">performance</span>(pred, <span class="st">"sens"</span>, <span class="st">"spec"</span>)</span>
<span id="cb94-5"><a href="#cb94-5"></a> <span class="kw">return</span>(<span class="ot">NULL</span>)</span>
<span id="cb94-6"><a href="#cb94-6"></a>}</span>
<span id="cb94-7"><a href="#cb94-7"></a></span>
<span id="cb94-8"><a href="#cb94-8"></a><span class="co"># pROC package</span></span>
<span id="cb94-9"><a href="#cb94-9"></a>proc_roc <-<span class="st"> </span><span class="cf">function</span>(x, class) {</span>
<span id="cb94-10"><a href="#cb94-10"></a> r <-<span class="st"> </span>pROC<span class="op">::</span><span class="kw">roc</span>(class, x, <span class="dt">algorithm =</span> <span class="dv">2</span>, <span class="dt">levels =</span> <span class="kw">c</span>(<span class="dv">0</span>, <span class="dv">1</span>), <span class="dt">direction =</span> <span class="st">"<"</span>)</span>
<span id="cb94-11"><a href="#cb94-11"></a> <span class="kw">return</span>(<span class="ot">NULL</span>)</span>
<span id="cb94-12"><a href="#cb94-12"></a>}</span></code></pre></div>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkAAAAHgCAMAAAB6sCJ3AAABdFBMVEUAAAAAADoAAGYAOmYAOpAAZrYAsPYAv30ZGT8ZGWIZP2IZP4EZYp8aGhozMzM6AAA6ADo6AGY6OmY6OpA6kJA6kNs/GRk/GT8/GWI/P2I/P4E/gYE/gb1NTU1NTW5NTY5NbqtNjshiGRliGT9iGWJiPxliYj9in9lmAABmADpmAGZmOgBmZgBmZjpmtrZmtv9uTU1uTW5uTY5ubo5ubqtuq+SBPxmBPz+BYhmBgT+BgWKBn4GBvdmOTU2OTW6OTY6Obk2ObquOyP+QOgCQOjqQZgCQkDqQkGaQtpCQ27aQ2/+fYhmfvYGf2Z+f2dmjpQCrbk2rbm6rbo6rjk2ryKur5OSr5P+2ZgC225C22/+2/7a2//+9gT+9vYG92Z+92b292dnIjk3I///Zn2LZvYHZ2Z/Z2b3Z2dnbkDrb25Db/7bb///kq27k///na/Pr6+v4dm3/tmb/yI7/25D/27b/29v/5Kv//7b//8j//9v//+T///8WwKCJAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAgAElEQVR4nO2dC2PdtnWA5SymmiVemm3Oui5y27Sd3XSb3K1p90j2kLulc9qgyTKrq70tipNYa1PqSt7kB//88CRB4gAECYAvnBNHlxc8BAHcjwcPAgd7FQpKgOzNnQCUdQsChBIkCBBKkCBAKEGCAKEECQKEEiRxAfq9nASznwCgR/kIBNDcaZpQEKBQQYAQoCBBgBCgIEGAEKAgQYAQoCBBgBCgIEGAEKAgQYAWCdDDa9euvfyJ+vbZ1z5on24CxNFnr1P9a195pwk0LnGFh0hMgEQ2XmIpfEAPbrKwdkn4R9XJZ6vEIhbCQgF6wArx/brcHBmWALUVbPrR6XkUGSCevgc03+z/z998wyiJgVFB3+MWwjIB+vzbzJZ88aObLLOffe3nb1576eff+RtuYT5/kz2h7Cn6+l/TZ1R8bQrlIXuAWeDPvvbBFz/+22vX3qAhb4hn+6YMV3HwCAJTmgIg+vfz76gDVRLsNE33V94RRfLBZ3/8XWaYvvjxO7II1HmRVR5VS599JCiEZQL0UD1xTe4/f/PlTx7SHL//BjvLwl5XR48agFjBP3hFXvLFj+jR66+o30EEMk0Zh4ggMKlpLJBIF+VGTyDD6OHL/ymL5PWbFB4KwCc0NzTL9fn/qrPa1k9UCAsF6BV5oAN0kz9ulBBaEOqRUtW5aDy8/Il4YFUB0gJm/8tAeqmMqo4jhjmP3waiv6csgPdv1iXxSK+tZb4fUHTeYLmrDZbUrLPa1n+UoBCWCRB9ruRB6/GhBcoCKBQGQKoM6E/wlXcggN5ndVvz/Ik4FgcQTc5DVlE/VCalLolHqlT0Kunr/81rMNl9ULp1Vrv6KQphmQB12kAKIIcF0spA1nAtgJj9qgtuwRaIJYc1m8E2kGFRvvjx3339E6FaX/2oldVHXSMevRCWCVDd92Dl9+AlkftXeGXdtIEggJgGBBBX/iPFXR3HIgFiTTdBUacXxjCiPYq6SFhfn2q8r1ox4vzP6qx29ZMUwkIBqkc/Hly79iffoUXx0s+//aedXpjIOj31gWo80O4EN9K0KEUvrKnCaDy/+92bIryJY5EAPWINW14AN/WSeKR6WaJIuCrFQYW2ztOsfqZ6YZp+ikJYKkCGyJbw8gRHohGgIEGA1gHQYgUBQoCCBAFCgIIEAUoBUE6C2U8AUHN4Bp0HA9OoJr8VBNDcaUqtCmQfAUKAEKBZboUAcUGAECAEaJZbIUBcnABd/uWnVfX03YNv/tbyUelH1aJKEAFKoDoQoIuDb3xaPf/FUfXlt+APKs2RHmuqDCBA86oOA+jkxsfUAj39h0+ZJQI/qFJzpMeaKgMI0LyqY6qwyx/+tnr69/fAD6ZSH/HRpbN8BAJo7jRNKN4AXXyTEwJ+UJXmSMfSg+C+0GVaINIc5miBgOzHs0B6rKkyMDdAhDRFmCFAUPZ7AcI2kCYIUH3sDdDzX7wt+l3AR1VpR3qsiTIwN0CEaEWYH0Bg9oPGgbiFymYcqKyFf80UIEUQjkQPVi1LUYKZAmR5fhAgf4AIJ4h+8K/5AQQ+PwjQAICI/Me/ZgcQ/PwgQEMAIvwxzBUgwQ8CNFKVNADxhmRmANmyHxmgmcbVJ5HyjJTsL/t3lt+rDJp9kfN29tEC+Vdh9Ck8Yx+ZVmE0+7wBjVXYSFXWfCzYZ6YAkYo+PyUCNFq1JFkPJLLnBzDACJC3Kut+FM3X3ABiA0AsEMeBxqoiQDgSHdaNr3IGiLCqSwtEgIaq5g5QWY0E6MuDAza5PvdVGaQsNKlyA4gboHEAnRyxv9mvyhAGKFsLJF5gjAHo+Yd8vmruMxJFAWYLkDBAowCiVdTBwRE8NbrKZ1UGe4txVtD/asnqVQYpuyHeAF3+4B6zQpmvypAGKNdGtDRA43thJ0eZr8pQXbBcASqNwKEA5d0GUgYoU4CUARoFEKujnn/0ad6rMuoxoEwBKs3AQeNAN+7lvSqjNkBQCVZg0NzZj6laG6DxVZivLKgEowNUdAIzAqgEAhEgf9XGAGUJUGOAEKAAgIquaj4AlZAmAuStqhmgHAHS+EGAxgNUGKoIEBcEqE9VN0AZAqTzgwCNBqgwVREgLghQj2rLAOUHUIsfBGi4qujDFoAqAsQFAeoBqD2RNTeA2vwgQINV2xUYAtQcIkBeqt2Z9JkB1OFn3QDtKxl5/RjVrgFCgGpBgHxUuwYoM4C6/CBAA1UNA4QA1bJigHbjrh+hahigvAAy+NkAQLspATINEAJUy0oBmtQCmQYoK4BMfqAZvS2ATve4XAdv4yOpAdpN2QYCDBACVIsJ0JNbipzTvSt3wTv1yqYa0YABygkggB8nQE/eut+cbX0ZIFsCCDJACFAtidpACVfXqhpsP+E9dJGLma0CATRR0qYQczHz2Rm0sns9jWgq+wNiDX0uQQOUkQWCDFC/BaKt6MPTF38D3sZHtgZQl598AAL56QXo+MVf3zp8ducqeBsfSVuCuwkB4i6hDX7yAajsBBaFxb+WDtCTW4f0X3X+wn3wPh6CAKVM04SqpAsQF8g90poA2h8Sa9ituE9kk59sACpB1T6AqlNWhT25tcyBRGaAEKBpVAkIEOgeqd2IPg8biN4MQKwJeQbwkwtAJajqAVCoJC3B/UGxIkDjVQkIEOweaT0A7aYDCO7Cd1Q3DFAJqvYA9OTWnpJlNqIRoKlUuz7FhVjcI63HAu0PizXgVoKfPtXtAlSCqgiQt2rmABk+xbnY3CO1AHr86nKrsN1kAMkKLFuASlDVB6Bnd64/u8PHEsdKwhLcHxgrAjRSlYAAWd0jdUaiq+Pr1fn4t6lbAEi1oHMFqARVvQE6vbrMVxm7qQCqvSlkChDgU7xyuUdqv43n9ATM59gCQKoLnylAJRToCxBtBFXHoydEVylLcH9orCNvBbqEBlW3CRDkU9zpHmkl3fjdpAB1exyg6jYBKqFABMhbFfboC6puEiDQp7jTv5ZRhYVMSExXgvtgaPxb5Q5QCQX6A3TM2FnilNbdRABZPPqC128RoBIKdPvX6nbjqRjdeL6d06yb7iJA06imAeiCbdo876a7+2Bo9FvZPPqC128QoBIK7PGv1Z7SytDpTmk9ufEx29Vpzg3ndgjQJKrBAFmmtDJAZt10dzfNatS+tahtgQBKmLgJBFqLai2QId14BtCsm+7ug6HRb9WeR5afBSqhwD4Hbb4Azbnp7m4agOwuocHrNwdQCQX2+tdqAcQXhpnTgS5nbgMhQJOoRgDo+CprR592x4EYIHNuursPhsa+lcMlNBi4NYBKKLDfv1anG89GEf3HgabZdHeHAE2hGgcg1odf2nygaQByuYQGAzcGUAkFevjX6kxpPb9yV7zQGCcIUNQ0TakaAyA2qf5qdbww/0A6P+kAcrqEBgO3BVCXH6Hq4V9r+dM5pgPIWt+DgQgQFwSITYPucQkNBm4HIALww1V9HLQtHqAWP0kAIoQABigPgMpaIFUEyCuUAtTjEhoM3AhAPP8wQF4O2nIHSDx+rAQd9T0YuA2AqlI8P5sFqM1PCoB4AZJsAaJ5p/8ggPwctLUHEpWDl7EjQSsEiBUgxcdZ34OBWwGIWV8CbcwzHKB6QtnoadFrBIibH0IyBYg/PQQCyNNBGzyldezbjOgl2OEnCUAMIVLAAyGOQASICwIkSjBfgEpeixvZ93XQBs+JHrs8fnUAiRYQKQvWkBwW6xYAYlkXrWgj+6MAEnOiD6vTxewXlt4C0ULk3iMztUCl8ClujAN5O2hbdje+y08CgJj5YfzkCpBlJBoB8m4DsdIrLGP5rus3AZDdp/g4gE5DPdXHXGVyNsl6HlKeFd5LeXSBAIqduAlk0FoeTdyN6NEANYcxHhbDAMW3QPwtal+DEQzchgUqwdABDtp81sb7y9oAKmjvq7UL1oBYESAueQPkMEB5AFSCoUMctC25CjP5QYAiq0YGKLwRDd6rJ3BGgMQ8snwB8nNKD16/hm48ApRatQRDtwIQwE9kgORE1mwB8nRKD15vALS87Z4QoNSqJRg6EqAosi6A1Ez6XAHydUoPXr98gHbVBADZq/sMACrB0O0ABPATFaB6KU+mAHk7pQevR4BqA5QrQCUYuhmAwBosJkDNWsI8AfJ3Sg9ev3yAIH4iA+Sq7jcPUAmGxgDofG/0hj0rAkhbzJwlQAQEaLiHv6VaoN1ZeoCc1f3WASrB0A0BBPITDyDdm0KOABEQoBEe/jIGyF3dI0DuSEGALG5+uVzePjg4qqovDw7Y1hmpN1uB+2DxACJV2WOttw1Qax1YHTrGw5+Pm18mzJn45Q/uVSdH7FvyzVZ2+6El2ANQ2VfdI0DuSC0zEkE3v0wuGCQnR88/5E7pkzsaTwxQywBlCFB7IaoKHeUicoibX2qFaIXFarLUm63QPlikmGBhSzFC41jzqgwCrsUYVCQQQD1ufplTelaLUSuUerMV2oROaYHaBig/C9RZCS9Dx7mI9Hfz+/Tdt+XRyVHqzVZSA1T2V/cIkDvSwd34y9tH6vDkKHEbiPXBEgLUMUDZAdR1xSFCR65q8gVI8sNqrOcffZp4s5XUAJUeDxsC5I506JRWNv7Dms/088a91JutsFHEdAB1DVBuABm+gHjoWBeR0LqwQzBFPhKnBPkoYkKAuv40EaAqDkBLWZmaFiDDAGUGkOmRlYWOdhG5SID2vVVH3MowQLkB1OUHXlgQUoXNvbQ5KUCmAcoLIMAlNLiyaRRAysXdaIlSguJFajKADH4yA8jgJyZAoRIHoH1v1eG3Ah7ArACC8g8u7kaA4OsBA5QXQCY/EQF6/OoCljbLqUBpAIIewJwAAvMP+rcZA9DoHQ6MWD0SYA2VcxETAQQYoKwAAviJB1DIEKKMNcJUg5SONYnFp+QYWeN0jk7+i5YMiwq2QMEANYdjHxY1mTWJBQINUEYWCDRAvU7pwUCwDRQwhNiO1SMBttCUAIEtgIwAogYICA1zUgvtFzZrI1otx0gCUBnxx1ojQLABimeBgiW8BOvlGAkAogYoa4BgA2xx8YcAmUGkQIBMiQrQ/F5a6wWF8QEiYmOaMamCAlcHEGyAbS7+RgE0/8vUZkFhAoAKBMjUiwnQAqZzJASIWDamyQUg2AD7+BQHA5cK0D4Q2BeBJ0AFApQYoPmrMG1JfGyAiGVfkdG3WhlAsAH28ikOBi6zEZ0SoAIBSg+QU/jyL7kII5F3Ds2pS2SA5LbM2QIEG2A/n+Jg4HCALphbF+mMI5F3Dt2pCwIUVXUagJ7duW6d03Fy42O2CkwsRE20MjUdQGpf+FwBIuDGTp4+xcFAECDuVsE6K4hxIpfCJ/LOkc4pBxuEjiyrms4Bz2MJ8VAyohvPAJLOONJ452i5JYtqgZQBytUCEXBrOV+f4mDgSIAcFkiPdVCqEKDkqgTcGSw6QD3jQJep20DJAKr5yRQgAm4t5+1THAx0rAuzjQMxTqQzjiTeOdqONWMCVFg2pgm61aoAggxwCoCcYo4DxfXOkQygxgDlCRABt5bz9ykOBi5wPlAygArLxjRht1oTQJABTgLQ6d7e4anFxZ2PhJRgxzV0PIA0A5QlQATcWm6AT3EwEB4HevHXwtPvWFkkQIVlY5rAW60IIMgApwCIu/k9nGs6RyqAdAOUI0AE3FpuiE9xMHBxAHU3N4gGUGHZmCb0VusBCDLASQCqTlkVNtN8oFQAFbadjUJvtRKACLi13CCn9GDgiHEgD1kkQKWv6rBbIUBcFtON7+7vFAmgtgHKDyACGuBhPsXBwKUBZOzvFA2g0ld14K0QIC7e+4V5yeIA6hig7AAioAEe6FMcDLTNB7LtF+YnCwSo9FUdeisEiIvvfmF+MroEzR0KowDUNUC5AURAAzzUpzgYaB0HcuwX1i/LA6j0VR18KwSIy4D9wjxkYQAZBigzgLr81GZ5bKx9bSDnfmE+MrYEgT12Q0uQrfmGvfp7p8odiABxWUY3PhFAhgHKCyCDH2WWR8eaE0AEMkAZAUSmBkiMAYW6uBuzQGS3S7CchxBSnEX0yWoKBFDC2w0Umn8C5j94s2Epi7FAOy6GAQp9BCk/3rsabNECMYDAGnyEU3owcOMAkcwBIjaAxvgUBwOBKsy55aWfDC9Byc9u16/qGytzms34ISU7Gn69p+qSASICIKAJmA6gKLIIgCpRgRHIqW1WAJkFcDbKmx0YuGmAyrIoSclk3PVeqgsGyMZPUoDmqcL2qezYn37VAbFSgFj5kUwBIgog0j0D8rNqC5QIIGqAGD+ADfeOdMUAMdMLG+BNAsT5iQoQb0DTEgSewRwA4jlnBqjsZr8ITYC7CosxkDg0VUkAKljB8ccvUwvExqDPaAF0s58UoCgyAiDBTwKA2FG2AJUV25ink/0iOAF5AFTwl0A5WyCLAU4P0PR7pip+IgJUiLeIZ5X5COYNUBGegB6AZtgzNT5ABd8WiztlNxsBeQDE+TGrsPQARdgzdWiqUgBEeP3FAvMcB+LPD7fArexbNzWIaYEmB2hHGYpagrwCK2sZfL2/6oIBUhZ4aoBm2DM1NkCiAWTdlycHgCzPj31TgzU3otkbsJgANfxkCxDLP/QATQEQc1QfJiMA2o9ZgoXNJfKgVPmprgsgx6YGK25ECwMUrwQ1A5QrQIqfWQDyaURf3j44OKqqLw8O2N4rVdhuPZEBYoWkDFCmANX8dFRdu2LEbAO9dh+8QyPMKf3lD+5VJ0cyIGS3HslPrBKU/CBAswHk8TL1grFycvT8Q7W5QYin+rgAtfnJE6CGn7aqc1ODiBbIT6gVovUWr8nCduvZncVczsPWq5CztEt5dIEAmujWNiH8b8K1PJqMB4htbsBqMWGFAnbrUQYoziPYMUBZWiDNALVU3ZsaRLJAT96635xtfWnL03fflke8HRSwW09UgLr85AiQzs/0ALEWkOyEne5duQveqWK9MNV8FgCNbwPV/MQoQV5ECFDV45Q+dRvoWDSiHcOJkh9WcT3/iGEzfree6AC1+MkQoBY/8wDUL2z8hzWf6eeNe0G79TT8RChB0wDlBxDnp8cp/fwA9chMAAH8ZAoQ9ABtEyC+kDASQBA/2QHUMUCNau+mBusFaN9PtTcUATL5QYD888oXq3T5yRIg6AHq39RglQDpNVhgCfK1Bkb5ZQaQYYBmAmiyHQt1AxRWgmKquGGA8gLI5EepemxqEBGgyXYsbBmgCACZBggBEh+TAjTdhnMtAxRUgmKquGmAsgII4Eeq+vjXWiFAbQMUUoJipi/AT04AQfzMAlCEHQv9pgLs2J8YMznENAXC/kw1i6OW5UznIPwvVAAJpnEogRvR0+xY2DFAAY+geMAgA5SRBQINkFD12hVjfd34aABJflggAgSo+m1qsE6A9o1QUNUZquzPGcBPPgDx5wd+gCYHaJqFhV0DNLYEZfFkDhCprAB5bmoQD6CJvHNEAqjmpzoD+MkMILMAihkAmmZhocFPOEAAP7kAxJ8f6AEqvDc1iGmBVgRQw0/OAJFqSQBN4p1DeBQPBkjjByq+vACCCqC7yUOUBCyhEW0aoDElqEonc4CICFwKQFN45wAMUABAgp/JGqxLA4iIQPAB8t/UYF2NaMAAjShBnZ/sAerwU7QkbgKiNKL5KrBmLcbAVRlRAOpUYLkCJPiBa3CwBkvfBur1zlFVF8ytS+OTY6B3DqgGG1KCRf2nsjvjcly/JYAUP6DqLAD5bHVwcuNjthisXo86cGUqZIAQoHGqLoDATnx6C+QlDJdmRfww7xx8HkfIRI7irJmjQNifyWdx1DL3dA4iPsACKJLO5OASBlDjk2OYdw7QAA20QC37Y/enuXkLVLeATNWimqEX5r1bj80C6bFaErDjgWEA1QWjKrBMAWpa0AsByFsux7eBYgKkdWFzBEjrgRmqrYZi1AT0WCAwdkMYLo1PjkHeOThAnrsaWAAC+EGA2uJwyLoYgOToz0DvHLtqPEDd4bGmAssSII0fo6QSJiAKQC5JBxCTAqzAcgRI8pMZQDseaPLjmYFCbxrqBihfgKAa3OmQNS1Aqbf9DgJINn7aL8EGXB9fdUaAWgaorep2h7hqC7TjgQA//RloXgo263j63CFuGKA2PwhQX1r11zoIUGW8RG09amkTkB4g+9C3eIsx/DVG0RqT518IP5zvJYaQ2V5lEPk5jUNxm0xugfhbjDPIALlg775UJpXLGVdvpGu3QE32+9xpTm+BokhcgIz5UBUhJGeAtOxnBxA4EQhWlYHQjBZWggY/eQFk8NOo9jsjywggcEIU4cIPcwRIyz5kgD18Sa0XIMGPL0Cs7gIyYOEnF4B0fgCAfKY/rx2gfTgDnawXtv1hSV2E7WnAWQFkNcBLACjZujBlgDwAKqzvckjeABE3QF6ugBIDFNW5QitD3gAV9pFUUouXO8StAaRlH6rB/Ty5JAZowFBQ/6a7eo7qFpAboFa3HQFqyxoAGuBcoX/T3cEAFe5eRFOApMNPFgBp2e8+QEzV05NL6jaQt3MFj013TYBsG3yLKbz9dThRoTkCpGUfMMC+jjiSV2G+0zk8Nt3VXsv0vQYrOi+8QCFEHc39EkzKxO/CSHNoFMCEr8BqCevGe2y6qz0UzSCQjnXRlfYtOk8AqUO79icLCyQ7XxYD7O2II3UVNkzcm+42mdoZqHR46V9AwMqvrMWp6gxcK0BEDzUA8l+AkRqgYeNAlk13bajsWzPQWwC8/EqxAiFHgIgeamQfLL6ZemHXaUfMpzPvselunS1ZgQUAJMqvFIvoSk+X0NsBqH59AdfgRfKcgoG2caDj69W5x35P/ZvudgGy7xDfA5D2+rDkzpQyA0hl31aDLwyg06uRNltRVOgGaAxA9ePHCo4XX14AqfzbanDwtXPMBFgCwTbQMacnYMc5O0COHeKdADXmWwBUZQaQ9vyANXgxQU7BQBAg5iTxeO/KXfA2PgKMA42dC82lGfzhox9lyf+Oiiq6TDEOpOW/1P41MscIkJDk0zmkWWnVYEOfAKIHlMz+5NWI1vMP1+DFiFgDVPssULBYAXLtEG9LK2nxI0rvTFZjPtcPuNUY1fQAdZ4foAa3zZyKlABXoAGQt38glxgl2DZAgzJAOgG5DSR2n5/KBMjhh2P9FqgFkHODbzi0y89QV0BrB8h4foBRjBUDdKGmA7ndu3QMkH8GCOkGlvYItggQ8PwYTUCXH445qrABzhXYCKKPm9+xABEjsHREsD2AzOcHaAI6/XDMY4FOGTq+8xI9XNx1+fHMAK/9I+Y1jWpCgMznB2gCut0ozAKQRMdzJFrYHaebXzkGNHAQiABhCxn70SXdOBCBAhcxCUiTUIAub9/gczicbn5HWSACBJaw6uDANVgg0fvqbQL2+OGYtQrz27On382vwY9HBurOqx5YgqojAlcAEPT8VCZAruUHYQnwDIR7YXzfeL8mkHU+UHOvEQARKLAEVccELh8g6PmpzD5ErxuFhXfjm4rL4ebX5Kc3AwQKLEHVUYGLB4hAgYAvqbUD1D8faEdFHvYCpEqtNfZaq7YGnrcNUHvqWCNGE7DfD8dMAJ3SCixgNkcd606I+NIHkFquS1qhSrUEQ0cGLhsg8PmpgCaghx+OeQA6fvHXtw5DFjirfqwEiB/39uEJn7Sgz9zQZIH9dynRuvGk82nI0nrwXCzdeNaTD5+RuNtpJkh36QJhLRZbkm6wUO2+ON2gBZIGuPPu1NWHiLyPZTwLlASgvg2+dZ8ThmrAi/eVAdQtALgPAW6jshyAqlNWhfmOA0ESGSCDnw0C5DTAFTB/zM+NwkyNaD4O5MPP5W25tBnuhVlqMCBVxAGQyc/2AHIa4KpbBosHyFfYGBBb3mx7G28xQO0hj7aYaQX4WQ9AJhJwlIMB8vTDsXCALhgyrpFoojrxAEB6mTkAgvhZDUAQE0MNMJNOIYB76a4RICbOTXd5qbADrROvCuvMDOoEr0Kc3fi+/JCuwGrdDryHF5OpJBgg9g6jsr2Nrx+r/eYb+zrgCQAN0FosEGxVqoYbXc9Sg1fQMmZvPxzTW6BndwZt9/T0Xc4P/Da+KZf9duH4ZwDcDX2tANkwcbQBO53QAY5wPJM/StVZhZ37v4dnvTDp4w5sA9WF0vUL7Z0BaDNrawSLA0hlvwvHwBt1DZC59cOYWENU+9pAx74M1fyAb+M1A+STKiC0jJrXNKr9AMH1kveNug1osAs2ONYg1f5GNKvIPBhiTloPDo4sb+ODASr9VYcFTgOQu2Hje6NOBz7dXt6xe2HP7gS+ytBbQD6pMkJLf9WBgSsCyBxBrNYAELVAAb4VukOxxtYYXhkwpv/2RbAwgNwtY88btd9gtHf8HB9rYoAC6am6AHlu8N0J7c7e7I9gYQBFuVGrB9YcBsaauBcWSE8VBSBj9mZ/BBsESBvFSL6LXCSABo4DwRIOkHsBTy4ANaMYRf/s1YUAFEVaU/LG+JRa7gREQ6LNSDSlLoUFvbroSiKA+N/9Wrrn+56AvgU8mVggOQxmjhtu3wLxv6MB6l3AkwdAchhsqg0wtgNQ/wKeLAASw2D+700RoNR5TaOaCiDOD/zWCwFyZsBjBVgGAAl+pqRiKwCVUKBXBFsDqJjW+fOqASqtrjMzBai0b1odEGss1ZgAqUlAwNv4QQCJUE/Xq5sHqLRvWh0QazTViABdSCeb0KqMIQBJB/4IEJdyDr91swB0cuNjYYGcPhL7U1UK57W+7uc3CVDT4Wr42TxAdRXm9JHYL2zQnpqfVb3DkBLtVYZ6XVEUayqEWAA5fSQaAlmgIVvwrMcCDZh4UU8WizkQv0ILpMfqnaq6A5Y7QEURdyB+NQCFtIFKufuM/zaEWwWI9b3ijqOuBiCHj0R3ApThEb2wrAHiXffI46jrAMi1Z6ozAXW3fbPjQD4AtRYLloVL1TNNqVVjAgSKD0CtUeeBezitHSBwgSn/jD0Qv1WAAFLS5zWNqh9ArjXJmvQPFhwAAArcSURBVGr052eTAIH0wKrrBahwIGPphKZ4FbhqgKBiKdV2Re5k9QSuACApfo1oXkCF2AssRppSq04GUKWaO5UMKB2pyhkgPnujAl7l5A2QnBVeNlsNu1KVNUBs/oaxF/PoNKVWTQ8Qf00iX23xjd7LNb3fGSTud2F+63HqV4HrexeY1gKxpZUzNw3XYYGsr3JytUD8Ly+MuYtlBQCpNzkVVmHtWKUFqnIGqPdGkh69tRieptSq0wE0/3O1bIBk5W70VwPTlFp1ll5Yb6qyA6hpG+JAYkcs40DuVOUFkCcrmQO0gOdqkQBN9ypw6QA1kziYv81vjJ5Un051eQBN+ipw4QBpi3lOjrRwBMh29dSvAhcOUDOR9fmH97RwBKiCanB1tEwq5gComUpP6zK5c9ioZT0rFuurjPrtjTzY5tucMICaxTxs17DGCqEFqjqjGK2GzzLNyrwWiEvdDkKAqtY4aqfZvEwq5m0DcUGA9CA5h2U1o4Pz9MLUYh5WmT3/CLvxWlAzCQoBsooYB2JG6MuDgxt1XYYAVQt5Fbh0gCyCAFULeRWIAK0YoCW8CkSA1gzQAl4FIkDrBWjGNKVWRYAQoCDV9ADNPcA+oVhfZeQhaIFCVdECcUGAECAEaJZbIUBcECAECAGa5VYIEBcECAFCgGa5FQLEBQFCgJYEUE6C2Y8PkF6YM6vOfP+5sz9ZoSJACNAyAULJQxAglCBBgFCCBAFCCRIECCVI4gGkLTRUIp3AXN6Wy+f7VS80XzE9qsJHiI9mywONW/X5L7RFSk5VFqmerbmznyb//dmPBtBFJ+tV7QSGrYFmS+g9VPlCs29VPqoVK5gjL82WB5pe1Ytm96qe++uqc2c/Tf49sh8LILmNM8WT35KtfVYLoC/Y7bRMOFTZaR16p+rlX/2jV6RtDzROVX7sp8q+at4BZs5+mvz7ZD9yFXYilzzze2kuGLSy7lNtPYIO1ecf/rtZhUGajQeaXtXLH/4bVIVZktpK6dzZT5P//uzHBYjFTG9zwSvItxsnMGwtvZ/q5W3zB4RVv3zbbAOBmm0PNG7V20e8hLySalIxZ/bT5L8/+5EBYrCzImjD+vTdt31Vwd8FUqUfAEC2SNvtAFekvvev2o2l2bOfJv/92Y8MkKpDW9Ul49pTFc4rpCq6AW/HjfTpP4EAwbGemGZlxuynyX9/9uO3gZoHUzqBMQrQrtpY/V7VCuzG2yJtPNC4Iz2BqjBYtdM2nTv7afLfn/3IAFFzp1Xi8IiJQ7XtLMatCgIUGqnqcPio/r1fnBNlP03++7OPI9EoQYIAoQQJAoQSJAgQSpAgQChBggChBAkChBIkuQJ0ure3d+Vuj9Ljr3Y01FXGCSXP7hxaz/Xe48n3Bl25DMkUoNMX7lfV+d6hW6vLQv9Vp1ftcPXf4/zF3wy5dBGSJ0DUULCP454frMNC/1XMhgQAJG+wKskVoOvq8PGrtFq6Tn/Hn9KD6+zbIf3y3t4exYT9uM/u7O0xw9O6ip7g1zFlTaE61S66Sr8/ubV35aev3W9udFjV5+oba/dYoQnKEyBaD+0JGJ7cOuRV0+NX6W93yqjhX164/+wOr43YB+eidZU0G+ykrsAJkxex/5/cuk7/F3TxSx6/eijCD5sb6/cYaL2WIJkCJNrD9Hf7P/bLc4NyyH9f7Qv9/Ordc/bz8x9bv0r+0Kwy0xX4h7qI/uGfpxIgZYgkI/WN9Xs0N1qNZAtQxX4u9tSf856VMBDyD/+kPyb9POUV1d719lWCAo6GrlADJK2JsCqSnGNRc52r6q65cROFVrWuRXIGiP3GT25duVszAwFktkqUwjkfBtAVnADxFpG0SvxLfeMmCgRoJSLrEfVbn3ctEK/CXmPtk3NtsKi5ih3JM7pCDRALU1VYY3IUlPxQu3ETBVZha5Fj9puxxiv78R6/agCkNaJZO0f+wvVVCrKKfdUV7I1ozpF2TrtxEwU2olcjp6rhQpsmV/5VGoamCnuvbiuzLnZtIdRV9MQxb7dcaSucA93496QFOpej2OqcduM6CqDCXLrkCpBTRhsC6GXEgLEdHEjciIyvSU6v6t/OuYG6alM2ZIXjiAgQJOMB6tgQOWzkKfgyFSU/QYBQggQBQgkSBAglSBAglCBBgFCCBAFCCRIECCVIECCUIEGAUIIEAUIJEgQIJUgQIJQgQYBQggQBQgkSBAglSBAglCBBgFCCBAFCCRIECCVIECCUIEGAUIIEAUIJEgQIJUgQIJQgQYBQgiRrgMjcCYgsxQz3zBkgQjZG0IIB6jp2190P/O9/+Hsj2O12AxKXWKICtK8kXpSDJRVAZS3mOT+ADBftGjEDXFnshIB3EE6UVZyv3W9cC2rCA5mnpsYPiqHXBIBRNEIITJDwAM3dQtWpkvfUTnXFDhBzH8Uu/x9bctrplN/Ypgn2hMDiA9CY8lLgjAbIdNEeGyDmmos5hFNxWrLCwjnMTVIclLgBIlaA+GUsRXWq1D2bU4Y4LRC/zpocK0D2hMARjQLI41RZldpHWzwBut4cKudv0h+ccPDeuHUTIaCnrd3ORtCTt+6LDDx5619oVcncmf7ytfvPfvIejVt495Yu5KkGd6Iji/fxa7+iqr/iF0kvqNx6/cE/U23hFNViMBRAAEGiHHli1IF+T0tB+wD0ZzwPv//9F+6LpFYidfJEk3529DvfP3QkBMyRFSDmae+uV3n9kiXOuL7U/nWj9qvCahftykWk5hiXfWrOI4XL91YqdhbR4ufPFC0Y5oT5/IX7ogoT0V1VJScCay9eokBYkLyIubg851bi8avqiBV922sYsUijUT/4dapa9+xaoH2L6DcVAMlUMe+cNKk0WTJ18kSTfn60d+hISCOFTRoVBtw5r0F7y0u5DhVSWkRT8QSo9rWlHNeaANVnpJd3IA67BToXv/HxIfNz++wndyVAP7nLvijPXbS4GUAKBx0gfhH7OaiqbD/JUzanXxA5KmLe9GCOU1WqmnuqU6b4VWEycSxZNLkydfJEk352VLeBwITAYrFAjd3sLS+4FmPEcGxGWyChe0s+C4KaLkDNGWu7yNoGEgXDLBAr0uNDAKBjZu25uTUtUH1RA5/6qeiPAG4L5gKIXiY8PatUte55Dm8zNgygW6JXK1KnnWjSLzNkSQgsNoD4VV7lZQUosAqTCWlhEhOgViUPWSD20Og2pW4D3W8ugp4oFjnkuhKuvGQ22WWsEra0gU4H9sIqAKC36ihkpQVaIHtCoJv4WyB7edkAsjai/QDSHLtLH+wAQM0ZR8/MNg7Ei4m75b6qauMWQDyLX+UfqiPCiuH0BWGSr7bbEE2BsOChvk95IYoddGSq2r0w2O/qIIBkUmXq5Am4DQQnBE65BSAGHG0++5SXBSB7N97TAtUu2rVGNMvQsztXhHHQGtFOgKxyLgY6nnzvz4XzbtELa6ow2gZT/RI1FEKD/vAt2tJ+4VfiIq1XIQqEnrp/bO2FWUU+sLRhqVLVjAM1p4ZHqQPE+0WyXlYntF4Y7dE2vTAoIbA4e2Fe5fVLC0DWgUTfKqx27N5043mL8i/ob3u8d7XVjQ9x2T/K0+0q3eOmEM+R6JjlNagNNIEgQCGCAKGsThAglCBBgFCCBAFCCRIECCVIECCUIEGAUIIEAUIJEgQIJUgQIJQgQYBQggQBQgkSBAglSBAglCBBgFCCBAFCCRIECCVIECCUIEGAUILk/wFGEJaXLZnH3AAAAABJRU5ErkJggg==" style="display: block; margin: auto;" /></p>
<table>
<thead>
<tr class="header">
<th align="right">n</th>
<th align="left">task</th>
<th align="right">OptimalCutpoints</th>
<th align="right">ROCR</th>
<th align="right">ThresholdROC</th>
<th align="right">cutpointr</th>
<th align="right">pROC</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td align="right">1e+02</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">2.288702</td>
<td align="right">1.812802</td>
<td align="right">1.194301</td>
<td align="right">4.5018015</td>
<td align="right">0.662101</td>
</tr>
<tr class="even">
<td align="right">1e+03</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">45.056801</td>
<td align="right">2.176401</td>
<td align="right">36.239852</td>
<td align="right">4.8394010</td>
<td align="right">0.981001</td>
</tr>
<tr class="odd">
<td align="right">1e+04</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">2538.612001</td>
<td align="right">5.667101</td>
<td align="right">2503.801251</td>
<td align="right">8.5662515</td>
<td align="right">4.031701</td>
</tr>
<tr class="even">
<td align="right">1e+05</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">NA</td>
<td align="right">43.118751</td>
<td align="right">NA</td>
<td align="right">45.3845010</td>
<td align="right">37.150151</td>
</tr>
<tr class="odd">
<td align="right">1e+06</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">NA</td>
<td align="right">607.023851</td>
<td align="right">NA</td>
<td align="right">465.0032010</td>
<td align="right">583.095000</td>
</tr>
<tr class="even">
<td align="right">1e+07</td>
<td align="left">Cutpoint Estimation</td>
<td align="right">NA</td>
<td align="right">7850.258700</td>
<td align="right">NA</td>
<td align="right">5467.3328010</td>
<td align="right">7339.356101</td>
</tr>
<tr class="odd">
<td align="right">1e+02</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">1.732651</td>
<td align="right">NA</td>
<td align="right">0.7973505</td>
<td align="right">0.447701</td>
</tr>
<tr class="even">
<td align="right">1e+03</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">2.035852</td>
<td align="right">NA</td>
<td align="right">0.8593010</td>
<td align="right">0.694802</td>
</tr>
<tr class="odd">
<td align="right">1e+04</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">5.662151</td>
<td align="right">NA</td>
<td align="right">1.8781510</td>
<td align="right">3.658050</td>
</tr>
<tr class="even">
<td align="right">1e+05</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">42.820852</td>
<td align="right">NA</td>
<td align="right">11.0992510</td>
<td align="right">35.329301</td>
</tr>
<tr class="odd">
<td align="right">1e+06</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">612.471901</td>
<td align="right">NA</td>
<td align="right">159.8100505</td>
<td align="right">610.433700</td>
</tr>
<tr class="even">
<td align="right">1e+07</td>
<td align="left">ROC curve calculation</td>
<td align="right">NA</td>
<td align="right">7806.385452</td>
<td align="right">NA</td>
<td align="right">2032.6935510</td>
<td align="right">7081.897251</td>
</tr>
</tbody>
</table>
</div>
<!-- code folding -->
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
var script = document.createElement("script");
script.type = "text/javascript";
script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
document.getElementsByTagName("head")[0].appendChild(script);
})();
</script>
</body>
</html>
|