{ "ensemble": [ "gliner:urchade/gliner_multi-v2.1", "gliner:urchade/gliner_multi_pii-v1", "hf:Davlan/xlm-roberta-large-ner-hrl", "hf:StanfordAIMI/stanford-deidentifier-base", "hf:obi/deid_roberta_i2b2", "llm:GaleneAI/Magistral-Small-2509-FP8-Dynamic", "llm:Microsoft/Phi-4-mini-instruct", "llm:RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8", "llm:RedHatAI/gemma-4-31B-it-FP8-block", "llm:google/gemma-4-E4B-it", "llm:ibm-granite/granite-4.1-3b", "presidio", "privacy_tagger" ], "operating_points": { "cardiode": { "MAX-SENSITIVITY": { "rule": "union", "sensitivity": 0.9919319904700021, "specificity": 0.9670102229583519, "precision": 0.8191532850903301, "information_weighted_precision": 0.790061971443917, "cost_parallel_s": 57.63300814807415, "cost_serial_s": 78.22202599406242, "token_recall_all_gold": 0.9462804782064772, "ensemble": [ "gliner:urchade/gliner_multi_pii-v1", "llm:RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8", "llm:RedHatAI/gemma-4-31B-it-FP8-block" ], "leakage": { "gold_tokens": 120701, "tokens": 6484, "token_rate": 0.05371952179352284, "tokens_per_document": 16.21, "gold_entities": 30413, "entities": 2973, "entity_rate": 0.09775424982737645, "entities_per_document": 7.4325 } }, "FAST-SENSITIVITY": { "rule": "union", "sensitivity": 0.8947368421052632, "specificity": 0.9768943348535634, "precision": 0.8657519668579682, "information_weighted_precision": 0.8268531037688619, "cost_parallel_s": 0.30126263737678527, "cost_serial_s": 0.5644275319576264, "token_recall_all_gold": 0.9436044440394031, "ensemble": [ "gliner:urchade/gliner_multi_pii-v1", "hf:StanfordAIMI/stanford-deidentifier-base" ], "leakage": { "gold_tokens": 120701, "tokens": 6807, "token_rate": 0.056395555960596845, "tokens_per_document": 17.0175, "gold_entities": 30413, "entities": 4370, "entity_rate": 0.1436885542366751, "entities_per_document": 10.925 } }, "MAX-SPECIFICITY": { "rule": "vote", "sensitivity": 0.523012778860732, "specificity": 0.9997370342169507, "precision": 0.9796146044624746, "information_weighted_precision": 0.9815937365128613, "cost_parallel_s": 116.87502154827118, "cost_serial_s": 117.59259123206138, "token_recall_all_gold": 0.08002419201166519, "ensemble": [ "gliner:urchade/gliner_multi-v2.1", "hf:Davlan/xlm-roberta-large-ner-hrl", "llm:gpt-oss-120b" ], "leakage": { "gold_tokens": 120701, "tokens": 111042, "token_rate": 0.9199758079883348, "tokens_per_document": 277.605, "gold_entities": 30413, "entities": 26420, "entity_rate": 0.8687074606253905, "entities_per_document": 66.05 } }, "FAST-SPECIFICITY": { "rule": "single", "sensitivity": 0.7807017543859649, "specificity": 0.9862184997082519, "precision": 0.9122729581851645, "information_weighted_precision": 0.8807485883185682, "cost_parallel_s": 0.2631648945808411, "cost_serial_s": 0.2631648945808411, "token_recall_all_gold": 0.9075566896711709, "ensemble": [ "hf:StanfordAIMI/stanford-deidentifier-base" ], "leakage": { "gold_tokens": 120701, "tokens": 11158, "token_rate": 0.09244331032882909, "tokens_per_document": 27.895, "gold_entities": 30413, "entities": 6791, "entity_rate": 0.22329267089731364, "entities_per_document": 16.9775 } } }, "enron": { "MAX-SENSITIVITY": { "rule": "union", "sensitivity": 0.996766948372044, "specificity": 0.7745200983265216, "precision": 0.5762484831024987, "information_weighted_precision": 0.5081117118490986, "cost_parallel_s": 25.557465940894822, "cost_serial_s": 25.662220647678392, "ensemble": [ "hf:StanfordAIMI/stanford-deidentifier-base", "llm:gpt-oss-120b", "presidio" ], "gold": "corrected 2026-09-25", "sweep_sources": 1743, "selection_rerun": false, "superseded": { "sensitivity": 0.9725563572680621, "specificity": 0.7719922491751702, "precision": 0.5789658327048046, "information_weighted_precision": 0.5143704456400028, "token_recall_all_gold": 0.9725563572680621, "leakage": { "gold_tokens": 3780657, "tokens": 103755, "token_rate": 0.027443642731937862, "tokens_per_document": 1.7694760897742001, "gold_entities": 420890, "entities": 60368, "entity_rate": 0.14342939960559767, "entities_per_document": 1.0295381676785593 } }, "weight_model": "unigram:enron" }, "FAST-SENSITIVITY": { "rule": "union", "sensitivity": 0.9222118732895938, "specificity": 0.7420067963110943, "precision": 0.5237190619999847, "information_weighted_precision": 0.49883949822438156, "cost_parallel_s": 0.05693825925365868, "cost_serial_s": 0.13374275004831618, "ensemble": [ "hf:Davlan/xlm-roberta-large-ner-hrl", "hf:StanfordAIMI/stanford-deidentifier-base", "hf:obi/deid_roberta_i2b2" ], "gold": "corrected 2026-09-25", "sweep_sources": 1743, "selection_rerun": false, "superseded": { "sensitivity": 0.8987017335875749, "specificity": 0.7419265936448285, "precision": 0.5288898394643113, "information_weighted_precision": 0.5068390441678622, "token_recall_all_gold": 0.8987017335875749, "leakage": { "gold_tokens": 3780657, "tokens": 382974, "token_rate": 0.10129826641242515, "tokens_per_document": 6.5313800395661366, "gold_entities": 420890, "entities": 126653, "entity_rate": 0.3009171042315094, "entities_per_document": 2.1599870386793096 } }, "weight_model": "unigram:enron" }, "MAX-SPECIFICITY": { "rule": "intersection", "sensitivity": 0.5292397244053049, "specificity": 0.9447789816056418, "precision": 0.7467216871220809, "information_weighted_precision": 0.6718197107212109, "cost_parallel_s": 20.931386979400106, "cost_serial_s": 35.78167065115025, "ensemble": [ "llm:GaleneAI/Magistral-Small-2509-FP8-Dynamic", "llm:RedHatAI/gemma-4-31B-it-FP8-block", "llm:google/gemma-4-E4B-it" ], "gold": "corrected 2026-09-25", "sweep_sources": 1743, "selection_rerun": false, "superseded": { "sensitivity": 0.5881448647682136, "specificity": 0.9428060312086327, "precision": 0.7682588865091181, "information_weighted_precision": 0.7026540107074426, "token_recall_all_gold": 0.5881448647682136, "leakage": { "gold_tokens": 3780657, "tokens": 1557083, "token_rate": 0.4118551352317864, "tokens_per_document": 26.555068558564706, "gold_entities": 420890, "entities": 276468, "entity_rate": 0.6568652141889805, "entities_per_document": 4.714987379766696 } }, "weight_model": "unigram:enron" }, "FAST-SPECIFICITY": { "rule": "single", "sensitivity": 0.6919892682969616, "specificity": 0.878058907040323, "precision": 0.6357900993141068, "information_weighted_precision": 0.5847487840386945, "cost_parallel_s": 0.025934466386818854, "cost_serial_s": 0.025934466386818854, "ensemble": [ "hf:StanfordAIMI/stanford-deidentifier-base" ], "gold": "corrected 2026-09-25", "sweep_sources": 1743, "selection_rerun": false, "superseded": { "sensitivity": 0.6740886570773281, "specificity": 0.8787239278577074, "precision": 0.6418202171638249, "information_weighted_precision": 0.5927287236731719, "token_recall_all_gold": 0.6740886570773281, "leakage": { "gold_tokens": 3780657, "tokens": 1232159, "token_rate": 0.32591134292267193, "tokens_per_document": 21.013694658571527, "gold_entities": 420890, "entities": 383121, "entity_rate": 0.9102639644562712, "entities_per_document": 6.533887031857562 } }, "weight_model": "unigram:enron" } }, "ontonotes": { "MAX-SENSITIVITY": { "rule": "union", "sensitivity": 0.7465669254887107, "specificity": 0.9763428438776106, "precision": 0.822435182171075, "information_weighted_precision": 0.8449429836185717, "cost_parallel_s": 9.449679761041114, "cost_serial_s": 9.651369657955412, "token_recall_all_gold": 0.5925333456245687, "ensemble": [ "hf:Davlan/xlm-roberta-large-ner-hrl", "llm:RedHatAI/gemma-4-31B-it-FP8-block", "presidio" ], "leakage": { "gold_tokens": 737653, "tokens": 300569, "token_rate": 0.4074666543754313, "tokens_per_document": 50.14497831164498, "gold_entities": 239062, "entities": 87216, "entity_rate": 0.36482586107369636, "entities_per_document": 14.55055055055055 } }, "FAST-SENSITIVITY": { "rule": "union", "sensitivity": 0.6842981293268523, "specificity": 0.9731023848184267, "precision": 0.7998003470602504, "information_weighted_precision": 0.8264246968322811, "cost_parallel_s": 5.375846266308983, "cost_serial_s": 5.577536163223279, "token_recall_all_gold": 0.5810821619379302, "ensemble": [ "hf:Davlan/xlm-roberta-large-ner-hrl", "llm:RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8", "presidio" ], "leakage": { "gold_tokens": 737653, "tokens": 309016, "token_rate": 0.41891783806206984, "tokens_per_document": 51.55422088755422, "gold_entities": 239062, "entities": 82759, "entity_rate": 0.34618216194961976, "entities_per_document": 13.806973640306973 } }, "MAX-SPECIFICITY": { "rule": "union", "sensitivity": 0.5034467520989836, "specificity": 0.9959761249915704, "precision": 0.9417299852246613, "information_weighted_precision": 0.9462477670371567, "cost_parallel_s": 0.1030080695877403, "cost_serial_s": 0.20198920627693595, "token_recall_all_gold": 0.3516667050767773, "ensemble": [ "hf:Davlan/xlm-roberta-large-ner-hrl", "hf:obi/deid_roberta_i2b2" ], "leakage": { "gold_tokens": 737653, "tokens": 478245, "token_rate": 0.6483332949232227, "tokens_per_document": 79.78728728728728, "gold_entities": 239062, "entities": 139814, "entity_rate": 0.5848440990203378, "entities_per_document": 23.32565899232566 } }, "FAST-SPECIFICITY": { "rule": "union", "sensitivity": 0.5034467520989836, "specificity": 0.9959761249915704, "precision": 0.9417299852246613, "information_weighted_precision": 0.9462477670371567, "cost_parallel_s": 0.1030080695877403, "cost_serial_s": 0.20198920627693595, "token_recall_all_gold": 0.3516667050767773, "ensemble": [ "hf:Davlan/xlm-roberta-large-ner-hrl", "hf:obi/deid_roberta_i2b2" ], "leakage": { "gold_tokens": 737653, "tokens": 478245, "token_rate": 0.6483332949232227, "tokens_per_document": 79.78728728728728, "gold_entities": 239062, "entities": 139814, "entity_rate": 0.5848440990203378, "entities_per_document": 23.32565899232566 } } }, "tab": { "MAX-SENSITIVITY": { "rule": "union", "sensitivity": 0.925341796875, "specificity": 0.8985822229838305, "precision": 0.630115986736287, "information_weighted_precision": 0.6971925486574702, "cost_parallel_s": 14.210042568035306, "cost_serial_s": 14.639629975666383, "token_recall_all_gold": 0.8954616598909404, "ensemble": [ "gliner:urchade/gliner_multi_pii-v1", "llm:RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-FP8", "presidio" ], "leakage": { "gold_tokens": 298369, "tokens": 31191, "token_rate": 0.10453834010905959, "tokens_per_document": 24.59858044164038, "gold_entities": 67798, "entities": 7656, "entity_rate": 0.11292368506445617, "entities_per_document": 6.037854889589905 } }, "FAST-SENSITIVITY": { "rule": "union", "sensitivity": 0.892645263671875, "specificity": 0.9242043797508463, "precision": 0.6759305595107425, "information_weighted_precision": 0.7351196880747097, "cost_parallel_s": 0.21866347135429504, "cost_serial_s": 0.49627622726963894, "token_recall_all_gold": 0.819381370048497, "ensemble": [ "gliner:urchade/gliner_multi_pii-v1", "hf:Davlan/xlm-roberta-large-ner-hrl", "hf:StanfordAIMI/stanford-deidentifier-base" ], "leakage": { "gold_tokens": 298369, "tokens": 53891, "token_rate": 0.18061862995150302, "tokens_per_document": 42.50078864353312, "gold_entities": 67798, "entities": 15022, "entity_rate": 0.22156995781586478, "entities_per_document": 11.847003154574132 } }, "MAX-SPECIFICITY": { "rule": "vote", "sensitivity": 0.501361083984375, "specificity": 0.9959487466333858, "precision": 0.9646912954039507, "information_weighted_precision": 0.9676715015244834, "cost_parallel_s": 98.11002685877051, "cost_serial_s": 98.3865751718121, "token_recall_all_gold": 0.5736856040674467, "ensemble": [ "gliner:urchade/gliner_multi-v2.1", "hf:StanfordAIMI/stanford-deidentifier-base", "llm:Qwen/Qwen3.6-35B-A3B-FP8" ], "leakage": { "gold_tokens": 298369, "tokens": 127199, "token_rate": 0.4263143959325533, "tokens_per_document": 100.31466876971609, "gold_entities": 67798, "entities": 26102, "entity_rate": 0.38499660756954485, "entities_per_document": 20.585173501577287 } }, "FAST-SPECIFICITY": { "rule": "single", "sensitivity": 0.519781494140625, "specificity": 0.9933770252225279, "precision": 0.9454298425553453, "information_weighted_precision": 0.9393410046159307, "cost_parallel_s": 0.1171352803143041, "cost_serial_s": 0.1171352803143041, "token_recall_all_gold": 0.5947099061899862, "ensemble": [ "hf:StanfordAIMI/stanford-deidentifier-base" ], "leakage": { "gold_tokens": 298369, "tokens": 120926, "token_rate": 0.4052900938100138, "tokens_per_document": 95.36750788643533, "gold_entities": 67798, "entities": 26196, "entity_rate": 0.3863830791468775, "entities_per_document": 20.65930599369085 } } } }, "recommended": { "cardiode": { "union": { "token_recall": 0.9988401090297512, "entity_recall": 0.9954295860322888, "precision": 0.5455002036106964, "information_weighted_precision": 0.5455002036106964, "replacements": 68869, "documents_missing_a_detector": 327, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 7, "a2_bound_observed": 25677, "a2_bound_chance": 0.00024826216484607745, "a2_bound_prior_size": 4028, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 7, "a2_real_observed": 25677, "a2_real_chance": 1.823819077147547e-05, "a2_real_prior": "German surnames", "a2_real_prior_size": 54830, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.00027261751762277527, "a2_real_mean_candidate_set": 54830.0, "a2_rho": 0.8975424343492916, "a3_ceiling_rank1": 0.0273224043715847, "a3_gallery": 207, "a3_queries": 138, "a3f_rank1": 0.0, "a3f_rank1_sd": 0.0, "a3f_rank5": 0.007142857142857143, "a3f_rank5_sd": 0.015971914124998498, "a3f_map": 0.032576295971073116, "a3f_map_sd": 0.005907450810364029, "a5_rank1": 0.007142857142857143, "a5_rank1_sd": 0.015971914124998498, "a5_rank5": 0.050793650793650794, "a5_rank5_sd": 0.07045187625768935, "a5_map": 0.059189112549952926, "a5_map_sd": 0.02564804725432865, "a5_folds": 5, "relational_note": null }, "_bands": { "1": 0.00029432540616906053, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "_abstention": [ { "prior": "corpus-internal", "phi": 0.0, "attempted": 25677, "abstained": 0, "accuracy_attempted": 0.0003115628772831717, "c_at_1": 0.0003115628772831717, "lift_over_chance": 0.15691468640436307 }, { "prior": "corpus-internal", "phi": 0.05, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 0.25, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 1.5, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German surnames", "phi": 0.0, "attempted": 25677, "abstained": 0, "accuracy_attempted": 0.00027261751762277527, "c_at_1": 0.00027261751762277527, "lift_over_chance": 0.0 }, { "prior": "German surnames", "phi": 0.05, "attempted": 11, "abstained": 25666, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German surnames", "phi": 0.25, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German surnames", "phi": 1.5, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (male)", "phi": 0.0, "attempted": 25677, "abstained": 0, "accuracy_attempted": 0.00027261751762277527, "c_at_1": 0.00027261751762277527, "lift_over_chance": 0.0 }, { "prior": "German given names (male)", "phi": 0.05, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (male)", "phi": 0.25, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (male)", "phi": 1.5, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (female)", "phi": 0.0, "attempted": 25677, "abstained": 0, "accuracy_attempted": 0.00027261751762277527, "c_at_1": 0.00027261751762277527, "lift_over_chance": 0.0 }, { "prior": "German given names (female)", "phi": 0.05, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (female)", "phi": 0.25, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "German given names (female)", "phi": 1.5, "attempted": 0, "abstained": 25677, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 } ], "_prior_sweep": [ { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German surnames top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "2010", "prior_provenance": "abydos rank order x US Census 2010 counts \u2014 a US Zipf shape on German ranks, not German counts; truncated to the 100 commonest" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German surnames top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "2010", "prior_provenance": "abydos rank order x US Census 2010 counts \u2014 a US Zipf shape on German ranks, not German counts; truncated to the 1,000 commonest" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German surnames top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "2010", "prior_provenance": "abydos rank order x US Census 2010 counts \u2014 a US Zipf shape on German ranks, not German counts; truncated to the 10,000 commonest" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 1.823819077147547e-05, "lift_over_chance": 0.0, "mean_candidate_set": 54830.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German surnames", "prior_kind": "external", "prior_size": 54830, "prior_vintage": "2010", "prior_provenance": "abydos rank order x US Census 2010 counts \u2014 a US Zipf shape on German ranks, not German counts" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German given names (male) top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "1930-1969", "prior_provenance": "Stadt Bielefeld Einwohnermelderegister, CC BY 4.0; truncated to the 100 commonest" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.0018248175182481751, "lift_over_chance": 0.0, "mean_candidate_set": 548.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German given names (male)", "prior_kind": "external", "prior_size": 548, "prior_vintage": "1930-1969", "prior_provenance": "Stadt Bielefeld Einwohnermelderegister, CC BY 4.0" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German given names (female) top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "1930-1969", "prior_provenance": "Stadt Bielefeld Einwohnermelderegister, CC BY 4.0; truncated to the 100 commonest" }, { "observed": 25677, "attempted": 25677, "abstained": 0, "correct_1_to_1": 7, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 25670, "correct": 7, "expected_correct": 7.0, "accuracy_attempted": 0.00027261751762277527, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.00027261751762277527, "chance": 0.0016750418760469012, "lift_over_chance": 0.0, "mean_candidate_set": 597.0, "eccentricity": 0.0, "undetected_recovered": 7, "by_band": { "1": 0.0002354603249352484, "2-4": 0.0001971608832807571, "20+": 0.0016246953696181965, "5-19": 0.0 }, "prior": "German given names (female)", "prior_kind": "external", "prior_size": 597, "prior_vintage": "1930-1969", "prior_provenance": "Stadt Bielefeld Einwohnermelderegister, CC BY 4.0" } ], "intersection": { "token_recall": 0.0001822685810390966, "entity_recall": 9.864202808009732e-05, "precision": 1.0, "information_weighted_precision": 1.0, "replacements": 22, "documents_missing_a_detector": 327, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 7, "a2_bound_observed": 20424, "a2_bound_chance": 0.05263157894736842, "a2_bound_prior_size": 19, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 7, "a2_real_observed": 20424, "a2_real_chance": 1.823819077147547e-05, "a2_real_prior": "German surnames", "a2_real_prior_size": 54830, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.0003427340383862123, "a2_real_mean_candidate_set": 54830.0, "a2_rho": null, "a3_ceiling_rank1": 0.0273224043715847, "a3_gallery": 207, "a3_queries": 6, "a3f_rank1": 0.0, "a3f_rank1_sd": 0.0, "a3f_rank5": 0.1, "a3f_rank5_sd": 0.22360679774997896, "a3f_map": 0.05018690517142529, "a3f_map_sd": 0.05665323788424045, "a5_rank1": 0.0, "a5_rank1_sd": 0.0, "a5_rank5": 0.1, "a5_rank5_sd": 0.22360679774997896, "a5_map": 0.03859649122807017, "a5_map_sd": 0.08630437807893925, "a5_folds": 5, "relational_note": null }, "vote2": { "token_recall": 0.9079543665752562, "entity_recall": 0.839147732877388, "precision": 0.908857946110913, "information_weighted_precision": 0.908857946110913, "replacements": 30276, "documents_missing_a_detector": 327, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 75, "a2_bound_observed": 20731, "a2_bound_chance": 0.0009250693802035153, "a2_bound_prior_size": 1081, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 75, "a2_real_observed": 20731, "a2_real_chance": 1.823819077147547e-05, "a2_real_prior": "German surnames", "a2_real_prior_size": 54830, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.0036177704886402005, "a2_real_mean_candidate_set": 54830.0, "a2_rho": 0.8012841589763514, "a3_ceiling_rank1": 0.0273224043715847, "a3_gallery": 207, "a3_queries": 73, "a3f_rank1": 0.04, "a3f_rank1_sd": 0.05962847939999439, "a3f_rank5": 0.13428571428571429, "a3f_rank5_sd": 0.1241022181277099, "a3f_map": 0.11042083824349158, "a3f_map_sd": 0.0727327694122356, "a5_rank1": 0.04, "a5_rank1_sd": 0.05962847939999439, "a5_rank5": 0.1476190476190476, "a5_rank5_sd": 0.14388518980702977, "a5_map": 0.10978772010159765, "a5_map_sd": 0.07050734315577985, "a5_folds": 5, "relational_note": null }, "vote3": { "token_recall": 0.9009204563342474, "entity_recall": 0.8248775194818005, "precision": 0.9743295671418459, "information_weighted_precision": 0.9743295671418459, "replacements": 26226, "documents_missing_a_detector": 327, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 79, "a2_bound_observed": 20226, "a2_bound_chance": 0.0011890606420927466, "a2_bound_prior_size": 841, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 79, "a2_real_observed": 20226, "a2_real_chance": 1.823819077147547e-05, "a2_real_prior": "German surnames", "a2_real_prior_size": 54830, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.0039058637397409277, "a2_real_mean_candidate_set": 54830.0, "a2_rho": 0.745292407873824, "a3_ceiling_rank1": 0.0273224043715847, "a3_gallery": 207, "a3_queries": 75, "a3f_rank1": 0.026666666666666665, "a3f_rank1_sd": 0.03651483716701107, "a3f_rank5": 0.16, "a3f_rank5_sd": 0.0760116950066092, "a3f_map": 0.11176184112905503, "a3f_map_sd": 0.0364463921339691, "a5_rank1": 0.04, "a5_rank1_sd": 0.03651483716701107, "a5_rank5": 0.16, "a5_rank5_sd": 0.0760116950066092, "a5_map": 0.11916990950425148, "a5_map_sd": 0.03266400365344309, "a5_folds": 5, "relational_note": null } }, "tab": { "union": { "token_recall": 0.9251262698202561, "entity_recall": 0.9262515118440072, "precision": 0.5440377948286362, "information_weighted_precision": 0.5440377948286362, "replacements": 162285, "documents_missing_a_detector": 736, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 95, "a2_bound_observed": 20147, "a2_bound_chance": 0.00010319917440660474, "a2_bound_prior_size": 9690, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 95, "a2_real_observed": 20147, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.004715342234575867, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.8487917692165491, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (4160 and 4541 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "_bands": { "1": 0.0022983392645314353, "2-4": 0.0071130733382389014, "20+": 0.015625, "5-19": 0.013230429988974642 }, "_abstention": [ { "prior": "corpus-internal", "phi": 0.0, "attempted": 20147, "abstained": 0, "accuracy_attempted": 0.004764977415992456, "c_at_1": 0.004764977415992456, "lift_over_chance": 0.4832435667265111 }, { "prior": "corpus-internal", "phi": 0.05, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 0.25, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 1.5, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.0, "attempted": 20147, "abstained": 0, "accuracy_attempted": 0.004715342234575867, "c_at_1": 0.004715342234575867, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.05, "attempted": 34, "abstained": 20113, "accuracy_attempted": 0.029411764705882353, "c_at_1": 9.91865986912114e-05, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.25, "attempted": 27, "abstained": 20120, "accuracy_attempted": 0.037037037037037035, "c_at_1": 9.92038442498512e-05, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 1.5, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.0, "attempted": 20147, "abstained": 0, "accuracy_attempted": 0.004715342234575867, "c_at_1": 0.004715342234575867, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.05, "attempted": 7, "abstained": 20140, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.25, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 1.5, "attempted": 0, "abstained": 20147, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 } ], "_prior_sweep": [ { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "US Census 2010 surnames top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 100 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "US Census 2010 surnames top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 1,000 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "US Census 2010 surnames top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 10,000 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 6.16370808678501e-06, "lift_over_chance": 0.0, "mean_candidate_set": 162240.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "US Census 2010 surnames", "prior_kind": "external", "prior_size": 162240, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "UCI given names top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 100 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "UCI given names top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 1,000 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "UCI given names top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 10,000 commonest" }, { "observed": 20147, "attempted": 20147, "abstained": 0, "correct_1_to_1": 95, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 20052, "correct": 95, "expected_correct": 95.0, "accuracy_attempted": 0.004715342234575867, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.004715342234575867, "chance": 7.480942299492044e-06, "lift_over_chance": 0.0, "mean_candidate_set": 133673.0, "eccentricity": 0.0, "undetected_recovered": 95, "by_band": { "1": 0.0022983392645314353, "2-4": 0.006867794947265146, "20+": 0.015625, "5-19": 0.013230429988974642 }, "prior": "UCI given names", "prior_kind": "external", "prior_size": 133673, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X" } ], "intersection": { "token_recall": 0.0032409533161957176, "entity_recall": 1.474969763119856e-05, "precision": 0.9928131416837782, "information_weighted_precision": 0.9928131416837782, "replacements": 729, "documents_missing_a_detector": 736, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 71, "a2_bound_observed": 16832, "a2_bound_chance": 0.00273224043715847, "a2_bound_prior_size": 366, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 71, "a2_real_observed": 16832, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.004218155893536121, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.5767507904730114, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (4160 and 4541 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "vote2": { "token_recall": 0.8101344308557524, "entity_recall": 0.7716304315761527, "precision": 0.7027287104764341, "information_weighted_precision": 0.7027287104764341, "replacements": 130663, "documents_missing_a_detector": 736, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 204, "a2_bound_observed": 19456, "a2_bound_chance": 0.0001336005344021376, "a2_bound_prior_size": 7485, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 204, "a2_real_observed": 19456, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.010485197368421052, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.9464286226228886, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (4160 and 4541 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "vote3": { "token_recall": 0.7380089754632686, "entity_recall": 0.7186790170801498, "precision": 0.8785013544621448, "information_weighted_precision": 0.8785013544621448, "replacements": 85093, "documents_missing_a_detector": 736, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 176, "a2_bound_observed": 17154, "a2_bound_chance": 0.00016672224074691563, "a2_bound_prior_size": 5998, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 176, "a2_real_observed": 17154, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.010259997668182348, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.946657772280946, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (4160 and 4541 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" } }, "ontonotes": { "union": { "token_recall": 0.7222555862987069, "entity_recall": 0.761066166935774, "precision": 0.6619469994123194, "information_weighted_precision": 0.6619469994123194, "replacements": 291493, "documents_missing_a_detector": 3489, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 166, "a2_bound_observed": 28153, "a2_bound_chance": 3.46092614383609e-05, "a2_bound_prior_size": 28894, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 166, "a2_real_observed": 28153, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.005896352076155294, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.788787387861662, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (6661 and 6569 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "_bands": { "1": 0.00389321468298109, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "_abstention": [ { "prior": "corpus-internal", "phi": 0.0, "attempted": 28153, "abstained": 0, "accuracy_attempted": 0.005931872269385146, "c_at_1": 0.005931872269385146, "lift_over_chance": 1.032407903669561 }, { "prior": "corpus-internal", "phi": 0.05, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 0.25, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 1.5, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.0, "attempted": 28153, "abstained": 0, "accuracy_attempted": 0.005896352076155294, "c_at_1": 0.005896352076155294, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.05, "attempted": 17, "abstained": 28136, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.25, "attempted": 12, "abstained": 28141, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 1.5, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.0, "attempted": 28153, "abstained": 0, "accuracy_attempted": 0.005896352076155294, "c_at_1": 0.005896352076155294, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.05, "attempted": 7, "abstained": 28146, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.25, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 1.5, "attempted": 0, "abstained": 28153, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 } ], "_prior_sweep": [ { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "US Census 2010 surnames top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 100 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 167, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27986, "correct": 167, "expected_correct": 167.0, "accuracy_attempted": 0.005931872269385146, "correct_by_alignment": 1, "accuracy_alignment": 3.5730875049129956e-05, "c_at_1": 0.005931872269385146, "chance": 0.001, "lift_over_chance": 0.03573087504912995, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008402029169308814, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "US Census 2010 surnames top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 1,000 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "US Census 2010 surnames top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 10,000 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 6.16370808678501e-06, "lift_over_chance": 0.0, "mean_candidate_set": 162240.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "US Census 2010 surnames", "prior_kind": "external", "prior_size": 162240, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "UCI given names top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 100 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "UCI given names top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 1,000 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "UCI given names top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 10,000 commonest" }, { "observed": 28153, "attempted": 28153, "abstained": 0, "correct_1_to_1": 166, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 27987, "correct": 166, "expected_correct": 166.0, "accuracy_attempted": 0.005896352076155294, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.005896352076155294, "chance": 7.480942299492044e-06, "lift_over_chance": 0.0, "mean_candidate_set": 133673.0, "eccentricity": 0.0, "undetected_recovered": 166, "by_band": { "1": 0.0038375973303670747, "2-4": 0.008243500317057704, "20+": 0.010804321728691477, "5-19": 0.011873350923482849 }, "prior": "UCI given names", "prior_kind": "external", "prior_size": 133673, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X" } ], "intersection": { "token_recall": 0.002025342539107141, "entity_recall": 0.0003220921769248145, "precision": 0.9907161803713528, "information_weighted_precision": 0.9907161803713528, "replacements": 1259, "documents_missing_a_detector": 3489, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 127, "a2_bound_observed": 34976, "a2_bound_chance": 0.0017391304347826088, "a2_bound_prior_size": 575, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 127, "a2_real_observed": 34976, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.003631061299176578, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.3896159456640396, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (6661 and 6569 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "vote2": { "token_recall": 0.49598117271942227, "entity_recall": 0.5261480285448963, "precision": 0.8367724081147222, "information_weighted_precision": 0.8367724081147222, "replacements": 178224, "documents_missing_a_detector": 3489, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 284, "a2_bound_observed": 32950, "a2_bound_chance": 4.6724605177086255e-05, "a2_bound_prior_size": 21402, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 284, "a2_real_observed": 32950, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.008619119878603945, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.9200500268904476, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (6661 and 6569 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" }, "vote3": { "token_recall": 0.36945691266760927, "entity_recall": 0.4121524960052204, "precision": 0.9026267590939652, "information_weighted_precision": 0.9026267590939652, "replacements": 129934, "documents_missing_a_detector": 3489, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 322, "a2_bound_observed": 31760, "a2_bound_chance": 5.299136240792751e-05, "a2_bound_prior_size": 18871, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 322, "a2_real_observed": 31760, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.010138539042821158, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.9158102915100802, "a3_ceiling_rank1": null, "a3_gallery": null, "a3_queries": null, "a3f_rank1": null, "a3f_rank1_sd": null, "a3f_rank5": null, "a3f_rank5_sd": null, "a3f_map": null, "a3f_map_sd": null, "a5_rank1": null, "a5_rank1_sd": null, "a5_rank5": null, "a5_rank5_sd": null, "a5_map": null, "a5_map_sd": null, "a5_folds": null, "relational_note": "no PERSON identity appears in both the gallery and the query half of the document-disjoint split (6661 and 6569 identities, 0 shared) \u2014 co-reference in this corpus is document-scoped, so a cross-document linkage attack has no true match to find" } }, "enron": { "intersection": { "token_recall": 0.008048229071644096, "entity_recall": 0.00020769014875105248, "precision": 0.8371204561653599, "information_weighted_precision": 0.8371204561653599, "replacements": 34402, "documents_missing_a_detector": 12505, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 728, "a2_bound_observed": 90019, "a2_bound_chance": 0.0007855459544383347, "a2_bound_prior_size": 1273, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 728, "a2_real_observed": 90019, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.008087181594996612, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.4408479289722474, "a3_ceiling_rank1": 0.7198024247867085, "a3_gallery": 3697, "a3_queries": 494, "a3f_rank1": 0.255081426509998, "a3f_rank1_sd": 0.0361669765180931, "a3f_rank5": 0.42112966398680685, "a3f_rank5_sd": 0.049611059617932496, "a3f_map": 0.32702544632102953, "a3f_map_sd": 0.03626031291712802, "a5_rank1": 0.2875077303648732, "a5_rank1_sd": 0.03385155327116466, "a5_rank5": 0.47577819006390437, "a5_rank5_sd": 0.05786509592085009, "a5_map": 0.37505336784072735, "a5_map_sd": 0.03798088220749492, "a5_folds": 5, "relational_note": null }, "union": { "token_recall": 0.9980892240018244, "entity_recall": 0.9885124894751613, "precision": 0.4491949787854945, "information_weighted_precision": 0.4491949787854945, "replacements": 1487379, "documents_missing_a_detector": 12505, "a2_bound_correct_by_alignment": 0, "a2_bound_undetected_recovered": 152, "a2_bound_observed": 106587, "a2_bound_chance": 1.4461943395953548e-05, "a2_bound_prior_size": 69147, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 152, "a2_real_observed": 106587, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.0014260650923658609, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.7303485633583007, "a3_ceiling_rank1": 0.7198024247867085, "a3_gallery": 3697, "a3_queries": 19059, "a3f_rank1": 0.009286931875283427, "a3f_rank1_sd": 0.0010089892787110753, "a3f_rank5": 0.012749860058817975, "a3f_rank5_sd": 0.001058790021196171, "a3f_map": 0.011795978181979328, "a3f_map_sd": 0.0010994460266306278, "a5_rank1": 0.039351308949104366, "a5_rank1_sd": 0.0030055517790447852, "a5_rank5": 0.06983577114130604, "a5_rank5_sd": 0.0027160160721858926, "a5_map": 0.05515780501352739, "a5_map_sd": 0.0020679410850880267, "a5_folds": 5, "relational_note": null }, "_bands": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "_abstention": [ { "prior": "corpus-internal", "phi": 0.0, "attempted": 106587, "abstained": 0, "accuracy_attempted": 0.0014260650923658609, "c_at_1": 0.0014260650923658609, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 0.05, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 0.25, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "corpus-internal", "phi": 1.5, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.0, "attempted": 106587, "abstained": 0, "accuracy_attempted": 0.0014260650923658609, "c_at_1": 0.0014260650923658609, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.05, "attempted": 15, "abstained": 106572, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 0.25, "attempted": 8, "abstained": 106579, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "US Census 2010 surnames", "phi": 1.5, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.0, "attempted": 106587, "abstained": 0, "accuracy_attempted": 0.0014260650923658609, "c_at_1": 0.0014260650923658609, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.05, "attempted": 4, "abstained": 106583, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 0.25, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 }, { "prior": "UCI given names", "phi": 1.5, "attempted": 0, "abstained": 106587, "accuracy_attempted": 0.0, "c_at_1": 0.0, "lift_over_chance": 0.0 } ], "_prior_sweep": [ { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "US Census 2010 surnames top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 100 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "US Census 2010 surnames top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 1,000 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "US Census 2010 surnames top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105; truncated to the 10,000 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 6.16370808678501e-06, "lift_over_chance": 0.0, "mean_candidate_set": 162240.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "US Census 2010 surnames", "prior_kind": "external", "prior_size": 162240, "prior_vintage": "2010", "prior_provenance": "US Census 2010, 162,253 surnames covering 90 % of those recorded; public domain under 17 U.S.C. 105" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.01, "lift_over_chance": 0.0, "mean_candidate_set": 100.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "UCI given names top-100", "prior_kind": "external", "prior_size": 100, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 100 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.001, "lift_over_chance": 0.0, "mean_candidate_set": 1000.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "UCI given names top-1,000", "prior_kind": "external", "prior_size": 1000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 1,000 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 0.0001, "lift_over_chance": 0.0, "mean_candidate_set": 10000.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "UCI given names top-10,000", "prior_kind": "external", "prior_size": 10000, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X; truncated to the 10,000 commonest" }, { "observed": 106587, "attempted": 106587, "abstained": 0, "correct_1_to_1": 152, "correct_1_to_many": 0, "correct_many_to_1": 0, "correct_many_to_many": 0, "wrong": 106435, "correct": 152, "expected_correct": 152.0, "accuracy_attempted": 0.0014260650923658609, "correct_by_alignment": 0, "accuracy_alignment": 0.0, "c_at_1": 0.0014260650923658609, "chance": 7.480942299492044e-06, "lift_over_chance": 0.0, "mean_candidate_set": 133673.0, "eccentricity": 0.0, "undetected_recovered": 152, "by_band": { "1": 0.000545647741305534, "2-4": 0.0011168145014067567, "20+": 0.005216220763923944, "5-19": 0.0025957844460595993 }, "prior": "UCI given names", "prior_kind": "external", "prior_size": 133673, "prior_vintage": "1880-2019", "prior_provenance": "UCI Gender by Name, CC BY 4.0, DOI 10.24432/C55G7X" } ], "vote2": { "token_recall": 0.8909051338502562, "entity_recall": 0.7325624473758069, "precision": 0.5931070928509815, "information_weighted_precision": 0.5931070928509815, "replacements": 1319373, "documents_missing_a_detector": 12505, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 1816, "a2_bound_observed": 115000, "a2_bound_chance": 2.5334414268342115e-05, "a2_bound_prior_size": 39472, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 1816, "a2_real_observed": 115000, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.015791304347826086, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.8929196580319054, "a3_ceiling_rank1": 0.7198024247867085, "a3_gallery": 3697, "a3_queries": 9355, "a3f_rank1": 0.05088188134687334, "a3f_rank1_sd": 0.0029127582241451015, "a3f_rank5": 0.06948156066274719, "a3f_rank5_sd": 0.006664865800823046, "a3f_map": 0.06113946268957872, "a3f_map_sd": 0.004002833055821663, "a5_rank1": 0.09535008017103154, "a5_rank1_sd": 0.007488353922498504, "a5_rank5": 0.1572421165152325, "a5_rank5_sd": 0.00674369226962588, "a5_map": 0.12622329979534647, "a5_map_sd": 0.007930030281747054, "a5_folds": 5, "relational_note": null }, "vote3": { "token_recall": 0.7827208397656304, "entity_recall": 0.6033342688745439, "precision": 0.6253347231772532, "information_weighted_precision": 0.6253347231772532, "replacements": 1146795, "documents_missing_a_detector": 12505, "a2_bound_correct_by_alignment": 1, "a2_bound_undetected_recovered": 1890, "a2_bound_observed": 110243, "a2_bound_chance": 2.7842744180866465e-05, "a2_bound_prior_size": 35916, "a2_real_correct_by_alignment": 0, "a2_real_undetected_recovered": 1890, "a2_real_observed": 110243, "a2_real_chance": 6.16370808678501e-06, "a2_real_prior": "US Census 2010 surnames", "a2_real_prior_size": 162240, "a2_real_prior_vintage": "2010", "a2_real_c_at_1": 0.017143945647342687, "a2_real_mean_candidate_set": 162240.0, "a2_rho": 0.9029463641080513, "a3_ceiling_rank1": 0.7198024247867085, "a3_gallery": 3697, "a3_queries": 9408, "a3f_rank1": 0.0660065050075677, "a3f_rank1_sd": 0.006961683849310322, "a3f_rank5": 0.08428956492606585, "a3f_rank5_sd": 0.005587900060479931, "a3f_map": 0.07645134594731028, "a3f_map_sd": 0.005664253135622635, "a5_rank1": 0.11203194764355903, "a5_rank1_sd": 0.00808459365833283, "a5_rank5": 0.18005808970628032, "a5_rank5_sd": 0.00833525650529468, "a5_map": 0.14592302460378337, "a5_map_sd": 0.007598244099486395, "a5_folds": 5, "relational_note": null } } }, "exposure_denominators": { "cardiode": { "person": { "distinct_entities": 308, "distinct_entities_exposed": 1, "entity_document_pairs": 1957, "entity_document_pairs_exposed": 1, "mentions": 2800, "mentions_exposed": 1, "documents": 400, "documents_exposed": 1, "cases": 270, "cases_exposed": 1, "cases_are_documents": false }, "replaced": { "distinct_entities": 7209, "distinct_entities_exposed": 32, "entity_document_pairs": 9055, "entity_document_pairs_exposed": 41, "mentions": 9978, "mentions_exposed": 41, "documents": 400, "documents_exposed": 38, "cases": 270, "cases_exposed": 36, "cases_are_documents": false } }, "tab": { "person": { "distinct_entities": 8701, "distinct_entities_exposed": 72, "entity_document_pairs": 8701, "entity_document_pairs_exposed": 72, "mentions": 15955, "mentions_exposed": 145, "documents": 1268, "documents_exposed": 47, "cases": 1268, "cases_exposed": 47, "cases_are_documents": true }, "replaced": { "distinct_entities": 30468, "distinct_entities_exposed": 2028, "entity_document_pairs": 30468, "entity_document_pairs_exposed": 2028, "mentions": 59845, "mentions_exposed": 3381, "documents": 1268, "documents_exposed": 730, "cases": 1268, "cases_exposed": 730, "cases_are_documents": true } }, "ontonotes": { "person": { "distinct_entities": 13852, "distinct_entities_exposed": 1278, "entity_document_pairs": 39240, "entity_document_pairs_exposed": 2132, "mentions": 47713, "mentions_exposed": 2638, "documents": 5994, "documents_exposed": 790, "cases": 5994, "cases_exposed": 790, "cases_are_documents": true }, "replaced": { "distinct_entities": 30267, "distinct_entities_exposed": 5027, "entity_document_pairs": 145119, "entity_document_pairs_exposed": 15454, "mentions": 175249, "mentions_exposed": 18132, "documents": 5994, "documents_exposed": 2128, "cases": 5994, "cases_exposed": 2128, "cases_are_documents": true } }, "enron": { "person": { "distinct_entities": 3886, "distinct_entities_exposed": 125, "entity_document_pairs": 216342, "entity_document_pairs_exposed": 3469, "mentions": 272503, "mentions_exposed": 4097, "documents": 58636, "documents_exposed": 3265, "cases": 16, "cases_exposed": 15, "cases_are_documents": false }, "replaced": { "distinct_entities": 20018, "distinct_entities_exposed": 293, "entity_document_pairs": 356300, "entity_document_pairs_exposed": 4093, "mentions": 716835, "mentions_exposed": 4828, "documents": 58636, "documents_exposed": 3458, "cases": 16, "cases_exposed": 15, "cases_are_documents": false } } }, "a4": [ { "corpus": "cardiode", "condition": "A", "context": false, "rank1": 1.0, "rank5": 1.0, "map": 1.0, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "A", "context": true, "rank1": 0.995, "rank5": 1.0, "map": 0.996, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "B", "context": false, "rank1": 0.09, "rank5": 0.51, "map": 0.2753869047619051, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "B", "context": true, "rank1": 0.16, "rank5": 0.55, "map": 0.3403551587301592, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "C", "context": false, "rank1": 0.135, "rank5": 0.585, "map": 0.33608531746031806, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "C", "context": true, "rank1": 0.22, "rank5": 0.69, "map": 0.4217777777777779, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "cardiode", "condition": "B", "context": false, "rank1": 0.085, "rank5": 0.355, "map": 0.2513035714285719, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "cardiode", "condition": "C", "context": false, "rank1": 0.12, "rank5": 0.55, "map": 0.32220039682539736, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "ontonotes", "condition": "A", "context": false, "rank1": 0.99, "rank5": 0.995, "map": 0.9932142857142857, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "A", "context": true, "rank1": 0.985, "rank5": 0.995, "map": 0.9905555555555555, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "B", "context": false, "rank1": 0.31, "rank5": 0.9, "map": 0.5484325396825396, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "B", "context": true, "rank1": 0.365, "rank5": 0.915, "map": 0.5895297619047619, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "C", "context": false, "rank1": 0.59, "rank5": 0.9, "map": 0.7334325396825396, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "C", "context": true, "rank1": 0.66, "rank5": 0.925, "map": 0.7803392857142856, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "ontonotes", "condition": "B", "context": false, "rank1": 0.29, "rank5": 0.86, "map": 0.5232162698412698, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "ontonotes", "condition": "C", "context": false, "rank1": 0.63, "rank5": 0.91, "map": 0.7493769841269843, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "tab", "condition": "A", "context": false, "rank1": 0.995, "rank5": 1.0, "map": 0.996, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "A", "context": true, "rank1": 1.0, "rank5": 1.0, "map": 1.0, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "B", "context": false, "rank1": 0.085, "rank5": 0.5, "map": 0.2854503968253971, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "B", "context": true, "rank1": 0.12, "rank5": 0.51, "map": 0.3022976190476196, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "B-forewarned", "context": false, "rank1": 0.205, "rank5": 0.655, "map": 0.40754166666666697, "queries": 200, "candidates": 10, "tag": "15det", "adversary": "told the release is pseudonymised; the text itself is unchanged" }, { "corpus": "tab", "condition": "B-mask", "context": false, "rank1": 0.215, "rank5": 0.76, "map": 0.43785515873015873, "queries": 200, "candidates": 10, "tag": "15det", "adversary": "public name list over the released text, replaced spans unknown" }, { "corpus": "tab", "condition": "C", "context": false, "rank1": 0.29, "rank5": 0.85, "map": 0.5135456349206351, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "C", "context": true, "rank1": 0.31, "rank5": 0.715, "map": 0.4892242063492065, "queries": 200, "candidates": 10, "tag": "15det", "adversary": null }, { "corpus": "tab", "condition": "B", "context": false, "rank1": 0.11, "rank5": 0.575, "map": 0.317, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "tab", "condition": "B", "context": true, "rank1": 0.105, "rank5": 0.565, "map": 0.3197817460317467, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "tab", "condition": "B-forewarned", "context": false, "rank1": 0.165, "rank5": 0.605, "map": 0.359761904761905, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": "told the release is pseudonymised; the text itself is unchanged" }, { "corpus": "tab", "condition": "B-mask", "context": false, "rank1": 0.145, "rank5": 0.65, "map": 0.37422023809523836, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": "public name list over the released text, replaced spans unknown" }, { "corpus": "tab", "condition": "C", "context": false, "rank1": 0.34, "rank5": 0.845, "map": 0.547, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null }, { "corpus": "tab", "condition": "C", "context": true, "rank1": 0.3, "rank5": 0.735, "map": 0.47041666666666687, "queries": 200, "candidates": 10, "tag": "union-single0.5-13det-1797ba", "adversary": null } ], "utility": { "union": { "medication_ie": { "A": { "mean": 0.366753292044266, "sd": 0.18144589918064133, "n": 390, "median": 0.36363636363636365 }, "B": { "mean": 0.34640498031345446, "sd": 0.18032460437875106, "n": 390, "median": 0.3421602787456446 }, "C": { "mean": 0.3138429095217451, "sd": 0.19924901309256732, "n": 390, "median": 0.31968864468864466 } }, "medication_ie:in_narrative": { "A": { "mean": 0.0362211877770257, "sd": 0.06578874252307311, "n": 328, "median": 0.0 }, "B": { "mean": 0.030212995875389063, "sd": 0.05732843505852417, "n": 328, "median": 0.0 }, "C": { "mean": 0.015580506310581126, "sd": 0.03909931504015294, "n": 328, "median": 0.0 } }, "medication_ie:listed": { "A": { "mean": 0.37646645905541853, "sd": 0.19008180442652975, "n": 389, "median": 0.38 }, "B": { "mean": 0.3568139690829608, "sd": 0.1888760143259998, "n": 389, "median": 0.3584905660377358 }, "C": { "mean": 0.331396274467304, "sd": 0.20703330698000796, "n": 389, "median": 0.3448275862068966 } }, "section_classification": { "A": { "mean": 0.7492546251585532, "sd": 0.09283138206452367, "n": 400, "median": 0.75 }, "B": { "mean": 0.6770928836278101, "sd": 0.09736005900697943, "n": 400, "median": 0.6666666666666666 }, "C": { "mean": 0.7113011410351774, "sd": 0.10342109844360782, "n": 400, "median": 0.7272727272727273 } }, "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 400, "median": 1.0 }, "B": { "mean": 0.6107026046499481, "sd": 0.08158086718100307, "n": 400, "median": 0.6225240742190558 }, "C": { "mean": 0.003526841944170967, "sd": 0.0077362879379523075, "n": 400, "median": 0.0 } } }, "vote": { "medication_ie": { "A": { "mean": 0.3714427560321267, "sd": 0.18211298712911514, "n": 390, "median": 0.36363636363636365 }, "B": { "mean": 0.3638132025246152, "sd": 0.18925178840759688, "n": 390, "median": 0.3454907161803713 }, "C": { "mean": 0.38121096665302867, "sd": 0.18639133198235328, "n": 390, "median": 0.37932292426674447 } }, "medication_ie:in_narrative": { "A": { "mean": 0.041346195383436125, "sd": 0.07290141762609406, "n": 328, "median": 0.0 }, "B": { "mean": 0.036480317480015706, "sd": 0.06858139517217883, "n": 328, "median": 0.0 }, "C": { "mean": 0.03901960712315794, "sd": 0.0711977007161312, "n": 328, "median": 0.0 } }, "medication_ie:listed": { "A": { "mean": 0.3785712991974257, "sd": 0.189475183543318, "n": 389, "median": 0.3809523809523809 }, "B": { "mean": 0.3729744421118753, "sd": 0.19770512506829432, "n": 389, "median": 0.36363636363636365 }, "C": { "mean": 0.3901568860568582, "sd": 0.19524379634798972, "n": 389, "median": 0.39999999999999997 } }, "section_classification": { "A": { "mean": 0.752268518590113, "sd": 0.0904269271975863, "n": 400, "median": 0.75 }, "B": { "mean": 0.7088105517311206, "sd": 0.08949158853331018, "n": 400, "median": 0.7142857142857143 }, "C": { "mean": 0.7175180227731505, "sd": 0.0901301471616429, "n": 400, "median": 0.7272727272727273 } }, "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 400, "median": 1.0 }, "B": { "mean": 0.9733322646783801, "sd": 0.01854397529358048, "n": 400, "median": 0.9785924713584288 }, "C": { "mean": 0.6740487674889263, "sd": 0.08201710090265713, "n": 400, "median": 0.674886018237082 } } } }, "statistics": [ { "span_source": "union", "task": "medication_ie", "condition": "B", "n": 390, "test": "wilcoxon", "q_value": 0.005562762015271986, "significant": true, "effect": -0.16587390815532302, "effect_name": "rank_biserial", "reference_mean": 0.366753292044266, "reference_sd": 0.1816789704381608, "condition_mean": 0.3464049803134545, "condition_sd": 0.18055623530837828, "mean_difference": -0.020348311730811516, "difference_sd": 0.18239315264139544, "median_difference": -0.015628815628815684 }, { "span_source": "union", "task": "medication_ie", "condition": "C", "n": 390, "test": "wilcoxon", "q_value": 5.708338438423041e-07, "significant": true, "effect": -0.30626737967914436, "effect_name": "rank_biserial", "reference_mean": 0.366753292044266, "reference_sd": 0.1816789704381608, "condition_mean": 0.3138429095217451, "condition_sd": 0.1995049528423809, "mean_difference": -0.05291038252252091, "difference_sd": 0.20463747931065612, "median_difference": -0.03528391334286776 }, { "span_source": "union", "task": "medication_ie:listed", "condition": "B", "n": 389, "test": "wilcoxon", "q_value": 0.014735032449421974, "significant": true, "effect": -0.1473014447189161, "effect_name": "rank_biserial", "reference_mean": 0.3764664590554186, "reference_sd": 0.1903265975793808, "condition_mean": 0.35681396908296087, "condition_sd": 0.18911925462553447, "mean_difference": -0.01965248997245773, "difference_sd": 0.1906724804403054, "median_difference": -0.009534706331044918 }, { "span_source": "union", "task": "medication_ie:listed", "condition": "C", "n": 389, "test": "wilcoxon", "q_value": 7.066677018325862e-05, "significant": true, "effect": -0.24879227053140096, "effect_name": "rank_biserial", "reference_mean": 0.3764664590554186, "reference_sd": 0.1903265975793808, "condition_mean": 0.3313962744673041, "condition_sd": 0.20729993079554737, "mean_difference": -0.04507018458811451, "difference_sd": 0.21410919157599614, "median_difference": -0.02405300302462904 }, { "span_source": "union", "task": "ner_agreement", "condition": "B", "n": 400, "test": "wilcoxon", "q_value": 2.729752294832392e-67, "significant": true, "effect": -1.0, "effect_name": "rank_biserial", "reference_mean": 1.0, "reference_sd": 0.0, "condition_mean": 0.6107026046499481, "condition_sd": 0.0816830348693542, "mean_difference": -0.389297395350052, "difference_sd": 0.0816830348693542, "median_difference": -0.37747592578094424 }, { "span_source": "union", "task": "ner_agreement", "condition": "C", "n": 400, "test": "wilcoxon", "q_value": 3.0257378125018234e-73, "significant": true, "effect": -1.0, "effect_name": "rank_biserial", "reference_mean": 1.0, "reference_sd": 0.0, "condition_mean": 0.0035268419441709664, "condition_sd": 0.007745976467657264, "mean_difference": -0.996473158055829, "difference_sd": 0.007745976467657263, "median_difference": -1.0 }, { "span_source": "union", "task": "section_classification", "condition": "B", "n": 400, "test": "wilcoxon", "q_value": 1.5791408834190577e-37, "significant": true, "effect": -0.8463132958246965, "effect_name": "rank_biserial", "reference_mean": 0.7492546251585531, "reference_sd": 0.09294763931992804, "condition_mean": 0.67709288362781, "condition_sd": 0.09748198774480964, "mean_difference": -0.07216174153074309, "difference_sd": 0.08550298073784177, "median_difference": -0.07692307692307687 }, { "span_source": "union", "task": "section_classification", "condition": "C", "n": 400, "test": "wilcoxon", "q_value": 2.570140369391001e-14, "significant": true, "effect": -0.5314563781717067, "effect_name": "rank_biserial", "reference_mean": 0.7492546251585531, "reference_sd": 0.09294763931992804, "condition_mean": 0.7113011410351774, "condition_sd": 0.10355061771595479, "mean_difference": -0.037953484123375764, "difference_sd": 0.08902798897887693, "median_difference": 0.0 }, { "span_source": "vote", "task": "medication_ie", "condition": "B", "n": 390, "test": "wilcoxon", "q_value": 0.7979097180446482, "significant": false, "effect": -0.015465229433340819, "effect_name": "rank_biserial", "reference_mean": 0.3714427560321266, "reference_sd": 0.18234691527581054, "condition_mean": 0.3638132025246151, "condition_sd": 0.1894948865019112, "mean_difference": -0.007629553507511488, "difference_sd": 0.1775096765722376, "median_difference": 0.0 }, { "span_source": "vote", "task": "medication_ie", "condition": "C", "n": 390, "test": "wilcoxon", "q_value": 0.23467748144399034, "significant": false, "effect": 0.09471624266144814, "effect_name": "rank_biserial", "reference_mean": 0.3714427560321266, "reference_sd": 0.18234691527581054, "condition_mean": 0.3812109666530286, "condition_sd": 0.18663075575732987, "mean_difference": 0.009768210620902018, "difference_sd": 0.1754481323107879, "median_difference": 0.002634781151405176 }, { "span_source": "vote", "task": "medication_ie:listed", "condition": "B", "n": 389, "test": "wilcoxon", "q_value": 0.9473415486400709, "significant": false, "effect": 0.004016620498614959, "effect_name": "rank_biserial", "reference_mean": 0.37857129919742566, "reference_sd": 0.18971919547126936, "condition_mean": 0.37297444211187536, "condition_sd": 0.19795973576628453, "mean_difference": -0.005596857085550382, "difference_sd": 0.1829700478042764, "median_difference": 0.0 }, { "span_source": "vote", "task": "medication_ie:listed", "condition": "C", "n": 389, "test": "wilcoxon", "q_value": 0.18483921059121516, "significant": false, "effect": 0.1023391812865497, "effect_name": "rank_biserial", "reference_mean": 0.37857129919742566, "reference_sd": 0.18971919547126936, "condition_mean": 0.3901568860568583, "condition_sd": 0.19549523727168477, "mean_difference": 0.011585586859432544, "difference_sd": 0.18297853772540412, "median_difference": 0.0 }, { "span_source": "vote", "task": "ner_agreement", "condition": "B", "n": 400, "test": "wilcoxon", "q_value": 1.790860632484196e-66, "significant": true, "effect": -1.0, "effect_name": "rank_biserial", "reference_mean": 1.0, "reference_sd": 0.0, "condition_mean": 0.9733322646783802, "condition_sd": 0.01856719881588482, "mean_difference": -0.02666773532161991, "difference_sd": 0.01856719881588482, "median_difference": -0.02140752864157125 }, { "span_source": "vote", "task": "ner_agreement", "condition": "C", "n": 400, "test": "wilcoxon", "q_value": 5.458793901272145e-67, "significant": true, "effect": -1.0, "effect_name": "rank_biserial", "reference_mean": 1.0, "reference_sd": 0.0, "condition_mean": 0.6740487674889263, "condition_sd": 0.08211981490771783, "mean_difference": -0.3259512325110737, "difference_sd": 0.08211981490771782, "median_difference": -0.32511398176291795 }, { "span_source": "vote", "task": "section_classification", "condition": "B", "n": 400, "test": "wilcoxon", "q_value": 4.355602990623361e-25, "significant": true, "effect": -0.7455895455895456, "effect_name": "rank_biserial", "reference_mean": 0.752268518590113, "reference_sd": 0.09054017323719968, "condition_mean": 0.7088105517311206, "condition_sd": 0.08960366320281613, "mean_difference": -0.0434579668589924, "difference_sd": 0.06846437244690719, "median_difference": -0.0625 }, { "span_source": "vote", "task": "section_classification", "condition": "C", "n": 400, "test": "wilcoxon", "q_value": 2.286135292314031e-16, "significant": true, "effect": -0.5877932877932878, "effect_name": "rank_biserial", "reference_mean": 0.752268518590113, "reference_sd": 0.09054017323719968, "condition_mean": 0.7175180227731505, "condition_sd": 0.09024302152918083, "mean_difference": -0.03475049581696254, "difference_sd": 0.07809740451835419, "median_difference": 0.0 } ], "stability": [ { "corpus": "cardiode", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 1549, "surfaces": 1123, "collision_rate": 0.10114068441064639, "fragmentation_rate": 0.014848289218850872, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 1277, "surfaces": 1123, "collision_rate": 0.10114068441064639, "fragmentation_rate": 0.014848289218850872, "drift_rate": 0.050117462803445575, "cross_document": true, "scope": "across_documents" }, { "corpus": "cardiode", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 1652, "surfaces": 1579, "collision_rate": 0.0, "fragmentation_rate": 0.010895883777239709, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 1548, "surfaces": 1579, "collision_rate": 0.0, "fragmentation_rate": 0.010895883777239709, "drift_rate": 0.006459948320413436, "cross_document": true, "scope": "across_documents" }, { "corpus": "cardiode", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 944, "surfaces": 819, "collision_rate": 0.009453781512605041, "fragmentation_rate": 0.019067796610169493, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 812, "surfaces": 819, "collision_rate": 0.009453781512605041, "fragmentation_rate": 0.019067796610169493, "drift_rate": 0.029556650246305417, "cross_document": true, "scope": "across_documents" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 1347, "surfaces": 733, "collision_rate": 0.09380378657487091, "fragmentation_rate": 0.0066815144766146995, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 1056, "surfaces": 733, "collision_rate": 0.09380378657487091, "fragmentation_rate": 0.0066815144766146995, "drift_rate": 0.04071969696969697, "cross_document": true, "scope": "across_documents" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 474, "surfaces": 471, "collision_rate": 0.0, "fragmentation_rate": 0.0, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 473, "surfaces": 471, "collision_rate": 0.0, "fragmentation_rate": 0.0, "drift_rate": 0.0021141649048625794, "cross_document": true, "scope": "across_documents" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 449, "surfaces": 436, "collision_rate": 0.0, "fragmentation_rate": 0.0, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "cardiode", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 430, "surfaces": 436, "collision_rate": 0.0, "fragmentation_rate": 0.0, "drift_rate": 0.004651162790697674, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 290893, "surfaces": 34754, "collision_rate": 0.0006811890372045162, "fragmentation_rate": 0.3595995778516499, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 12624, "surfaces": 34754, "collision_rate": 0.0006811890372045162, "fragmentation_rate": 0.3595995778516499, "drift_rate": 0.37658428390367554, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 3797, "surfaces": 916, "collision_rate": 0.0017044071098125152, "fragmentation_rate": 0.06347116144324466, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 417, "surfaces": 916, "collision_rate": 0.0017044071098125152, "fragmentation_rate": 0.06347116144324466, "drift_rate": 0.23741007194244604, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 64827, "surfaces": 4679, "collision_rate": 0.17308231467982504, "fragmentation_rate": 0.28494300214416834, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 2894, "surfaces": 4679, "collision_rate": 0.17308231467982504, "fragmentation_rate": 0.28494300214416834, "drift_rate": 0.4861782999308915, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 199536, "surfaces": 10148, "collision_rate": 0.004260453891493016, "fragmentation_rate": 0.08232098468446797, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 5319, "surfaces": 10148, "collision_rate": 0.004260453891493016, "fragmentation_rate": 0.08232098468446797, "drift_rate": 0.4459484865576236, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 231, "surfaces": 74, "collision_rate": 0.0, "fragmentation_rate": 0.08658008658008658, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 16, "surfaces": 74, "collision_rate": 0.0, "fragmentation_rate": 0.08658008658008658, "drift_rate": 0.625, "cross_document": true, "scope": "across_documents" }, { "corpus": "enron", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 2429, "surfaces": 545, "collision_rate": 0.00036153289949385393, "fragmentation_rate": 0.09510086455331412, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "enron", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 144, "surfaces": 545, "collision_rate": 0.00036153289949385393, "fragmentation_rate": 0.09510086455331412, "drift_rate": 0.4583333333333333, "cross_document": true, "scope": "across_documents" }, { "corpus": "ontonotes", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 12076, "surfaces": 8508, "collision_rate": 0.1370425364660669, "fragmentation_rate": 0.11510433918516065, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "ontonotes", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 9846, "surfaces": 2880, "collision_rate": 0.16009992051776994, "fragmentation_rate": 0.13843185049766402, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "ontonotes", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 6709, "surfaces": 3928, "collision_rate": 0.07649744086318994, "fragmentation_rate": 0.16962289461916827, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "ontonotes", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 6870, "surfaces": 4886, "collision_rate": 0.06649262899262899, "fragmentation_rate": 0.036244541484716154, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "ontonotes", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 2671, "surfaces": 915, "collision_rate": 0.025112443778110945, "fragmentation_rate": 0.027704979408461252, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "ontonotes", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 1860, "surfaces": 912, "collision_rate": 0.02147074610842727, "fragmentation_rate": 0.02956989247311828, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "union", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 9070, "surfaces": 6601, "collision_rate": 0.023024445707788516, "fragmentation_rate": 0.1461962513781698, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "union", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 5703, "surfaces": 2850, "collision_rate": 0.026845637583892617, "fragmentation_rate": 0.13098369279326671, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "union", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 11222, "surfaces": 3502, "collision_rate": 0.01342882721575649, "fragmentation_rate": 0.2212618071644983, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "vote", "entity_type": "PERSON", "condition": "B", "policy": "deterministic", "chains": 8155, "surfaces": 5063, "collision_rate": 0.01520290024558531, "fragmentation_rate": 0.0621704475781729, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "vote", "entity_type": "LOC", "condition": "B", "policy": "deterministic", "chains": 3756, "surfaces": 1468, "collision_rate": 0.019401778496362168, "fragmentation_rate": 0.00718849840255591, "drift_rate": null, "cross_document": false, "scope": "within_document" }, { "corpus": "tab", "rule": "vote", "entity_type": "ORG", "condition": "B", "policy": "deterministic", "chains": 5335, "surfaces": 696, "collision_rate": 0.011915844349283188, "fragmentation_rate": 0.01874414245548266, "drift_rate": null, "cross_document": false, "scope": "within_document" } ], "both_rates": { "cardiode": { "union": { "rule": "union", "k": null, "members": 13, "overall": { "sensitivity": 0.9977799436863765, "sensitivity_gold_tokens": 18468, "person_sensitivity": 0.9997725204731575, "person_gold_tokens": 4396, "specificity": 0.8685838311367187, "precision": 0.5455002036106964, "all_gold_tokens": 120701, "total_tokens": 885059, "predicted_tokens": 221010, "documents": 400 } }, "vote-2": { "rule": "vote", "k": 2, "members": 13, "overall": { "sensitivity": 0.651667749620966, "sensitivity_gold_tokens": 18468, "person_sensitivity": 0.578252957233849, "person_gold_tokens": 4396, "specificity": 0.985621920618349, "precision": 0.908857946110913, "all_gold_tokens": 120701, "total_tokens": 885059, "predicted_tokens": 120581, "documents": 400 } }, "vote-3": { "rule": "vote", "k": 3, "members": 13, "overall": { "sensitivity": 0.6234026424084904, "sensitivity_gold_tokens": 18468, "person_sensitivity": 0.5625568698817106, "person_gold_tokens": 4396, "specificity": 0.996251756375939, "precision": 0.9743295671418459, "all_gold_tokens": 120701, "total_tokens": 885059, "predicted_tokens": 111607, "documents": 400 } }, "intersection": { "rule": "intersection", "k": null, "members": 13, "overall": { "sensitivity": 0.0011912497292614252, "sensitivity_gold_tokens": 18468, "person_sensitivity": 0.004549590536851683, "person_gold_tokens": 4396, "specificity": 1.0, "precision": 1.0, "all_gold_tokens": 120701, "total_tokens": 885059, "predicted_tokens": 22, "documents": 400 } } }, "tab": { "union": { "rule": "union", "k": null, "members": 13, "overall": { "sensitivity": 0.961395263671875, "sensitivity_gold_tokens": 163840, "person_sensitivity": 0.9957954035187982, "person_gold_tokens": 48994, "specificity": 0.8504030237287697, "precision": 0.5440377948286362, "all_gold_tokens": 298369, "total_tokens": 1844804, "predicted_tokens": 507371, "documents": 1268 } }, "vote-2": { "rule": "vote", "k": 2, "members": 13, "overall": { "sensitivity": 0.85379638671875, "sensitivity_gold_tokens": 163840, "person_sensitivity": 0.9601787974037638, "person_gold_tokens": 48994, "specificity": 0.9338782425384837, "precision": 0.7027287104764341, "all_gold_tokens": 298369, "total_tokens": 1844804, "predicted_tokens": 343972, "documents": 1268 } }, "vote-3": { "rule": "vote", "k": 3, "members": 13, "overall": { "sensitivity": 0.77113037109375, "sensitivity_gold_tokens": 163840, "person_sensitivity": 0.948075274523411, "person_gold_tokens": 48994, "specificity": 0.9803069640819045, "precision": 0.8785013544621448, "all_gold_tokens": 298369, "total_tokens": 1844804, "predicted_tokens": 250653, "documents": 1268 } }, "intersection": { "rule": "intersection", "k": null, "members": 13, "overall": { "sensitivity": 0.005902099609375, "sensitivity_gold_tokens": 163840, "person_sensitivity": 0.01973711066661224, "person_gold_tokens": 48994, "specificity": 0.9999954734599256, "precision": 0.9928131416837782, "all_gold_tokens": 298369, "total_tokens": 1844804, "predicted_tokens": 974, "documents": 1268 } } }, "ontonotes": { "union": { "rule": "union", "k": null, "members": 13, "overall": { "sensitivity": 0.8010963112598111, "sensitivity_gold_tokens": 475230, "person_sensitivity": 0.9352167341864979, "person_gold_tokens": 111819, "specificity": 0.9317901668638368, "precision": 0.6619469994123194, "all_gold_tokens": 737653, "total_tokens": 4726594, "predicted_tokens": 804859, "documents": 5994 }, "by_language": { "english": { "sensitivity": 0.9890358792482045, "sensitivity_gold_tokens": 191169, "person_sensitivity": 0.9913257965056527, "person_gold_tokens": 48650, "specificity": 0.9358671204578185, "precision": 0.7152671847124606, "all_gold_tokens": 338095, "total_tokens": 2192105, "predicted_tokens": 417595, "documents": 3637 }, "chinese": { "sensitivity": 0.8679511278195489, "sensitivity_gold_tokens": 68096, "person_sensitivity": 0.9315481686672823, "person_gold_tokens": 16245, "specificity": 0.9136532912759877, "precision": 0.5179744902827698, "all_gold_tokens": 119398, "total_tokens": 996930, "predicted_tokens": 157195, "documents": 1911 }, "arabic": { "sensitivity": 0.6136549903919617, "sensitivity_gold_tokens": 215965, "person_sensitivity": 0.8783138692353593, "person_gold_tokens": 46924, "specificity": 0.9384364072183929, "precision": 0.6635357218921281, "all_gold_tokens": 280160, "total_tokens": 1537559, "predicted_tokens": 230069, "documents": 446 } } }, "vote-2": { "rule": "vote", "k": 2, "members": 13, "overall": { "sensitivity": 0.6153609831029185, "sensitivity_gold_tokens": 475230, "person_sensitivity": 0.8635741689694953, "person_gold_tokens": 111819, "specificity": 0.982108534570955, "precision": 0.8367724081147222, "all_gold_tokens": 737653, "total_tokens": 4726594, "predicted_tokens": 437230, "documents": 5994 }, "by_language": { "english": { "sensitivity": 0.8674994376703336, "sensitivity_gold_tokens": 191169, "person_sensitivity": 0.9492086330935252, "person_gold_tokens": 48650, "specificity": 0.9762886931569948, "precision": 0.8333459698391879, "all_gold_tokens": 338095, "total_tokens": 2192105, "predicted_tokens": 263786, "documents": 3637 }, "chinese": { "sensitivity": 0.6279664003759399, "sensitivity_gold_tokens": 68096, "person_sensitivity": 0.8394582948599569, "person_gold_tokens": 16245, "specificity": 0.9815414138743659, "precision": 0.7700715421303657, "all_gold_tokens": 119398, "total_tokens": 996930, "predicted_tokens": 70448, "documents": 1911 }, "arabic": { "sensitivity": 0.3881971615771074, "sensitivity_gold_tokens": 215965, "person_sensitivity": 0.783138692353593, "person_gold_tokens": 46924, "specificity": 0.9910855663158632, "precision": 0.8911705308944037, "all_gold_tokens": 280160, "total_tokens": 1537559, "predicted_tokens": 102996, "documents": 446 } } }, "vote-3": { "rule": "vote", "k": 3, "members": 13, "overall": { "sensitivity": 0.4993624139890158, "sensitivity_gold_tokens": 475230, "person_sensitivity": 0.7825145994866705, "person_gold_tokens": 111819, "specificity": 0.9926296227494966, "precision": 0.9026267590939652, "all_gold_tokens": 737653, "total_tokens": 4726594, "predicted_tokens": 301931, "documents": 5994 }, "by_language": { "english": { "sensitivity": 0.8076257133740303, "sensitivity_gold_tokens": 191169, "person_sensitivity": 0.9362384378211717, "person_gold_tokens": 48650, "specificity": 0.9889045905901263, "precision": 0.8970853094794982, "all_gold_tokens": 338095, "total_tokens": 2192105, "predicted_tokens": 199884, "documents": 3637 }, "chinese": { "sensitivity": 0.4335203242481203, "sensitivity_gold_tokens": 68096, "person_sensitivity": 0.6507848568790398, "person_gold_tokens": 16245, "specificity": 0.9944514843903128, "precision": 0.8807815675424207, "all_gold_tokens": 119398, "total_tokens": 996930, "predicted_tokens": 40841, "documents": 1911 }, "arabic": { "sensitivity": 0.24725302711087443, "sensitivity_gold_tokens": 215965, "person_sensitivity": 0.6687409428011252, "person_gold_tokens": 46924, "specificity": 0.9968506416817573, "precision": 0.9353004607391432, "all_gold_tokens": 280160, "total_tokens": 1537559, "predicted_tokens": 61206, "documents": 446 } } }, "intersection": { "rule": "intersection", "k": null, "members": 13, "overall": { "sensitivity": 0.003143740925446626, "sensitivity_gold_tokens": 475230, "person_sensitivity": 0.013280390631288063, "person_gold_tokens": 111819, "specificity": 0.9999964902965474, "precision": 0.9907161803713528, "all_gold_tokens": 737653, "total_tokens": 4726594, "predicted_tokens": 1508, "documents": 5994 }, "by_language": { "english": { "sensitivity": 0.007815074619838991, "sensitivity_gold_tokens": 191169, "person_sensitivity": 0.03052415210688592, "person_gold_tokens": 48650, "specificity": 0.9999924488001682, "precision": 0.9907161803713528, "all_gold_tokens": 338095, "total_tokens": 2192105, "predicted_tokens": 1508, "documents": 3637 }, "chinese": { "sensitivity": 0.0, "sensitivity_gold_tokens": 68096, "person_sensitivity": 0.0, "person_gold_tokens": 16245, "specificity": 1.0, "precision": null, "all_gold_tokens": 119398, "total_tokens": 996930, "predicted_tokens": 0, "documents": 1911 }, "arabic": { "sensitivity": 0.0, "sensitivity_gold_tokens": 215965, "person_sensitivity": 0.0, "person_gold_tokens": 46924, "specificity": 1.0, "precision": null, "all_gold_tokens": 280160, "total_tokens": 1537559, "predicted_tokens": 0, "documents": 446 } } } }, "enron": { "union": { "rule": "union", "k": null, "members": 13, "overall": { "sensitivity": 0.9980892240018244, "sensitivity_gold_tokens": 3648256, "person_sensitivity": 0.9905757753903209, "person_gold_tokens": 661699, "specificity": 0.6235172330694896, "precision": 0.4491949787854945, "all_gold_tokens": 3648256, "total_tokens": 15507925, "predicted_tokens": 8106246, "documents": 58636 } }, "vote-2": { "rule": "vote", "k": 2, "members": 13, "overall": { "sensitivity": 0.8909051338502562, "sensitivity_gold_tokens": 3648256, "person_sensitivity": 0.8310137993256753, "person_gold_tokens": 661699, "specificity": 0.811985562160293, "precision": 0.5931070928509815, "all_gold_tokens": 3648256, "total_tokens": 15507925, "predicted_tokens": 5480039, "documents": 58636 } }, "vote-3": { "rule": "vote", "k": 3, "members": 13, "overall": { "sensitivity": 0.7827208397656304, "sensitivity_gold_tokens": 3648256, "person_sensitivity": 0.8223996106991245, "person_gold_tokens": 661699, "specificity": 0.8557384695980975, "precision": 0.6253347231772532, "all_gold_tokens": 3648256, "total_tokens": 15507925, "predicted_tokens": 4566460, "documents": 58636 } }, "intersection": { "rule": "intersection", "k": null, "members": 13, "overall": { "sensitivity": 0.008048229071644096, "sensitivity_gold_tokens": 3648256, "person_sensitivity": 0.044328312419997615, "person_gold_tokens": 661699, "specificity": 0.9995182833517529, "precision": 0.8371204561653599, "all_gold_tokens": 3648256, "total_tokens": 15507925, "predicted_tokens": 35075, "documents": 58636 } } } }, "exposure_from_release": { "cardiode": { "person": { "distinct_people": 308, "distinct_people_exposed": 27, "entity_document_pairs": 1957, "entity_document_pairs_exposed": 28, "mentions": 2800, "mentions_exposed": 34, "documents": 400, "documents_exposed": 17, "cases": 270, "cases_exposed": 17, "cases_are_real": true, "exposed_mentions_clipped": 2, "exposed_mentions_never_found": 32 }, "replaced": { "distinct_people": 7209, "distinct_people_exposed": 211, "entity_document_pairs": 9055, "entity_document_pairs_exposed": 221, "mentions": 9978, "mentions_exposed": 227, "documents": 400, "documents_exposed": 155, "cases": 270, "cases_exposed": 131, "cases_are_real": true, "exposed_mentions_clipped": 102, "exposed_mentions_never_found": 125 } }, "tab": { "person": { "distinct_people": 8701, "distinct_people_exposed": 139, "entity_document_pairs": 8701, "entity_document_pairs_exposed": 139, "mentions": 15955, "mentions_exposed": 220, "documents": 1268, "documents_exposed": 90, "cases": 1268, "cases_exposed": 90, "cases_are_real": false, "exposed_mentions_clipped": 129, "exposed_mentions_never_found": 91 }, "replaced": { "distinct_people": 30468, "distinct_people_exposed": 2648, "entity_document_pairs": 30468, "entity_document_pairs_exposed": 2648, "mentions": 59845, "mentions_exposed": 4127, "documents": 1268, "documents_exposed": 879, "cases": 1268, "cases_exposed": 879, "cases_are_real": false, "exposed_mentions_clipped": 1601, "exposed_mentions_never_found": 2526 } }, "ontonotes": { "person": { "distinct_people": 13852, "distinct_people_exposed": 1577, "entity_document_pairs": 39240, "entity_document_pairs_exposed": 2707, "mentions": 47713, "mentions_exposed": 3216, "documents": 5994, "documents_exposed": 1156, "cases": 5994, "cases_exposed": 1156, "cases_are_real": false, "exposed_mentions_clipped": 672, "exposed_mentions_never_found": 2544 }, "replaced": { "distinct_people": 30267, "distinct_people_exposed": 5402, "entity_document_pairs": 145119, "entity_document_pairs_exposed": 16580, "mentions": 175249, "mentions_exposed": 19284, "documents": 5994, "documents_exposed": 2584, "cases": 5994, "cases_exposed": 2584, "cases_are_real": false, "exposed_mentions_clipped": 4398, "exposed_mentions_never_found": 14886 } }, "enron": { "person": { "distinct_people": 3886, "distinct_people_exposed": 434, "entity_document_pairs": 216342, "entity_document_pairs_exposed": 5737, "mentions": 272503, "mentions_exposed": 6464, "documents": 58636, "documents_exposed": 4810, "cases": 16, "cases_exposed": 15, "cases_are_real": true, "exposed_mentions_clipped": 4022, "exposed_mentions_never_found": 2442, "exposed_mentions_by_region": { "header": 2641, "body": 3823 } }, "replaced": { "distinct_people": 20018, "distinct_people_exposed": 2284, "entity_document_pairs": 356300, "entity_document_pairs_exposed": 24245, "mentions": 716835, "mentions_exposed": 25934, "documents": 58636, "documents_exposed": 18425, "cases": 16, "cases_exposed": 15, "cases_are_real": true, "exposed_mentions_clipped": 23343, "exposed_mentions_never_found": 2591, "exposed_mentions_by_region": { "header": 21032, "body": 4902 } } } }, "release_skip": { "cardiode": { "tag": "union-single0.5-13det-1797ba", "condition": "B", "documents": 400, "replacements_written": 68869, "replacements_skipped": 13623, "detections_resolved": 82492, "skip_rate": 0.16514328662173303 }, "tab": { "tag": "union-single0.5-13det-1797ba", "condition": "B", "documents": 1268, "replacements_written": 162285, "replacements_skipped": 43263, "detections_resolved": 205548, "skip_rate": 0.210476385077938 }, "ontonotes": { "tag": "union-single0.5-13det-1797ba", "condition": "B", "documents": 5994, "replacements_written": 291493, "replacements_skipped": 81741, "detections_resolved": 373234, "skip_rate": 0.21900737874898857 }, "enron": { "tag": "union-single0.5-13det-1797ba", "condition": "B", "documents": 58636, "replacements_written": 1487379, "replacements_skipped": 1267162, "detections_resolved": 2754541, "skip_rate": 0.46002655251818725 } }, "enron_header_split": { "corpus": "enron", "ensemble": "recommended 13, union", "regions": { "header": { "tokens": 4866950, "gold_tokens": 3246404, "person_gold_tokens": 498041, "sensitivity": 0.9988590452697815, "person_sensitivity": 0.9934985272296859, "specificity": 0.34587910494364243, "predicted_tokens": 4302733, "false_positive_tokens": 1060033, "documents_with_a_finding": 2195, "documents_with_a_person_in_clear": 2167, "documents": 58636 }, "body": { "tokens": 10640975, "gold_tokens": 401852, "person_gold_tokens": 163658, "sensitivity": 0.9918701412460309, "person_sensitivity": 0.9816813110266531, "specificity": 0.6674590196836194, "predicted_tokens": 3803513, "false_positive_tokens": 3404928, "documents_with_a_finding": 1322, "documents_with_a_person_in_clear": 1154, "documents": 58636 } }, "person_entities": { "exposed_total": 125, "exposed_header_only": 53, "exposed_body_only": 51, "exposed_both": 21, "mentioned_total": 3886, "mentioned_header_only": 1758, "mentioned_body_only": 201 } }, "enron_gold_correction": { "corrected": "2026-09-25", "superseded": { "union_overall": { "sensitivity": 0.9766564382857266, "specificity": 0.6236256389808778, "person_sensitivity": 0.8897884397431054, "sensitivity_gold_tokens": 3780657, "person_gold_tokens": 794100 }, "exposure_person": { "distinct_people": 4015, "distinct_people_exposed": 499, "entity_document_pairs": 281476, "entity_document_pairs_exposed": 54217, "mentions": 398295, "mentions_exposed": 89434, "documents_exposed": 27998, "exposed_mentions_clipped": 4104, "exposed_mentions_never_found": 85330 }, "header_split_person_sensitivity": { "header": 0.9764778934500189, "body": 0.7295223404064501 }, "union_linkage": { "a3_ceiling_rank1": 0.705720920413764, "a3_gallery": 3816, "a3f_rank1": 0.009513857770385239, "a5_rank1": 0.04065893583804994 } } }, "utility_by_corpus": { "cardiode": { "union": { "medication_ie": { "A": { "mean": 0.366753292044266, "sd": 0.18144589918064133, "n": 390, "median": 0.36363636363636365 }, "B": { "mean": 0.34640498031345446, "sd": 0.18032460437875106, "n": 390, "median": 0.3421602787456446 }, "C": { "mean": 0.3138429095217451, "sd": 0.19924901309256732, "n": 390, "median": 0.31968864468864466 } }, "medication_ie:in_narrative": { "A": { "mean": 0.0362211877770257, "sd": 0.06578874252307311, "n": 328, "median": 0.0 }, "B": { "mean": 0.030212995875389063, "sd": 0.05732843505852417, "n": 328, "median": 0.0 }, "C": { "mean": 0.015580506310581126, "sd": 0.03909931504015294, "n": 328, "median": 0.0 } }, "medication_ie:listed": { "A": { "mean": 0.3764664590554186, "sd": 0.19008180442652975, "n": 389, "median": 0.38 }, "B": { "mean": 0.35681396908296087, "sd": 0.1888760143259998, "n": 389, "median": 0.3584905660377358 }, "C": { "mean": 0.3313962744673041, "sd": 0.20703330698000796, "n": 389, "median": 0.3448275862068966 } }, "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 400, "median": 1.0 }, "B": { "mean": 0.6107026046499481, "sd": 0.08158086718100307, "n": 400, "median": 0.6225240742190558 }, "C": { "mean": 0.003526841944170967, "sd": 0.0077362879379523075, "n": 400, "median": 0.0 } }, "section_classification": { "A": { "mean": 0.7492546251585531, "sd": 0.09283138206452367, "n": 400, "median": 0.75 }, "B": { "mean": 0.6770928836278101, "sd": 0.09736005900697943, "n": 400, "median": 0.6666666666666666 }, "C": { "mean": 0.7113011410351774, "sd": 0.10342109844360782, "n": 400, "median": 0.7272727272727273 } } }, "vote": { "medication_ie": { "A": { "mean": 0.3714427560321267, "sd": 0.18211298712911514, "n": 390, "median": 0.36363636363636365 }, "B": { "mean": 0.3638132025246152, "sd": 0.18925178840759688, "n": 390, "median": 0.3454907161803713 }, "C": { "mean": 0.3812109666530287, "sd": 0.18639133198235328, "n": 390, "median": 0.37932292426674447 } }, "medication_ie:in_narrative": { "A": { "mean": 0.041346195383436125, "sd": 0.07290141762609406, "n": 328, "median": 0.0 }, "B": { "mean": 0.03648031748001571, "sd": 0.06858139517217883, "n": 328, "median": 0.0 }, "C": { "mean": 0.03901960712315794, "sd": 0.0711977007161312, "n": 328, "median": 0.0 } }, "medication_ie:listed": { "A": { "mean": 0.3785712991974257, "sd": 0.189475183543318, "n": 389, "median": 0.3809523809523809 }, "B": { "mean": 0.37297444211187536, "sd": 0.19770512506829432, "n": 389, "median": 0.36363636363636365 }, "C": { "mean": 0.3901568860568582, "sd": 0.19524379634798972, "n": 389, "median": 0.39999999999999997 } }, "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 400, "median": 1.0 }, "B": { "mean": 0.9733322646783801, "sd": 0.01854397529358048, "n": 400, "median": 0.9785924713584288 }, "C": { "mean": 0.6740487674889263, "sd": 0.08201710090265713, "n": 400, "median": 0.674886018237082 } }, "section_classification": { "A": { "mean": 0.752268518590113, "sd": 0.0904269271975863, "n": 400, "median": 0.75 }, "B": { "mean": 0.7088105517311206, "sd": 0.08949158853331018, "n": 400, "median": 0.7142857142857143 }, "C": { "mean": 0.7175180227731505, "sd": 0.0901301471616429, "n": 400, "median": 0.7272727272727273 } } } }, "tab": { "union": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 1268, "median": 1.0 }, "B": { "mean": 0.4779961455396702, "sd": 0.0967260828470693, "n": 1268, "median": 0.4734833266372037 }, "C": { "mean": 0.0031376669716333323, "sd": 0.010547703400588235, "n": 1268, "median": 0.0 } } }, "vote": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 1268, "median": 1.0 }, "B": { "mean": 0.8980730008733983, "sd": 0.04980949193343357, "n": 1268, "median": 0.9069767441860465 }, "C": { "mean": 0.634147380574409, "sd": 0.10265561422916696, "n": 1268, "median": 0.6461538461538462 } } } }, "ontonotes": { "union": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 5994, "median": 1.0 }, "B": { "mean": 0.2948017072924662, "sd": 0.24229545403131383, "n": 5994, "median": 0.3 }, "C": { "mean": 0.015133929376673069, "sd": 0.10583430754903883, "n": 5994, "median": 0.0 } } }, "vote": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 5994, "median": 1.0 }, "B": { "mean": 0.8877388384673429, "sd": 0.15570924753214974, "n": 5994, "median": 0.9282857142857143 }, "C": { "mean": 0.7882191015171823, "sd": 0.21942762679502933, "n": 5994, "median": 0.8049878408444145 } } } }, "enron": { "union": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 58636, "median": 1.0 }, "B": { "mean": 0.3296887735018888, "sd": 0.1718063746860689, "n": 58636, "median": 0.31999999999999995 }, "C": { "mean": 0.033519424756205084, "sd": 0.08775361885250156, "n": 58636, "median": 0.0 } } }, "vote": { "ner_agreement": { "A": { "mean": 1.0, "sd": 0.0, "n": 58636, "median": 1.0 }, "B": { "mean": 0.7537390534835187, "sd": 0.15961469534971792, "n": 58636, "median": 0.7777777777777777 }, "C": { "mean": 0.7143500435359076, "sd": 0.1787412772838378, "n": 58636, "median": 0.7499999999999999 } } } } } }