{
  "label": "REAL outcomes",
  "n_raters": 127,
  "n_complete": 90,
  "by_k": {
    "1": {
      "panel": {
        "draws": 10000,
        "both_cells_nonempty": 1.0,
        "median_gap": 0.7,
        "gap_2.5": 0.41666666666666663,
        "gap_97.5": 1.0,
        "rule_share": 0.8235,
        "pooled": {
          "act": {
            "held": 36130,
            "n": 36130,
            "rate": 1.0,
            "mean_size": 3.613
          },
          "try small": {
            "held": 74846,
            "n": 143515,
            "rate": 0.52152039856461,
            "mean_size": 14.3515
          },
          "wait": {
            "held": 29024,
            "n": 90355,
            "rate": 0.32122184715843066,
            "mean_size": 9.0355
          }
        },
        "decided_accuracy": 0.7705340554215915,
        "decided_coverage": 0.46846296296296297,
        "lay_alone_accuracy": 0.5811481481481482,
        "agreement_with_crowd": 0.6677111111111111
      },
      "independent": {
        "draws": 10000,
        "both_cells_nonempty": 0.9977,
        "median_gap": 0.7,
        "gap_2.5": 0.4444444444444444,
        "gap_97.5": 1.0,
        "rule_share": 0.8661,
        "pooled": {
          "act": {
            "held": 35889,
            "n": 35889,
            "rate": 1.0,
            "mean_size": 3.5889
          },
          "try small": {
            "held": 75824,
            "n": 143438,
            "rate": 0.5286186366234888,
            "mean_size": 14.3438
          },
          "wait": {
            "held": 28287,
            "n": 90673,
            "rate": 0.31196717876324814,
            "mean_size": 9.0673
          }
        },
        "decided_accuracy": 0.7764968948025474,
        "decided_coverage": 0.46874814814814814,
        "lay_alone_accuracy": 0.5863703703703703,
        "agreement_with_crowd": 0.6690296296296296
      },
      "verdict": "SMALL GROUP KEEPS THE SORT (k = 1)"
    },
    "3": {
      "panel": {
        "draws": 10000,
        "both_cells_nonempty": 1.0,
        "median_gap": 0.7272727272727273,
        "gap_2.5": 0.4285714285714286,
        "gap_97.5": 1.0,
        "rule_share": 0.8783,
        "pooled": {
          "act": {
            "held": 40336,
            "n": 40336,
            "rate": 1.0,
            "mean_size": 4.0336
          },
          "try small": {
            "held": 76738,
            "n": 149155,
            "rate": 0.5144849317823741,
            "mean_size": 14.9155
          },
          "wait": {
            "held": 22926,
            "n": 80509,
            "rate": 0.28476319417704854,
            "mean_size": 8.0509
          }
        },
        "decided_accuracy": 0.8102859034300136,
        "decided_coverage": 0.44757407407407407,
        "lay_alone_accuracy": 0.6088259259259259,
        "agreement_with_crowd": 0.7342111111111111
      },
      "independent": {
        "draws": 10000,
        "both_cells_nonempty": 0.9998,
        "median_gap": 0.7272727272727273,
        "gap_2.5": 0.5,
        "gap_97.5": 1.0,
        "rule_share": 0.9029,
        "pooled": {
          "act": {
            "held": 39986,
            "n": 39986,
            "rate": 1.0,
            "mean_size": 3.9986
          },
          "try small": {
            "held": 77692,
            "n": 148563,
            "rate": 0.5229565908065938,
            "mean_size": 14.8563
          },
          "wait": {
            "held": 22322,
            "n": 81451,
            "rate": 0.27405433941879165,
            "mean_size": 8.1451
          }
        },
        "decided_accuracy": 0.8161845236624752,
        "decided_coverage": 0.44976666666666665,
        "lay_alone_accuracy": 0.6148925925925925,
        "agreement_with_crowd": 0.7349518518518519
      },
      "verdict": "SMALL GROUP KEEPS THE SORT (k = 3)"
    },
    "5": {
      "panel": {
        "draws": 10000,
        "both_cells_nonempty": 1.0,
        "median_gap": 0.75,
        "gap_2.5": 0.5,
        "gap_97.5": 1.0,
        "rule_share": 0.9044,
        "pooled": {
          "act": {
            "held": 42944,
            "n": 42944,
            "rate": 1.0,
            "mean_size": 4.2944
          },
          "try small": {
            "held": 77716,
            "n": 152228,
            "rate": 0.5105236881519826,
            "mean_size": 15.2228
          },
          "wait": {
            "held": 19340,
            "n": 74828,
            "rate": 0.2584594002245149,
            "mean_size": 7.4828
          }
        },
        "decided_accuracy": 0.8357843969704174,
        "decided_coverage": 0.4361925925925926,
        "lay_alone_accuracy": 0.6252666666666666,
        "agreement_with_crowd": 0.7749777777777778
      },
      "independent": {
        "draws": 10000,
        "both_cells_nonempty": 1.0,
        "median_gap": 0.75,
        "gap_2.5": 0.5,
        "gap_97.5": 1.0,
        "rule_share": 0.9325,
        "pooled": {
          "act": {
            "held": 42521,
            "n": 42521,
            "rate": 1.0,
            "mean_size": 4.2521
          },
          "try small": {
            "held": 78861,
            "n": 151784,
            "rate": 0.5195606915089864,
            "mean_size": 15.1784
          },
          "wait": {
            "held": 18618,
            "n": 75695,
            "rate": 0.24596076359072594,
            "mean_size": 7.5695
          }
        },
        "decided_accuracy": 0.8425086282736686,
        "decided_coverage": 0.437837037037037,
        "lay_alone_accuracy": 0.6319592592592592,
        "agreement_with_crowd": 0.7750259259259259
      },
      "verdict": "SMALL GROUP KEEPS THE SORT (k = 5)"
    }
  },
  "smallest_passing_k": 1
}