diff --git a/data/calibration-maps.json b/data/calibration-maps.json new file mode 100644 index 0000000..a853a4e --- /dev/null +++ b/data/calibration-maps.json @@ -0,0 +1,1449 @@ +{ + "generatedAt": "2026-06-11T23:06:22.047Z", + "fitOn": [ + { + "from": "2014-01-01", + "to": "2016-01-01" + }, + { + "from": "2016-01-01", + "to": "2018-01-01" + }, + { + "from": "2018-01-01", + "to": "2020-01-01" + }, + { + "from": "2020-01-01", + "to": "2022-01-01" + }, + { + "from": "2022-01-01", + "to": "2024-01-01" + } + ], + "config": { + "xi": 0.25, + "k": 3, + "w": 0.45, + "m": 3, + "beta": 0.3, + "gamma": 0, + "homeAdv": 70, + "kScale": 1 + }, + "lofo": { + "rawRps": 0.1724, + "calRps": 0.1722, + "rawEce": 0.0146, + "calEce": 0.0122 + }, + "maps": { + "home": { + "x": [ + 0, + 0.00066, + 0.00094, + 0.00096, + 0.00186, + 0.00189, + 0.00195, + 0.00204, + 0.00204, + 0.00295, + 0.00299, + 0.00381, + 0.00384, + 0.00417, + 0.00456, + 0.00463, + 0.00475, + 0.00483, + 0.00484, + 0.00521, + 0.00537, + 0.00548, + 0.0055, + 0.00576, + 0.00593, + 0.00595, + 0.00603, + 0.0062, + 0.00635, + 0.00658, + 0.00659, + 0.00659, + 0.00694, + 0.00715, + 0.00727, + 0.00752, + 0.00811, + 0.00811, + 0.00823, + 0.00855, + 0.00901, + 0.00908, + 0.00933, + 0.00933, + 0.00958, + 0.00977, + 0.00983, + 0.00992, + 0.01009, + 0.01019, + 0.01032, + 0.01035, + 0.01052, + 0.01057, + 0.01071, + 0.01078, + 0.01081, + 0.01082, + 0.01083, + 0.01093, + 0.01097, + 0.01103, + 0.01106, + 0.01127, + 0.01144, + 0.01161, + 0.01183, + 0.012, + 0.01203, + 0.01223, + 0.01228, + 0.0125, + 0.01256, + 0.0128, + 0.01317, + 0.01324, + 0.01331, + 0.01366, + 0.0138, + 0.01396, + 0.01412, + 0.01416, + 0.01425, + 0.01441, + 0.01483, + 0.01487, + 0.01501, + 0.01525, + 0.01527, + 0.01536, + 0.01555, + 0.01559, + 0.01577, + 0.01594, + 0.01615, + 0.01622, + 0.01624, + 0.01634, + 0.01638, + 0.01653, + 0.01656, + 0.01667, + 0.01675, + 0.01676, + 0.01688, + 0.01709, + 0.01712, + 0.01714, + 0.0174, + 0.01745, + 0.01745, + 0.01746, + 0.0176, + 0.0178, + 0.01788, + 0.01802, + 0.01808, + 0.01812, + 0.01832, + 0.01832, + 0.01834, + 0.03484, + 0.06353, + 0.0815, + 0.09709, + 0.10697, + 0.11063, + 0.11733, + 0.1583, + 0.19992, + 0.21557, + 0.23215, + 0.25865, + 0.31658, + 0.35279, + 0.35463, + 0.3591, + 0.36452, + 0.36854, + 0.37322, + 0.38979, + 0.40511, + 0.43135, + 0.456, + 0.46674, + 0.47834, + 0.49017, + 0.51197, + 0.52389, + 0.53601, + 0.56432, + 0.5822, + 0.59955, + 0.61984, + 0.66122, + 0.70514, + 0.71194, + 0.71669, + 0.7402, + 0.81232, + 0.87231, + 0.88602, + 0.8991, + 0.91495, + 0.92784, + 0.94474, + 0.95845, + 0.95853, + 0.95881, + 0.95891, + 0.95902, + 0.95938, + 0.9612, + 0.96188, + 0.96191, + 0.96256, + 0.96268, + 0.96312, + 0.96334, + 0.96357, + 0.9639, + 0.96404, + 0.96427, + 0.9649, + 0.96523, + 0.96578, + 0.96661, + 0.9667, + 0.96676, + 0.96711, + 0.96746, + 0.9675, + 0.96766, + 0.96818, + 0.96965, + 0.96977, + 0.96986, + 0.97042, + 0.97081, + 0.97084, + 0.97086, + 0.97122, + 0.97261, + 0.97304, + 0.97339, + 0.97342, + 0.97345, + 0.97384, + 0.97405, + 0.97488, + 0.9756, + 0.97577, + 0.97627, + 0.97689, + 0.97767, + 0.97894, + 0.97928, + 0.97929, + 0.97933, + 0.97973, + 0.98028, + 0.98076, + 0.98138, + 0.98172, + 0.98273, + 0.98291, + 0.9842, + 0.98482, + 0.98635, + 0.98725, + 0.98795, + 0.98797, + 0.98899, + 0.98969, + 0.99052, + 0.99065, + 0.99114, + 0.99144, + 0.99146, + 0.99211, + 0.99314, + 0.99348, + 0.99564, + 0.99576, + 0.99616, + 1 + ], + "y": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.05128, + 0.06349, + 0.08642, + 0.09091, + 0.11111, + 0.12069, + 0.13514, + 0.15634, + 0.16393, + 0.21138, + 0.22785, + 0.27843, + 0.33073, + 0.33333, + 0.33333, + 0.35616, + 0.36111, + 0.36842, + 0.38043, + 0.38983, + 0.39286, + 0.4441, + 0.45455, + 0.48797, + 0.52174, + 0.53873, + 0.54064, + 0.56522, + 0.57095, + 0.58865, + 0.61111, + 0.62834, + 0.64368, + 0.66901, + 0.74561, + 0.75, + 0.75806, + 0.78816, + 0.82337, + 0.85246, + 0.87838, + 0.89286, + 0.95349, + 0.96667, + 0.96774, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ] + }, + "draw": { + "x": [ + 0, + 0.00355, + 0.0039, + 0.00399, + 0.00595, + 0.00629, + 0.00693, + 0.00719, + 0.00771, + 0.00778, + 0.00807, + 0.00843, + 0.00856, + 0.00865, + 0.00943, + 0.00965, + 0.00972, + 0.01063, + 0.01064, + 0.01161, + 0.01192, + 0.01337, + 0.01427, + 0.01485, + 0.01508, + 0.01535, + 0.01632, + 0.01649, + 0.01681, + 0.01708, + 0.01752, + 0.01774, + 0.01785, + 0.01825, + 0.01838, + 0.01853, + 0.01861, + 0.01885, + 0.01917, + 0.01933, + 0.01978, + 0.02042, + 0.02179, + 0.02181, + 0.02183, + 0.02264, + 0.02271, + 0.02279, + 0.02282, + 0.02307, + 0.02321, + 0.02357, + 0.02433, + 0.02461, + 0.02475, + 0.02483, + 0.02514, + 0.02542, + 0.02546, + 0.02584, + 0.02592, + 0.02622, + 0.02666, + 0.02668, + 0.02688, + 0.02698, + 0.02761, + 0.0277, + 0.02793, + 0.02806, + 0.02848, + 0.02856, + 0.02856, + 0.0291, + 0.02914, + 0.02944, + 0.02969, + 0.02995, + 0.03029, + 0.03033, + 0.03104, + 0.03119, + 0.03158, + 0.03163, + 0.03185, + 0.03197, + 0.03207, + 0.03242, + 0.03281, + 0.03342, + 0.03346, + 0.03362, + 0.03431, + 0.03457, + 0.03462, + 0.03488, + 0.03525, + 0.03551, + 0.03556, + 0.03585, + 0.03595, + 0.0361, + 0.04653, + 0.06511, + 0.07541, + 0.08125, + 0.08655, + 0.09202, + 0.09988, + 0.13949, + 0.18434, + 0.19402, + 0.19505, + 0.19636, + 0.19769, + 0.19935, + 0.21011, + 0.21991, + 0.22858, + 0.2466, + 0.2602, + 0.26554, + 0.26694, + 0.26738, + 0.28161, + 0.3061, + 0.31667, + 0.31734, + 0.31783, + 0.31799, + 0.31839, + 0.32774, + 1 + ], + "y": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.01681, + 0.01695, + 0.02128, + 0.03846, + 0.06061, + 0.08, + 0.08491, + 0.12116, + 0.12702, + 0.15152, + 0.2, + 0.2, + 0.20513, + 0.21127, + 0.21459, + 0.22222, + 0.23242, + 0.24487, + 0.26124, + 0.27397, + 0.28846, + 0.29032, + 0.31325, + 0.32314, + 0.33333, + 0.33333, + 0.33333, + 0.5, + 0.5, + 0.66667, + 1 + ] + }, + "away": { + "x": [ + 0, + 0.00029, + 0.00033, + 0.00037, + 0.00057, + 0.00057, + 0.0007, + 0.00076, + 0.00079, + 0.00079, + 0.00086, + 0.00088, + 0.00105, + 0.00114, + 0.00129, + 0.00139, + 0.00142, + 0.00153, + 0.00173, + 0.0018, + 0.00181, + 0.00182, + 0.00187, + 0.00215, + 0.00219, + 0.0022, + 0.00224, + 0.00231, + 0.00234, + 0.00244, + 0.00247, + 0.00253, + 0.00259, + 0.00259, + 0.00262, + 0.00268, + 0.0029, + 0.00325, + 0.00329, + 0.0033, + 0.00331, + 0.00333, + 0.0034, + 0.00357, + 0.00361, + 0.0037, + 0.00372, + 0.00372, + 0.00377, + 0.00386, + 0.00392, + 0.00399, + 0.00402, + 0.00407, + 0.00417, + 0.00424, + 0.00431, + 0.00432, + 0.00449, + 0.0045, + 0.00451, + 0.00452, + 0.00457, + 0.00461, + 0.00464, + 0.00478, + 0.00481, + 0.00485, + 0.00504, + 0.00506, + 0.00511, + 0.00512, + 0.00528, + 0.00544, + 0.00547, + 0.0056, + 0.00563, + 0.00568, + 0.00569, + 0.00571, + 0.00573, + 0.00585, + 0.00591, + 0.0062, + 0.00621, + 0.00625, + 0.0064, + 0.0064, + 0.00641, + 0.00645, + 0.00649, + 0.00653, + 0.00655, + 0.00661, + 0.00668, + 0.00677, + 0.00677, + 0.00681, + 0.00696, + 0.00708, + 0.00709, + 0.00714, + 0.00714, + 0.00722, + 0.00741, + 0.00744, + 0.00747, + 0.00748, + 0.00757, + 0.0076, + 0.00765, + 0.00776, + 0.00776, + 0.00777, + 0.00784, + 0.00792, + 0.00794, + 0.00805, + 0.00805, + 0.00816, + 0.00821, + 0.00821, + 0.00823, + 0.00825, + 0.0083, + 0.00841, + 0.00841, + 0.00842, + 0.00844, + 0.00856, + 0.00858, + 0.00861, + 0.00866, + 0.00867, + 0.00872, + 0.0088, + 0.0088, + 0.00883, + 0.00894, + 0.00911, + 0.00923, + 0.00926, + 0.00926, + 0.00927, + 0.00928, + 0.00937, + 0.00938, + 0.00942, + 0.00949, + 0.00952, + 0.00954, + 0.00958, + 0.0096, + 0.00965, + 0.00972, + 0.00974, + 0.00976, + 0.00978, + 0.00987, + 0.0099, + 0.00991, + 0.00997, + 0.01007, + 0.01008, + 0.01013, + 0.01023, + 0.0103, + 0.01031, + 0.01031, + 0.01044, + 0.01045, + 0.0105, + 0.01056, + 0.01059, + 0.01066, + 0.01223, + 0.02107, + 0.03314, + 0.04463, + 0.05375, + 0.07406, + 0.09237, + 0.10181, + 0.13354, + 0.16271, + 0.18192, + 0.19491, + 0.2034, + 0.21875, + 0.23718, + 0.25943, + 0.29081, + 0.31872, + 0.32671, + 0.33758, + 0.34823, + 0.34854, + 0.3591, + 0.38265, + 0.39624, + 0.42619, + 0.45842, + 0.46323, + 0.48181, + 0.51731, + 0.56136, + 0.61736, + 0.65419, + 0.66442, + 0.67794, + 0.69047, + 0.69376, + 0.73441, + 0.81669, + 0.86047, + 0.87672, + 0.89738, + 0.90554, + 0.9059, + 0.9059, + 0.90673, + 0.90727, + 0.90809, + 0.90852, + 0.91023, + 0.91062, + 0.91089, + 0.91118, + 0.91171, + 0.91271, + 0.91302, + 0.91353, + 0.91373, + 0.91384, + 0.91472, + 0.91498, + 0.91524, + 0.91562, + 0.9162, + 0.91651, + 0.91713, + 0.91758, + 0.91765, + 0.91809, + 0.92091, + 0.92092, + 0.92236, + 0.92347, + 0.92394, + 0.92518, + 0.92552, + 0.92852, + 0.92921, + 0.93079, + 0.93097, + 0.93097, + 0.93109, + 0.93181, + 0.93257, + 0.93397, + 0.93436, + 0.93508, + 0.93511, + 0.93589, + 0.93608, + 0.93694, + 0.93707, + 0.93853, + 0.939, + 0.93941, + 0.94014, + 0.94032, + 0.94044, + 0.94066, + 0.9412, + 0.94123, + 0.9419, + 0.94276, + 0.94668, + 0.94911, + 0.94923, + 0.95105, + 0.95152, + 0.95163, + 0.95334, + 0.95414, + 0.95414, + 0.95415, + 0.95541, + 0.95585, + 0.95594, + 0.95638, + 0.95853, + 0.95854, + 0.95903, + 0.9596, + 0.96191, + 0.96273, + 0.96519, + 0.96646, + 0.96847, + 0.96928, + 0.97028, + 0.97069, + 0.97231, + 0.97519, + 0.97863, + 0.97879, + 0.98062, + 0.98124, + 0.98276, + 0.98939, + 0.99041, + 0.99241, + 1 + ], + "y": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.01724, + 0.02941, + 0.04324, + 0.05085, + 0.05714, + 0.06631, + 0.07143, + 0.09477, + 0.11248, + 0.13559, + 0.16425, + 0.16667, + 0.17988, + 0.188, + 0.19382, + 0.21636, + 0.22741, + 0.25893, + 0.3, + 0.31429, + 0.33333, + 0.33333, + 0.34397, + 0.37582, + 0.4, + 0.41079, + 0.43396, + 0.43478, + 0.46392, + 0.51195, + 0.51485, + 0.5884, + 0.6, + 0.66667, + 0.66667, + 0.6875, + 0.75, + 0.75812, + 0.82979, + 0.875, + 0.93151, + 0.95238, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ] + } + } +} \ No newline at end of file diff --git a/data/cv-report.json b/data/cv-report.json index fef94b9..a230df2 100644 --- a/data/cv-report.json +++ b/data/cv-report.json @@ -1,5 +1,5 @@ { - "generatedAt": "2026-06-11T22:41:08.860Z", + "generatedAt": "2026-06-11T23:06:22.047Z", "protocol": { "folds": [ { @@ -56,5 +56,15 @@ "testRps": 0.1647 }, "shipChange": false, - "runtimeSec": 72 + "isotonic": { + "bar": "LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%", + "lofo": { + "rawRps": 0.1724, + "calRps": 0.1722, + "rawEce": 0.0146, + "calEce": 0.0122 + }, + "ship": true + }, + "runtimeSec": 73 } \ No newline at end of file diff --git a/scripts/buildRatings.ts b/scripts/buildRatings.ts index fe6736a..5c03796 100644 --- a/scripts/buildRatings.ts +++ b/scripts/buildRatings.ts @@ -3,7 +3,7 @@ // World-Football-Elo rating per team. Then calibrates the goals model // (goals-per-Elo slope, mean goals) on the modern era and MLE-fits Dixon-Coles // rho. The runtime re-uses these ratings + params to predict and simulate. -import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; import { canonicalTeam, ALL_TEAMS } from '../src/lib/teams'; @@ -160,6 +160,23 @@ function main(): void { teamDc.mu = Math.round(teamDc.mu * 1e4) / 1e4; teamDc.homeMult = Math.round(teamDc.homeMult * 1e4) / 1e4; + // Isotonic calibration maps — research artifact from scripts/cvSearch.ts + // (committed; only valid while the shipped config matches its fit config). + let calibration: unknown = null; + const calFile = join(ROOT, 'data', 'calibration-maps.json'); + if (existsSync(calFile)) { + const cal = JSON.parse(readFileSync(calFile, 'utf8')) as { + config: { xi: number; k: number; w: number; m: number; beta: number }; + maps: unknown; + }; + const c = cal.config; + if (c.xi === TEAMDC_XI && c.k === TEAMDC_SHRINK_K && c.w === ENSEMBLE_W && c.m === FAST_M && c.beta === FAST_BETA) { + calibration = cal.maps; + } else { + console.warn('calibration-maps.json was fit for a different config — skipping (rerun scripts/cvSearch.ts)'); + } + } + const out = { generatedAt: new Date().toISOString(), asOf: lastDate, @@ -178,6 +195,7 @@ function main(): void { fastBeta: FAST_BETA, fastM: FAST_M, teamDcFit: { xi: TEAMDC_XI, shrinkK: TEAMDC_SHRINK_K, windowYears: FIT_WINDOW_YEARS }, + ...(calibration ? { calibration } : {}), ratings, }; diff --git a/scripts/cvSearch.ts b/scripts/cvSearch.ts index f1f3cc9..fe6e760 100644 --- a/scripts/cvSearch.ts +++ b/scripts/cvSearch.ts @@ -257,6 +257,69 @@ function sweepW( return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) })); } +// --------------------------------------------------------------------------- +// Isotonic regression (PAV) for the calibration experiment: monotone map from +// predicted probability to observed frequency, fit per outcome class. +// --------------------------------------------------------------------------- +interface IsoMap { x: number[]; y: number[] } + +function fitIsotonic(pairs: { p: number; y: number }[]): IsoMap { + const sorted = [...pairs].sort((a, b) => a.p - b.p); + // pool-adjacent-violators on means, weighted by block size + const blocks: { sum: number; n: number; minP: number; maxP: number }[] = []; + for (const { p, y } of sorted) { + blocks.push({ sum: y, n: 1, minP: p, maxP: p }); + while (blocks.length > 1) { + const b = blocks[blocks.length - 1]!; + const a = blocks[blocks.length - 2]!; + if (a.sum / a.n <= b.sum / b.n) break; + blocks.splice(blocks.length - 2, 2, { sum: a.sum + b.sum, n: a.n + b.n, minP: a.minP, maxP: b.maxP }); + } + } + const x: number[] = [0]; + const y: number[] = [0]; + for (const b of blocks) { + x.push((b.minP + b.maxP) / 2); + y.push(b.sum / b.n); + } + x.push(1); y.push(1); + // enforce strictly increasing x for interpolation + for (let i = 1; i < x.length; i++) if (x[i]! <= x[i - 1]!) x[i] = x[i - 1]! + 1e-9; + return { x, y }; +} + +function applyIso(map: IsoMap, p: number): number { + const { x, y } = map; + if (p <= x[0]!) return y[0]!; + for (let i = 1; i < x.length; i++) { + if (p <= x[i]!) { + const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!); + return y[i - 1]! + f * (y[i]! - y[i - 1]!); + } + } + return y[y.length - 1]!; +} + +function calibrate(p: Probs, maps: { h: IsoMap; d: IsoMap; a: IsoMap }): Probs { + const h = Math.max(1e-6, applyIso(maps.h, p.h)); + const d = Math.max(1e-6, applyIso(maps.d, p.d)); + const a = Math.max(1e-6, applyIso(maps.a, p.a)); + const z = h + d + a; + return { h: h / z, d: d / z, a: a / z }; +} + +function eceOf(preds: { p: Probs; o: Outcome }[]): number { + const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 })); + for (const { p, o } of preds) { + for (const [v, hit] of [[p.h, o === 'h'], [p.d, o === 'd'], [p.a, o === 'a']] as [number, boolean][]) { + const i = Math.min(9, Math.floor(v * 10)); + bins[i]!.sp += v; bins[i]!.sy += hit ? 1 : 0; bins[i]!.n++; + } + } + const total = preds.length * 3; + return bins.reduce((s, b) => s + (b.n ? (b.n / total) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0); +} + function main(): void { const t0 = Date.now(); const rows = parse(); @@ -366,6 +429,82 @@ function main(): void { const shippedTest = testEval(SHIPPED); console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`); + // ---- isotonic calibration experiment (leave-one-fold-out, shipped config) ---- + // Bar (pre-registered): mean LOFO RPS must not worsen by >0.0002 AND mean + // LOFO ECE must improve by ≥10% relative — otherwise the layer stays out. + console.log('isotonic calibration experiment (LOFO)…'); + const probsFor = (preds: Pred[]): { p: Probs; o: Outcome }[] => { + const out: { p: Probs; o: Outcome }[] = []; + for (const p of preds) { + const r = rows[p.i]!; + out.push({ p: pooledFor(p), o: outcomeOf(r.hs, r.as) }); + } + return out; + }; + const pooledFor = (p: Pred): Probs => { + const el = eloLambdasFor(p, SHIPPED); + const mE = scoreMatrix(el.lh, el.la, baseParamsForRow(p).rho); + const mD = scoreMatrix(p.lDcH, p.lDcA, baseParamsForRow(p).rho); + return probsOf(SHIPPED.w === 0 ? mD : SHIPPED.w === 1 ? mE : poolMatrices(mE, mD, SHIPPED.w)); + }; + // helpers bound to the shipped config + per-fold params + const foldOfRow = (i: number): number => { + const t = rows[i]!.t; + return foldTs.findIndex((f) => t >= f.fromT && t < f.toT); + }; + const baseParamsForRow = (p: Pred): ModelParams => baseParams[Math.max(0, foldOfRow(p.i))]!; + const eloLambdasFor = (p: Pred, cfg: Cfg): { lh: number; la: number } => { + const fast = baseWalk.fast.get(cfg.m)!; + const eh = (1 - cfg.beta) * baseWalk.preH[p.i]! + cfg.beta * fast.h[p.i]!; + const ea = (1 - cfg.beta) * baseWalk.preA[p.i]! + cfg.beta * fast.a[p.i]!; + const params = baseParamsForRow(p); + const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[p.i]!.neutral ? 0 : params.homeAdvElo); + return { lh: lambdaHome, la: lambdaAway }; + }; + + const foldProbSets = foldPreds(SHIPPED.xi, SHIPPED.k).map(probsFor); + let rawRps = 0, calRps = 0, rawEce = 0, calEce = 0; + for (let held = 0; held < foldProbSets.length; held++) { + const train = foldProbSets.flatMap((s, i) => (i === held ? [] : s)); + const maps = { + h: fitIsotonic(train.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))), + d: fitIsotonic(train.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))), + a: fitIsotonic(train.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))), + }; + const heldSet = foldProbSets[held]!; + const calSet = heldSet.map(({ p, o }) => ({ p: calibrate(p, maps), o })); + rawRps += heldSet.reduce((s, x) => s + rps(x.p, x.o), 0) / heldSet.length; + calRps += calSet.reduce((s, x) => s + rps(x.p, x.o), 0) / calSet.length; + rawEce += eceOf(heldSet); + calEce += eceOf(calSet); + } + const nF = foldProbSets.length; + rawRps /= nF; calRps /= nF; rawEce /= nF; calEce /= nF; + const isoOk = calRps <= rawRps + 0.0002 && calEce <= rawEce * 0.9; + console.log(` LOFO raw: rps ${rawRps.toFixed(4)} ece ${rawEce.toFixed(4)} | calibrated: rps ${calRps.toFixed(4)} ece ${calEce.toFixed(4)} → ${isoOk ? 'SHIP isotonic layer' : 'KEEP raw (bar not met)'}`); + + // Final maps (fit on ALL folds) — committed via data/calibration-maps.json and + // embedded into ratings.json by buildRatings.ts when the experiment ships. + let finalMaps: { home: IsoMap; draw: IsoMap; away: IsoMap } | null = null; + if (isoOk) { + const all = foldProbSets.flat(); + finalMaps = { + home: fitIsotonic(all.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))), + draw: fitIsotonic(all.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))), + away: fitIsotonic(all.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))), + }; + const round = (m: IsoMap): IsoMap => ({ x: m.x.map((v) => +v.toFixed(5)), y: m.y.map((v) => +v.toFixed(5)) }); + finalMaps = { home: round(finalMaps.home), draw: round(finalMaps.draw), away: round(finalMaps.away) }; + writeFileSync(join(ROOT, 'data', 'calibration-maps.json'), JSON.stringify({ + generatedAt: new Date().toISOString(), + fitOn: FOLDS, + config: SHIPPED, + lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) }, + maps: finalMaps, + }, null, 2)); + console.log(' final maps → data/calibration-maps.json'); + } + mkdirSync(dirname(OUT), { recursive: true }); writeFileSync(OUT, JSON.stringify({ generatedAt: new Date().toISOString(), @@ -373,6 +512,11 @@ function main(): void { shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) }, champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) }, shipChange: ship, + isotonic: { + bar: 'LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%', + lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) }, + ship: isoOk, + }, runtimeSec: Math.round((Date.now() - t0) / 1000), }, null, 2)); console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`); diff --git a/src/lib/model/model.test.ts b/src/lib/model/model.test.ts index 2f2b7ae..2f200fb 100644 --- a/src/lib/model/model.test.ts +++ b/src/lib/model/model.test.ts @@ -114,3 +114,49 @@ describe('fast-Elo blend (v4 recency)', () => { expect(blendedRating(m, 'Strong')).toBeCloseTo(0.7 * 2050 + 0.3 * 1500, 9); }); }); + +describe('isotonic calibration (v5)', () => { + const base: RatingsModel = { + asOf: '2026-06-10', + params: PARAMS, + ratings: { Strong: 2050, Weak: 1650 }, + }; + // identity maps — calibration applied but should change nothing + const identity = { x: [0, 1], y: [0, 1] }; + // a map that inflates home probability + const homeBoost = { x: [0, 0.5, 1], y: [0, 0.6, 1] }; + + it('absent calibration ⇒ bit-identical (golden regression)', () => { + const a = predictMatch('Strong', 'Weak', base); + const b = predictMatch('Strong', 'Weak', { ...base }); + expect(b.probs).toEqual(a.probs); + }); + + it('identity maps leave probabilities (numerically) unchanged', () => { + const cal = { home: identity, draw: identity, away: identity }; + const a = predictMatch('Strong', 'Weak', base); + const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal }); + expect(b.probs.home).toBeCloseTo(a.probs.home, 9); + expect(b.probs.draw).toBeCloseTo(a.probs.draw, 9); + }); + + it('calibrated probabilities renormalize to 1 and shift as mapped', () => { + const cal = { home: homeBoost, draw: identity, away: identity }; + const a = predictMatch('Strong', 'Weak', base); + const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal }); + expect(b.probs.home + b.probs.draw + b.probs.away).toBeCloseTo(1, 9); + expect(b.probs.home).toBeGreaterThan(a.probs.home); + }); + + it('scoreline shapes survive within each outcome region', () => { + const cal = { home: homeBoost, draw: identity, away: identity }; + const a = predictMatch('Strong', 'Weak', base); + const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal }); + const ratio = (s: { home: number; away: number; p: number }[], h: number, a2: number) => + s.find((x) => x.home === h && x.away === a2)?.p ?? 0; + // two home-win scorelines keep their relative odds + const rawRatio = ratio(a.scorelines, 2, 0) / ratio(a.scorelines, 1, 0); + const calRatio = ratio(b.scorelines, 2, 0) / ratio(b.scorelines, 1, 0); + expect(calRatio).toBeCloseTo(rawRatio, 6); + }); +}); diff --git a/src/lib/model/predict.ts b/src/lib/model/predict.ts index f5582c1..352a7c6 100644 --- a/src/lib/model/predict.ts +++ b/src/lib/model/predict.ts @@ -27,6 +27,47 @@ export interface RatingsModel { fastM?: number; /** Team-DC fit hyper-params, shipped so the server can refit in-tournament. */ teamDcFit?: { xi: number; shrinkK: number; windowYears: number }; + /** Isotonic outcome calibration (fit walk-forward on the CV folds, validated + * leave-one-fold-out). Applied by scaling the matrix's outcome regions, so + * scorelines, simulations and displayed probabilities stay consistent. */ + calibration?: Calibration; +} + +export interface IsoMap { + x: number[]; + y: number[]; +} + +export interface Calibration { + home: IsoMap; + draw: IsoMap; + away: IsoMap; +} + +function isoApply(map: IsoMap, p: number): number { + const { x, y } = map; + if (p <= x[0]!) return y[0]!; + for (let i = 1; i < x.length; i++) { + if (p <= x[i]!) { + const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!); + return y[i - 1]! + f * (y[i]! - y[i - 1]!); + } + } + return y[y.length - 1]!; +} + +/** Scale the matrix's home/draw/away regions to the calibrated outcome masses. + * Scoreline shapes inside each region are preserved. */ +function calibrateMatrix(m: number[][], cal: Calibration): number[][] { + const p = outcomeProbs(m); + const h = Math.max(1e-6, isoApply(cal.home, p.home)); + const d = Math.max(1e-6, isoApply(cal.draw, p.draw)); + const a = Math.max(1e-6, isoApply(cal.away, p.away)); + const z = h + d + a; + const fh = p.home > 1e-12 ? h / z / p.home : 1; + const fd = p.draw > 1e-12 ? d / z / p.draw : 1; + const fa = p.away > 1e-12 ? a / z / p.away : 1; + return m.map((row, i) => row.map((v, j) => v * (i > j ? fh : i === j ? fd : fa))); } export const DEFAULT_ELO = 1500; @@ -72,7 +113,8 @@ export function matchMatrix( blendedRating(model, home), blendedRating(model, away), model.params, homeAdv, ); const mElo = scoreMatrix(lambdaHome, lambdaAway, model.params.rho); - if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return mElo; + const finish = (m: number[][]): number[][] => (model.calibration ? calibrateMatrix(m, model.calibration) : m); + if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return finish(mElo); // Team-DC orientation: homeAdv > 0 → home side at home; < 0 → away side at // home (compute flipped, then swap); 0 → neutral. @@ -85,7 +127,7 @@ export function matchMatrix( dcHome = f.lambdaHome; dcAway = f.lambdaAway; } const mDc = scoreMatrix(dcHome, dcAway, model.params.rho); - return poolMatrices(mElo, mDc, model.ensembleW); + return finish(poolMatrices(mElo, mDc, model.ensembleW)); } /** Expected goals per side under a scoreline matrix. */