#!/usr/bin/env python3 """Statistics for the report "The State of Free Soccer Data 2026" (/report/state-of-free-soccer-data-2026/). Single source of truth for every number on the report page, its charts and its OG image. Reads data/listings.json and data/taxonomy.json (standard library only, no network) and writes data/report-stats.json. Run: python3 scripts/report_stats.py && python3 scripts/make_report_charts.py Definitions (they mirror src/lib/coverage.js and scripts/coverage.py): listing One entry in data/listings.json. cell One (competition x data type) pair from data/taxonomy.json (26 x 9 = 234 cells). source cell listing.coverage.cells entry: a competition and data type that were seen together on the source's own site, docs, API response or repo. Each has access "free" or "paid". primary / via A cell with `via` set (client libraries, scrapers, MCP servers that re-fetch another source's data) is "via". Via cells are reported separately and never counted as sources. sources in a cell Distinct listings with a primary source cell for it. A listing is counted once per cell; if it has both a free and a paid source cell there, it counts as free (cellEntries()). free A free source cell. This includes the free tier of freemium listings (e.g. API-Football's free plan) and excludes their paid add-ons. "Paid" = the listing covers the cell, not for free. current free A free source cell that is not historical-only, sample-only or paused (see NOTE_RULES). A listing is "current free" in a cell when at least one of its free source cells there is current. mapped source A listing with at least one primary (non-via) source cell. Only mapped sources count in the map. not counted Listings whose data types are known but with no per-league cells (enterprise vendors without public league lists, free tracking samples without a named competition, libraries), via-only listings, listings outside the map's data types, and unverified listings. Historical / sample rule: every distinct cell note in the data is classified by NOTE_RULES (first match wins). The resulting table (note -> class -> listings) is written to report-stats.json under "note_classes" so the classification can be reviewed. The src/lib/coverage.js bestFree() penalty regex (/historical|selected seasons|10 matches|0\\.00/) was the starting point; differences: - "0.00" is not used: the only note with it (FootyStats, Liga MX Femenil) describes FootyStats' xG page, while that cell only claims match results, so the cell stays current and no xG is counted. - closed year ranges such as "1930-2022" (Fjelstul World Cup database) count as historical. - "Updates paused" (transfermarkt-datasets) counts as paused: not ongoing. - "limited seasons" (API-Football free plan) stays current but is flagged as a caveat, because the note does not say which seasons are included. """ import json, pathlib, re, statistics from collections import Counter, OrderedDict ROOT = pathlib.Path(__file__).resolve().parent.parent LISTINGS = json.loads((ROOT / "data" / "listings.json").read_text()) TAX = json.loads((ROOT / "data" / "taxonomy.json").read_text()) OUT = ROOT / "data" / "report-stats.json" PUBLISHED = "2026-10-27" COVERAGE_CHECKED = "2026-10-19" # COVERAGE_CHECKED in src/lib/coverage.js URL = "https://sportsdatastack.com/report/state-of-free-soccer-data-2026/" KEY_TYPES = ["xg", "event-data", "tracking-data", "player-stats"] LEAGUES = TAX["leagues"] TYPES = TAX["types"] L_IDS = [l["id"] for l in LEAGUES] T_IDS = [t["id"] for t in TYPES] LG = {l["id"]: l for l in LEAGUES} TY = {t["id"]: t for t in TYPES} # same order as src/lib/data.js (sorted by name) LISTINGS = sorted(LISTINGS, key=lambda l: l["name"].lower()) BY_SLUG = {l["slug"]: l for l in LISTINGS} NOTE_RULES = [ # (class, regex, explanation) ("historical", r"selected seasons|historical", "Selected past seasons only"), ("historical", r"\b(?:19|20)\d\d-(?:19|20)\d\d\b", "Closed range of past years (ends before the latest edition)"), ("sample", r"\b\d+ matches\b", "A fixed sample of matches"), ("paused", r"updates paused", "Updates paused; not ongoing"), ] CAVEAT_RULES = [("limited seasons", r"limited seasons", "Free plan has limited seasons; which seasons is not stated")] def note_class(note): for cls, rx, _ in NOTE_RULES: if note and re.search(rx, note, re.I): return cls return "current" def note_caveat(note): for name, rx, _ in CAVEAT_RULES: if note and re.search(rx, note, re.I): return name return None def cov(l): return l.get("coverage") or {"cells": [], "data_types": [], "competitions": []} def access_key(a): return a if a in ("free", "freemium", "paid") else "unknown" # ---------------------------------------------------------------- cells (mirrors cellEntries()) def cell_entries(league, typ): primary, via = OrderedDict(), OrderedDict() for l in LISTINGS: for c in cov(l)["cells"]: if league not in c["competitions"] or typ not in c["types"]: continue m = via if c["via"] else primary cur = c["access"] == "free" and note_class(c["note"]) == "current" prev = m.get(l["slug"]) if prev is None: m[l["slug"]] = {"slug": l["slug"], "access": c["access"], "note": c["note"], "via": c["via"], "cls": note_class(c["note"]) if c["access"] == "free" else None, "current": cur} else: if prev["access"] != "free" and c["access"] == "free": prev.update(access="free", note=c["note"], cls=note_class(c["note"])) prev["current"] = prev["current"] or cur return list(primary.values()), list(via.values()) CELLS = {} for lg in L_IDS: for t in T_IDS: p, v = cell_entries(lg, t) free = [e for e in p if e["access"] == "free"] CELLS[(lg, t)] = { "n": len(p), "free": len(free), "paid": len(p) - len(free), "current_free": sum(1 for e in free if e["current"]), "sources": [e["slug"] for e in p], "free_sources": [e["slug"] for e in free], "current_free_sources": [e["slug"] for e in free if e["current"]], "noncurrent_free": {e["slug"]: e["cls"] for e in free if not e["current"]}, "paid_sources": [e["slug"] for e in p if e["access"] != "free"], "via": [e["slug"] for e in v], } cell = lambda lg, t: CELLS[(lg, t)] def state(c): if c["n"] == 0: return "empty" if c["free"] == 0: return "paid-only" if c["current_free"] == 0: return "free-noncurrent-only" return "current-free" # ---------------------------------------------------------------- totals mapped = [l for l in LISTINGS if any(not c["via"] for c in cov(l)["cells"])] via_only = [l for l in LISTINGS if cov(l)["cells"] and all(c["via"] for c in cov(l)["cells"])] type_level_only = [l for l in LISTINGS if not cov(l)["cells"] and cov(l)["data_types"] and cov(l)["status"] != "unverified"] outside_map = [l for l in LISTINGS if not cov(l)["data_types"] and cov(l)["status"] != "unverified"] unverified = [l for l in LISTINGS if cov(l)["status"] == "unverified"] enterprise = [l for l in type_level_only if cov(l)["access"] == "paid"] n_cells = len(L_IDS) * len(T_IDS) states = Counter(state(c) for c in CELLS.values()) totals = { "listings": len(LISTINGS), "by_access": dict(Counter(access_key(cov(l)["access"]) for l in LISTINGS)), "by_format": dict(Counter(cov(l)["format"] for l in LISTINGS)), "by_coverage_status": dict(Counter(cov(l)["status"] for l in LISTINGS)), "by_listing_verification_status": dict(Counter(l["verification_status"] for l in LISTINGS)), "by_checked_date": dict(sorted(Counter(cov(l)["checked"] for l in LISTINGS).items())), "mapped_sources": len(mapped), "mapped_sources_by_access": dict(Counter(access_key(cov(l)["access"]) for l in mapped)), "not_counted": { "via_only": {"n": len(via_only), "slugs": [l["slug"] for l in via_only]}, "type_level_only": {"n": len(type_level_only), "slugs": [l["slug"] for l in type_level_only]}, "enterprise_paid_without_league_list": {"n": len(enterprise), "slugs": [l["slug"] for l in enterprise]}, "outside_map_types": {"n": len(outside_map), "slugs": [l["slug"] for l in outside_map]}, "unverified": {"n": len(unverified), "slugs": [l["slug"] for l in unverified], "notes": {l["slug"]: cov(l)["note"] for l in unverified}}, }, "leagues": len(L_IDS), "leagues_men": sum(1 for l in LEAGUES if l["gender"] == "men"), "leagues_women": sum(1 for l in LEAGUES if l["gender"] == "women"), "data_types": len(T_IDS), "cells_total": n_cells, "cells_covered": n_cells - states["empty"], "cells_with_free": sum(1 for c in CELLS.values() if c["free"] > 0), "cells_with_current_free": sum(1 for c in CELLS.values() if c["current_free"] > 0), "cells_free_noncurrent_only": states["free-noncurrent-only"], "cells_paid_only": states["paid-only"], "cells_empty": states["empty"], "source_cells_primary": sum(c["n"] for c in CELLS.values()), "source_cells_via": sum(len(c["via"]) for c in CELLS.values()), } assert totals["mapped_sources"] + sum(v["n"] for k, v in totals["not_counted"].items() if k != "enterprise_paid_without_league_list") == len(LISTINGS) # ---------------------------------------------------------------- note classification table note_classes = OrderedDict() for l in LISTINGS: for c in cov(l)["cells"]: if c["access"] != "free": continue k = c["note"] or "" e = note_classes.setdefault(k, {"note": c["note"], "class": note_class(c["note"]), "caveat": note_caveat(c["note"]), "listings": []}) tag = l["slug"] + (" (via)" if c["via"] else "") if tag not in e["listings"]: e["listings"].append(tag) note_classes = sorted(note_classes.values(), key=lambda e: (e["class"] == "current", e["note"] or "")) # ---------------------------------------------------------------- matrix matrix = { "leagues": [{k: l[k] for k in ("id", "short", "name", "gender", "group")} for l in LEAGUES], "types": [{k: t[k] for k in ("id", "short", "name")} for t in TYPES], "cells": {f"{lg}|{t}": {**{k: v for k, v in cell(lg, t).items()}, "state": state(cell(lg, t))} for lg in L_IDS for t in T_IDS}, } # ---------------------------------------------------------------- per data type by_type = [] for t in T_IDS: declaring = [l for l in LISTINGS if t in cov(l)["data_types"]] cs = [cell(lg, t) for lg in L_IDS] srcs = sorted({s for c in cs for s in c["sources"]}) free_srcs = sorted({s for c in cs for s in c["free_sources"]}) by_type.append({ "id": t, "short": TY[t]["short"], "name": TY[t]["name"], "leagues_covered": sum(1 for c in cs if c["n"]), "leagues_with_free": sum(1 for c in cs if c["free"]), "leagues_with_current_free": sum(1 for c in cs if c["current_free"]), "leagues_paid_only": sum(1 for c in cs if c["n"] and not c["free"]), "leagues_empty": sum(1 for c in cs if not c["n"]), "listings_declaring": dict(Counter(access_key(cov(l)["access"]) for l in declaring)), "listings_declaring_total": len(declaring), "listings_declaring_without_cells": sum(1 for l in declaring if not any(t in c["types"] for c in cov(l)["cells"])), "mapped_sources": len(srcs), "mapped_free_sources": len(free_srcs), "free_source_slugs": free_srcs, "source_cells": {"free": sum(c["free"] for c in cs), "paid": sum(c["paid"] for c in cs), "current_free": sum(c["current_free"] for c in cs)}, }) for b in by_type: for k in ("free", "freemium", "paid", "unknown"): b["listings_declaring"].setdefault(k, 0) BT = {b["id"]: b for b in by_type} # ---------------------------------------------------------------- xG def xg_row(lg): c = cell(lg, "xg") return {"id": lg, "short": LG[lg]["short"], "gender": LG[lg]["gender"], "group": LG[lg]["group"], "free": c["free"], "current_free": c["current_free"], "paid": c["paid"], "n": c["n"], "free_sources": c["free_sources"], "current_free_sources": c["current_free_sources"], "noncurrent_free": c["noncurrent_free"], "paid_sources": c["paid_sources"]} xg_rows = sorted([xg_row(lg) for lg in L_IDS], key=lambda r: (-r["free"], -r["current_free"], L_IDS.index(r["id"]))) def summ(vals): return {"n": len(vals), "mean": round(statistics.mean(vals), 2), "median": statistics.median(vals), "total": sum(vals)} xg_gender = {} for g in ("men", "women"): rows = [r for r in xg_rows if r["gender"] == g] xg_gender[g] = {"any_free": summ([r["free"] for r in rows]), "current_free": summ([r["current_free"] for r in rows]), "competitions_zero_free": [r["id"] for r in rows if r["free"] == 0], "competitions_zero_current_free": [r["id"] for r in rows if r["current_free"] == 0]} women_ids = [l["id"] for l in LEAGUES if l["gender"] == "women"] xg_sources_free = sorted({s for lg in L_IDS for s in cell(lg, "xg")["free_sources"]}) xg_sources_current = sorted({s for lg in L_IDS for s in cell(lg, "xg")["current_free_sources"]}) fem = cell("liga-mx-femenil", "xg") footy_fem = [c for c in cov(BY_SLUG["footystats"])["cells"] if "liga-mx-femenil" in c["competitions"]] xg = { "by_competition": xg_rows, "by_gender": xg_gender, "competitions_with_free": sum(1 for r in xg_rows if r["free"]), "competitions_with_current_free": sum(1 for r in xg_rows if r["current_free"]), "competitions_zero_free": [r["id"] for r in xg_rows if r["free"] == 0], "competitions_zero_current_free": [r["id"] for r in xg_rows if r["current_free"] == 0], "competitions_no_source": [r["id"] for r in xg_rows if r["n"] == 0], "free_xg_sources": xg_sources_free, "current_free_xg_sources": xg_sources_current, "women": { "competitions": len(women_ids), "with_current_free": [lg for lg in women_ids if cell(lg, "xg")["current_free"]], "without_current_free": [lg for lg in women_ids if not cell(lg, "xg")["current_free"]], "historical_only": [lg for lg in women_ids if cell(lg, "xg")["free"] and not cell(lg, "xg")["current_free"]], "no_free_at_all": [lg for lg in women_ids if not cell(lg, "xg")["free"]], }, "liga_mx_femenil": { "sources": fem["n"], "free": fem["free"], "paid": fem["paid"], "footystats_note": next((c["note"] for c in footy_fem if c["note"] and "0.00" in c["note"]), None), "footystats_checked": cov(BY_SLUG["footystats"])["checked"], "fotmob_note": next((c["note"] for c in cov(BY_SLUG["fotmob"])["cells"] if c["competitions"] == ["liga-mx-femenil"]), None), }, "type_level_only": [l["slug"] for l in LISTINGS if "xg" in cov(l)["data_types"] and not any("xg" in c["types"] for c in cov(l)["cells"])], } # ---------------------------------------------------------------- event data ev = [cell(lg, "event-data") for lg in L_IDS] event = { "competitions_with_any_source": sum(1 for c in ev if c["n"]), "competitions_with_free": sum(1 for c in ev if c["free"]), "competitions_with_current_free": sum(1 for c in ev if c["current_free"]), "free_sources": sorted({s for c in ev for s in c["free_sources"]}), "free_source_classes": sorted({cls for c in ev for cls in c["noncurrent_free"].values()}), "paid_sources": sorted({s for c in ev for s in c["paid_sources"]}), "competitions_zero_source": [lg for lg in L_IDS if cell(lg, "event-data")["n"] == 0], "competitions_zero_free": [lg for lg in L_IDS if cell(lg, "event-data")["free"] == 0], "competitions_zero_current_free": [lg for lg in L_IDS if cell(lg, "event-data")["current_free"] == 0], "type_level_only": [l["slug"] for l in LISTINGS if "event-data" in cov(l)["data_types"] and not any("event-data" in c["types"] for c in cov(l)["cells"])], } # ---------------------------------------------------------------- tracking def tracking_kind(l): c = cov(l) notes = " ".join([c["note"] or ""] + [x["note"] or "" for x in c["cells"] if "tracking-data" in x["types"]]) if c["format"] in ("library", "scraper"): return "library" # reads or models tracking data; ships no league data if re.search(r"your own", notes, re.I): return "own-video" if re.search(r"\b\w+ (anonymised )?matches\b|sample", notes, re.I) or c["access"] == "free" and c["format"] == "dataset": return "sample" return "league-data" trk_listings = [l for l in LISTINGS if "tracking-data" in cov(l)["data_types"]] trk = [cell(lg, "tracking-data") for lg in L_IDS] tracking = { "listings_declaring": dict(Counter(access_key(cov(l)["access"]) for l in trk_listings)), "listings": [{"slug": l["slug"], "name": l["name"], "access": access_key(cov(l)["access"]), "format": cov(l)["format"], "kind": tracking_kind(l), "has_league_cells": any("tracking-data" in c["types"] for c in cov(l)["cells"]), "note": cov(l)["note"] or next((c["note"] for c in cov(l)["cells"] if "tracking-data" in c["types"]), None)} for l in trk_listings], "competitions_with_any_cell": [lg for lg in L_IDS if trk[L_IDS.index(lg)]["n"]], "competitions_with_free": [lg for lg in L_IDS if cell(lg, "tracking-data")["free"]], "competitions_with_current_free": [lg for lg in L_IDS if cell(lg, "tracking-data")["current_free"]], "competitions_with_paid_cell": [lg for lg in L_IDS if cell(lg, "tracking-data")["paid"]], } tracking["kinds"] = {k: {a: sum(1 for x in tracking["listings"] if x["kind"] == k and x["access"] == a) for a in ("free", "freemium", "paid", "unknown")} for k in ("league-data", "sample", "own-video", "library")} for k in ("free", "freemium", "paid", "unknown"): tracking["listings_declaring"].setdefault(k, 0) tracking["free_cells_detail"] = [ {"league": lg, "short": LG[lg]["short"], "slug": s, "name": BY_SLUG[s]["name"], "class": cell(lg, "tracking-data")["noncurrent_free"].get(s, "current"), "note": next(x["note"] for x in cov(BY_SLUG[s])["cells"] if lg in x["competitions"] and "tracking-data" in x["types"]), "matches": (lambda m: int(m.group(1)) if m else None)(re.search(r"\b(\d+) matches\b", next(x["note"] or "" for x in cov(BY_SLUG[s])["cells"] if lg in x["competitions"] and "tracking-data" in x["types"])))} for lg in tracking["competitions_with_free"] for s in cell(lg, "tracking-data")["free_sources"]] event["free_source_names"] = [BY_SLUG[s]["name"] for s in event["free_sources"]] tracking["free_cell_notes"] = {lg: {s: next(x["note"] for x in cov(BY_SLUG[s])["cells"] if lg in x["competitions"] and "tracking-data" in x["types"]) for s in cell(lg, "tracking-data")["free_sources"]} for lg in tracking["competitions_with_free"]} # ---------------------------------------------------------------- gaps (key types) SEV = {"empty": 0, "paid-only": 1, "free-noncurrent-only": 2} gap_rows = [] for t in KEY_TYPES: for lg in L_IDS: c = cell(lg, t) s = state(c) if s in SEV: gap_rows.append({"league": lg, "short": LG[lg]["short"], "gender": LG[lg]["gender"], "type": t, "type_short": TY[t]["short"], "state": s, "n": c["n"], "free": c["free"], "paid_sources": c["paid_sources"], "noncurrent_free": c["noncurrent_free"]}) gap_rows.sort(key=lambda g: (SEV[g["state"]], KEY_TYPES.index(g["type"]), L_IDS.index(g["league"]))) gaps = { "rule": "Key types only (xG, event, tracking, player stats). Ranked: no source at all, then paid only, then free only as historical/sample/paused data; within a tier by type (xG, events, tracking, player stats) then competition order.", "ranked": gap_rows, "counts_by_type": {t: {s: sum(1 for g in gap_rows if g["type"] == t and g["state"] == s) for s in SEV} for t in KEY_TYPES}, "all_types": {"empty": states["empty"], "paid_only": states["paid-only"], "free_noncurrent_only": states["free-noncurrent-only"]}, "leagues_by_empty_key_cells": sorted( [{"league": lg, "short": LG[lg]["short"], "empty_key_types": [t for t in KEY_TYPES if cell(lg, t)["n"] == 0], "no_current_free_key_types": [t for t in KEY_TYPES if cell(lg, t)["current_free"] == 0]} for lg in L_IDS], key=lambda x: (-len(x["no_current_free_key_types"]), -len(x["empty_key_types"]), L_IDS.index(x["league"]))), } # ---------------------------------------------------------------- best free source per need def free_cells(l, leagues=None, types=None, current=True, include_via=False): """Distinct (competition, type) pairs where listing l has a free cell (current only if current=True).""" out = set() for c in cov(l)["cells"]: if c["access"] != "free" or (c["via"] and not include_via): continue if current and note_class(c["note"]) != "current": continue for lg in c["competitions"]: for t in c["types"]: if (leagues is None or lg in leagues) and (types is None or t in types): out.add((lg, t)) return out def rank(cands, leagues, types, include_via=False, current=True): scored = [] for l in cands: cells = free_cells(l, leagues, types, current, include_via) if not cells: continue c = cov(l) scored.append({ "slug": l["slug"], "name": l["name"], "cells": len(cells), "competitions": len({x[0] for x in cells}), "types": sorted({x[1] for x in cells}, key=T_IDS.index), "format": c["format"], "access": access_key(c["access"]), "status": c["status"], "checked": c["checked"], "license": l["license"], "commercial_use": l["commercial_use"], "notes": sorted({x["note"] for x in c["cells"] if x["access"] == "free" and x["note"] and any((lg in (leagues or L_IDS)) and (t in (types or T_IDS)) for lg in x["competitions"] for t in x["types"])}), "_k": (len(cells), c["status"] == "verified", c["format"] in ("api", "dataset"), len(c["data_types"]), l.get("github_stars") or 0), }) scored.sort(key=lambda s: (tuple(-int(x) for x in s["_k"]), s["name"])) for s in scored: s.pop("_k") return scored ALL = LISTINGS NEEDS = [ ("xg", "xG", dict(types=["xg"])), ("event-data", "Event data", dict(types=["event-data"])), ("tracking-data", "Tracking data", dict(types=["tracking-data"])), ("player-stats", "Player stats", dict(types=["player-stats"])), ("match-results", "Match results & fixtures", dict(types=["match-results"])), ("lineups", "Lineups", dict(types=["lineups"])), ("transfers-market-values", "Transfers & market values", dict(types=["transfers-market-values"])), ("odds", "Betting odds", dict(types=["odds"])), ("api", "API access", dict(filter=lambda l: cov(l)["format"] == "api")), ("libraries", "Python/R libraries", dict(filter=lambda l: cov(l)["format"] in ("library", "scraper"), via=True)), ("womens", "Women's football", dict(leagues=women_ids)), ("liga-mx", "Liga MX", dict(leagues=["liga-mx"])), ("mls", "MLS", dict(leagues=["mls"])), ("bulk", "Bulk dataset download", dict(filter=lambda l: cov(l)["format"] == "dataset")), ] best_free = [] for nid, label, o in NEEDS: cands = [l for l in ALL if o.get("filter", lambda l: True)(l)] lgs, tps, via = o.get("leagues"), o.get("types"), o.get("via", False) cur = rank(cands, lgs, tps, via, True) anyf = rank(cands, lgs, tps, via, False) scope_cells = (len(lgs) if lgs else len(L_IDS)) * (len(tps) if tps else len(T_IDS)) entry = {"need": nid, "label": label, "scope": {"leagues": lgs or "all", "types": tps or "all", "cells": scope_cells, "formats": {"api": ["api"], "libraries": ["library", "scraper"], "bulk": ["dataset"]}.get(nid), "via_counted": via}, "pick": cur[0] if cur else None, "current": bool(cur), "runners_up": cur[1:3], "noncurrent_pick": None if cur else (anyf[0] if anyf else None)} if entry["noncurrent_pick"]: s = entry["noncurrent_pick"]["slug"] entry["noncurrent_pick"]["class"] = sorted({note_class(c["note"]) for c in cov(BY_SLUG[s])["cells"] if c["access"] == "free" and any(t in (tps or T_IDS) for t in c["types"])} - {"current"}) if tps and len(tps) == 1: # free listings that offer the type without a named competition (e.g. samples) entry["free_without_league"] = [l["slug"] for l in ALL if tps[0] in cov(l)["data_types"] and access_key(cov(l)["access"]) == "free" and cov(l)["format"] in ("dataset", "api", "website") and not any(tps[0] in c["types"] for c in cov(l)["cells"])] best_free.append(entry) best_free_rule = ("Candidates: listings with free source cells in the need's scope (primary cells only, except Python/R libraries, " "where via cells are what the library fetches). Score = number of current free (competition x type) cells in scope; " "ties broken by verified status, API/dataset format, number of data types offered, GitHub stars, then name. " "If no current free cell exists, the best historical/sample option is shown and flagged.") # ---------------------------------------------------------------- FBref fb = BY_SLUG["fbref"] fbref = {"slug": "fbref", "name": fb["name"], "source_url": fb["verified_source"], "notes": fb["notes"], "coverage_note": cov(fb)["note"], "data_types": cov(fb)["data_types"], "has_xg_in_map": "xg" in cov(fb)["data_types"], "stats_xg": fb["stats"]["xg"], "last_verified": fb["last_verified"]} # ---------------------------------------------------------------- headline numbers (all derived above) def short(ids): return [LG[i]["short"] for i in ids] xr = {r["id"]: r for r in xg_rows} headline = { "listings": totals["listings"], "mapped_sources": totals["mapped_sources"], "cells_covered": totals["cells_covered"], "cells_total": totals["cells_total"], "cells_with_free": totals["cells_with_free"], "cells_with_current_free": totals["cells_with_current_free"], "xg_free": {k: xr[k]["free"] for k in ("premier-league", "mls", "liga-mx", "liga-mx-femenil")}, "xg_men_mean": xg_gender["men"]["any_free"]["mean"], "xg_women_mean": xg_gender["women"]["any_free"]["mean"], "xg_men_current_mean": xg_gender["men"]["current_free"]["mean"], "xg_women_current_mean": xg_gender["women"]["current_free"]["mean"], "women_xg_without_current_free": len(xg["women"]["without_current_free"]), "women_competitions": len(women_ids), "event_free_competitions": event["competitions_with_free"], "event_current_free_competitions": event["competitions_with_current_free"], "event_free_sources": len(event["free_sources"]), "event_zero_source": len(event["competitions_zero_source"]), "tracking_free_competitions": len(tracking["competitions_with_free"]), "tracking_current_free_competitions": len(tracking["competitions_with_current_free"]), "tracking_paid_listings": tracking["listings_declaring"]["paid"], "tracking_free_listings": tracking["listings_declaring"]["free"], } # ---------------------------------------------------------------- charts manifest (alt text from the numbers) top_xg = ", ".join(f"{r['short']} {r['free']}" for r in xg_rows[:4]) zero_xg = ", ".join(short(xg["competitions_zero_free"])) or "none" bt_bits = "; ".join(f"{b['short']}: {b['listings_declaring']['free']} free, {b['listings_declaring']['freemium']} freemium, {b['listings_declaring']['paid']} paid" for b in by_type) tfd = tracking["free_cells_detail"] trk_desc = "; ".join(f"{d['short']}, {d['name']}: {d['note'].rstrip('.')}" for d in tfd) or "none" ev_desc = " and ".join(event["free_source_names"]) or "no source" headline["tracking_free_detail"] = trk_desc headline["event_free_source_names"] = event["free_source_names"] gap_bits = "; ".join(f"{b['short']} {b['leagues_covered']} any source, {b['leagues_with_free']} free, {b['leagues_with_current_free']} current free" for b in by_type) CH = "/charts/state-of-free-soccer-data-2026-" charts = [ {"id": "heatmap", "file": CH + "coverage-heatmap.png", "width": 1200, "height": 1500, "title": "Free sources by competition and data type", "alt": (f"Heatmap of {totals['leagues']} competitions by {totals['data_types']} data types showing the number of free sources per cell. " f"{totals['cells_with_free']} of {totals['cells_total']} cells have a free source, {totals['cells_paid_only']} are paid only and " f"{totals['cells_empty']} have no verified source. Free event data comes only from {ev_desc} " f"({', '.join(event['free_source_classes'])} data), and tracking data has {len(tfd)} free cell ({trk_desc}).")}, {"id": "free-vs-paid", "file": CH + "free-vs-paid-by-data-type.png", "width": 1200, "height": 675, "title": "Free vs freemium vs paid listings by data type", "alt": f"Stacked horizontal bars: listings that offer each data type, split by access. {bt_bits}."}, {"id": "free-xg", "file": CH + "free-xg-sources-by-league.png", "width": 1200, "height": 1100, "title": "Free xG sources by competition", "alt": (f"Bar chart of free xG sources per competition, men's and women's colour-coded. Highest: {top_xg}. " f"Zero free xG sources: {zero_xg}. Men's competitions average {xg_gender['men']['any_free']['mean']} free xG sources, " f"women's {xg_gender['women']['any_free']['mean']}; excluding historical-only data, {xg_gender['men']['current_free']['mean']} vs " f"{xg_gender['women']['current_free']['mean']}.")}, {"id": "gaps", "file": CH + "biggest-gaps.png", "width": 1200, "height": 900, "title": "Competitions with any, free and current free sources by data type", "alt": (f"Grouped bars per data type: how many of the {totals['leagues']} competitions have any source, a free source, and a current " f"(not historical or sample-only) free source. {gap_bits}.")}, ] og = {"file": "/og-state-of-free-soccer-data-2026.png", "width": 1200, "height": 630, "alt": (f"The State of Free Soccer Data 2026 by SportsDataStack: free xG sources Premier League {headline['xg_free']['premier-league']}, " f"Liga MX Femenil {headline['xg_free']['liga-mx-femenil']}; {headline['event_current_free_competitions']} of {totals['leagues']} " f"competitions with current free event data; {headline['tracking_free_competitions']} of {totals['leagues']} competitions " f"with any free tracking data ({trk_desc}).")} out = { "meta": {"title": "The State of Free Soccer Data 2026", "url": URL, "published": PUBLISHED, "coverage_checked": COVERAGE_CHECKED, "generated_by": "scripts/report_stats.py", "inputs": ["data/listings.json", "data/taxonomy.json"], "note_rules": [{"class": c, "regex": r, "meaning": m} for c, r, m in NOTE_RULES], "caveat_rules": [{"caveat": c, "regex": r, "meaning": m} for c, r, m in CAVEAT_RULES], "best_free_rule": best_free_rule}, "headline": headline, "totals": totals, "note_classes": note_classes, "by_type": by_type, "xg": xg, "event": event, "tracking": tracking, "gaps": gaps, "best_free": best_free, "fbref": fbref, "charts": charts, "og": og, "matrix": matrix, } OUT.write_text(json.dumps(out, indent=1, ensure_ascii=False) + "\n") if __name__ == "__main__": h = headline print(f"wrote {OUT.relative_to(ROOT)}") print(f"listings {h['listings']} | mapped {h['mapped_sources']} | cells covered {h['cells_covered']}/{h['cells_total']} | " f"free {h['cells_with_free']} | current free {h['cells_with_current_free']} | paid only {totals['cells_paid_only']} | empty {totals['cells_empty']}") print("free xG:", h["xg_free"], "| men/women mean", h["xg_men_mean"], h["xg_women_mean"], "| current", h["xg_men_current_mean"], h["xg_women_current_mean"]) print("event: free comps", h["event_free_competitions"], "current", h["event_current_free_competitions"], "no source:", short(event["competitions_zero_source"])) print("tracking: free comps", tracking["competitions_with_free"], "| listings", tracking["listings_declaring"], "| kinds", tracking["kinds"]) for b in best_free: p = b["pick"] or b["noncurrent_pick"] print(f" best {b['label']}: {p['slug'] if p else None} ({p['cells'] if p else 0} cells){'' if b['current'] else ' [NOT CURRENT]'}")