""" score.py — the exact scorer behind the Sheetly OMR benchmark. https://www.trysheetly.com/compare/omr-benchmark Strict F1 at the note level: a transcribed note counts only when its pitch, its onset and its duration all match the ground truth. Ground truth and transcription are flattened to onset-ordered note sequences and aligned by longest common subsequence under that equality, so one misread rest cannot cascade into a hundred "wrong" notes that are merely shifted. This file is lifted verbatim out of Sheetly's evaluation harness — the same functions that produced every figure on the benchmark page. Standard library only, no dependencies. python score.py . # re-score the published bundle python score.py . --all-pages # include pages only some engines got python score.py truth.musicxml scan.musicxml # score one pair """ import json import sys import xml.etree.ElementTree as ET from array import array from pathlib import Path STEP_TO_SEMITONE = {"C": 0, "D": 2, "E": 4, "F": 5, "G": 7, "A": 9, "B": 11} # Strict-match tolerances, in quarter-note beats. Forgive representation noise # (divisions rounding, tuplet decimals) but fail any real displacement: the # smallest musical offset (a 32nd, 0.125) is over both thresholds. ONSET_TOL = 0.0625 DUR_TOL = 0.0625 def _midi(step: str, alter: float, octave: int) -> int: return int(round((octave + 1) * 12 + STEP_TO_SEMITONE.get(step, 0) + alter)) def note_events(xml_str: str, max_measures: int | None = None): """[(measure, onset_beats, midi, dur_beats, staff, step, alter)] for every pitched note in every part, tuple-ordered. Follows divisions/backup/forward/ chord. `staff` is a document-global staff index (a part with S staves takes S consecutive indices, top-down), so grand-staff hand assignment stays comparable when GT and OMR partition parts differently.""" root = ET.fromstring(xml_str.encode("utf-8")) for el in root.iter(): el.tag = el.tag.rsplit("}", 1)[-1] # strip namespaces if present events = [] staff_base = 0 for part in root.findall(".//part"): divisions = 1 part_events = [] part_staves = 1 for ordinal, measure in enumerate(part.findall("measure"), start=1): if max_measures is not None and ordinal > max_measures: break d = measure.findtext("attributes/divisions") if d and d.strip().isdigit() and int(d) > 0: divisions = int(d) declared = measure.findtext("attributes/staves") if declared and declared.strip().isdigit(): part_staves = max(part_staves, int(declared)) cursor = 0 last_onset = 0 last_staff = 1 for child in measure: if child.tag == "backup": cursor -= int(child.findtext("duration") or 0) elif child.tag == "forward": cursor += int(child.findtext("duration") or 0) elif child.tag == "note": dur = int(child.findtext("duration") or 0) if child.find("grace") is not None: dur = 0 is_chord = child.find("chord") is not None onset = last_onset if is_chord else cursor staff_txt = (child.findtext("staff") or "").strip() staff = int(staff_txt) if staff_txt.isdigit() else last_staff last_staff = staff part_staves = max(part_staves, staff) pitch = child.find("pitch") if pitch is not None: step = (pitch.findtext("step") or "C").strip() alter = float(pitch.findtext("alter") or 0) midi = _midi(step, alter, int(pitch.findtext("octave") or 4)) div = max(divisions, 1) part_events.append( (ordinal, round(onset / div, 3), midi, round(dur / div, 3), staff, step, alter) ) if not is_chord: last_onset = cursor cursor += dur events.extend( (m, onset, midi, dur, staff_base + staff - 1, step, alter) for (m, onset, midi, dur, staff, step, alter) in part_events ) staff_base += part_staves events.sort() return events def lcs_length(a: list, b: list) -> int: if not a or not b: return 0 prev = [0] * (len(b) + 1) for x in a: cur = [0] for j, y in enumerate(b, start=1): cur.append(prev[j - 1] + 1 if x == y else max(prev[j], cur[j - 1])) prev = cur return prev[-1] def lcs_match(a: list, b: list, eq) -> list: """LCS index pairs [(i, j)] under a custom equality predicate (classic DP; equality-with-tolerance is fine in practice). O(len(a)·len(b)).""" if not a or not b: return [] from array import array n, m = len(a), len(b) dp = [array("i", bytes(4 * (m + 1))) for _ in range(n + 1)] for i in range(1, n + 1): ai = a[i - 1] row, prev = dp[i], dp[i - 1] for j in range(1, m + 1): if eq(ai, b[j - 1]): row[j] = prev[j - 1] + 1 else: up, left = prev[j], row[j - 1] row[j] = up if up >= left else left pairs = [] i, j = n, m while i > 0 and j > 0: if eq(a[i - 1], b[j - 1]) and dp[i][j] == dp[i - 1][j - 1] + 1: pairs.append((i - 1, j - 1)) i -= 1 j -= 1 elif dp[i - 1][j] >= dp[i][j - 1]: i -= 1 else: j -= 1 pairs.reverse() return pairs def _strict_eq(x, y) -> bool: """Same pitch, same time, same length — measure-local, so a structural slip in one bar doesn't cascade into every later bar.""" return x[2] == y[2] and abs(x[1] - y[1]) <= ONSET_TOL and abs(x[3] - y[3]) <= DUR_TOL def measure_exact_rate(gt_events, omr_events): """Fraction of measures whose full (onset, midi, duration) content matches exactly, over every measure that has notes on either side.""" gt_by_m: dict = {} omr_by_m: dict = {} for e in gt_events: gt_by_m.setdefault(e[0], []).append((e[1], e[2], e[3])) for e in omr_events: omr_by_m.setdefault(e[0], []).append((e[1], e[2], e[3])) ordinals = set(gt_by_m) | set(omr_by_m) if not ordinals: return None ok = sum(1 for m in ordinals if sorted(gt_by_m.get(m, [])) == sorted(omr_by_m.get(m, []))) return round(ok / len(ordinals), 3) def score_pair(gt_events, omr_events) -> dict: # ── legacy pitch-sequence LCS — the historic headline & regression guard ── gt_pitches = [e[2] for e in gt_events] omr_pitches = [e[2] for e in omr_events] common = lcs_length(gt_pitches, omr_pitches) precision = common / len(omr_pitches) if omr_pitches else 0.0 recall = common / len(gt_pitches) if gt_pitches else 0.0 f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 # Interval (pitch-delta) LCS is transposition-invariant: contour F1 far above # note F1 diagnoses a SYSTEMATIC pitch-reference error (clef/octave misread) # rather than note-level garbage. gt_iv = [b - a for a, b in zip(gt_pitches, gt_pitches[1:])] omr_iv = [b - a for a, b in zip(omr_pitches, omr_pitches[1:])] iv_common = lcs_length(gt_iv, omr_iv) contour_f1 = 2 * iv_common / (len(gt_iv) + len(omr_iv)) if gt_iv and omr_iv else 0.0 # ── strict: (onset, midi, duration) with tolerance — the new headline ── pairs = lcs_match(gt_events, omr_events, _strict_eq) strict = len(pairs) strict_precision = strict / len(omr_events) if omr_events else 0.0 strict_recall = strict / len(gt_events) if gt_events else 0.0 strict_f1 = ( 2 * strict_precision * strict_recall / (strict_precision + strict_recall) if strict_precision + strict_recall else 0.0 ) # Over strict-matched pairs only: hand/staff assignment, and spelling. # Step equality suffices for spelling: with MIDI already equal, an equal # step forces equal octave+alter for any sane alter range. staff_ok = sum(1 for gi, oj in pairs if gt_events[gi][4] == omr_events[oj][4]) spell_ok = sum(1 for gi, oj in pairs if gt_events[gi][5] == omr_events[oj][5]) return { "gt_notes": len(gt_pitches), "omr_notes": len(omr_pitches), "matched": common, "note_precision": round(precision, 3), "note_recall": round(recall, 3), "note_f1": round(f1, 3), "contour_f1": round(contour_f1, 3), "strict_matched": strict, "strict_precision": round(strict_precision, 3), "strict_recall": round(strict_recall, 3), "strict_f1": round(strict_f1, 3), "staff_acc": round(staff_ok / strict, 3) if strict else None, "spell_acc": round(spell_ok / strict, 3) if strict else None, "measure_exact": measure_exact_rate(gt_events, omr_events), } # ── Pipeline ────────────────────────────────────────────────────────────────── # ── Command line ────────────────────────────────────────────────────────────── def score_files(gt_path: Path, omr_path: Path) -> dict: gt = note_events(Path(gt_path).read_text(errors="ignore")) omr = note_events(Path(omr_path).read_text(errors="ignore")) return score_pair(gt, omr) def score_piece(suite_dir: Path, engine: str, piece: dict, all_pages: bool = False) -> float | None: """Concatenate the engine's per-page transcriptions, offsetting each page by the measures that came before it, and score against the ground truth truncated to the measures actually scanned. Returns None if the engine produced nothing for this piece. By default only page one of each piece is scored, because that is the only page every engine received: the photographed suite was staged to the competitor runner as page one per piece, so scoring their later pages would charge them for ground truth they were never shown. Pass all_pages=True to score every page present, which is a fair comparison only between engines that were given every page. Note that in that mode a page an engine has no file for still contributes its measures to the ground-truth window, so a missing page is charged in full — that is precisely the defect page-one scoring exists to avoid, kept here so the earlier published figures remain reproducible.""" slug = piece["slug"] piece_dir = suite_dir / engine / slug pages = piece["pages"] if all_pages else piece["pages"][:1] events, scanned, found = [], 0, 0 for page in pages: xml = piece_dir / f"p{page['page']}.musicxml" if xml.exists(): found += 1 for ev in note_events(xml.read_text(errors="ignore")): events.append((scanned + ev[0],) + tuple(ev[1:])) scanned += page["measures"] if not found: return None gt_xml = (suite_dir / "ground-truth" / f"{slug}.musicxml").read_text(errors="ignore") return score_pair(note_events(gt_xml, max_measures=scanned), events)["strict_f1"] def score_bundle(root: Path, all_pages: bool = False) -> int: """Re-score the published bundle and print the table on the benchmark page.""" manifest = json.loads((root / "manifest.json").read_text()) print("scoring", "every page present" if all_pages else "page one of each piece") for suite, spec in manifest["suites"].items(): print(f"\n== {suite}: {spec['description']} ==") for engine in ("sheetly", "audiveris", "oemer"): if not (root / suite / engine).is_dir(): continue scores = [score_piece(root / suite, engine, p, all_pages) for p in spec["pieces"]] produced = [s for s in scores if s is not None] # A piece an engine transcribed nothing for scores zero, not "excluded". mean = sum(produced) / len(scores) if scores else 0.0 def med(v): v = sorted(v) return 0.0 if not v else (v[len(v) // 2] if len(v) % 2 else (v[len(v) // 2 - 1] + v[len(v) // 2]) / 2) # Two populations, because they answer different questions: the mean # counts a piece an engine could not transcribe as zero, so the median # is shown both ways rather than silently over one of them. print(f" {engine:<10} strict F1 {mean * 100:6.2f}% " f"median(all) {med([s or 0.0 for s in scores]) * 100:5.1f}% " f"median(transcribed) {med(produced) * 100:5.1f}% " f"({len(produced)} of {len(scores)} pieces transcribed)") return 0 def main() -> int: args = [a for a in sys.argv[1:] if a != "--all-pages"] all_pages = "--all-pages" in sys.argv[1:] if len(args) == 1: return score_bundle(Path(args[0]), all_pages) if len(args) == 2: print(json.dumps(score_files(Path(args[0]), Path(args[1])), indent=1)) return 0 print(__doc__) return 1 if __name__ == "__main__": sys.exit(main())