{ "schema": "asym-benchmark-ssot-v1", "updated_at": "2026-10-22T11:12:00Z", "canonical_url": "https://a-i.gi/benchmark/", "trace_url": "https://a-i.gi/benchmark/trace/", "ssot_url": "https://a-i.gi/benchmark/summary.json", "head_to_head_deep_research": { "benchmark_id": "topic-benchmark-v1-2026-10-16", "baseline_name": "Google Gemini Deep Research Max", "cells_evaluated": 8, "jurisdictions": [ "GB", "GI", "US", "SG" ], "sectors": [ "Gambling (Advennt)", "Payments (World Payments Monitor)" ], "verified_points_multiplier_median": 3.3, "verified_points_multiplier_range": "1.3x\u20134.4x", "depth_multiplier_median": 2.1, "arti_provision_precision_pct_range": "72%\u201385%", "baseline_provision_precision_pct_range": "29%\u201346%", "blind_judge": "Claude Sonnet 4.5 (position-randomized Arm A / Arm B)" }, "m3_interpretation_agreement": { "benchmark_id": "m3-blind-panel-2026-10-12", "items_evaluated": 120, "challenger_models": [ { "model": "Claude Opus 5.5", "verdict_agreement_pct": 95.0, "severity_within_one_band_pct": 100.0 }, { "model": "Gemini 4 Argon", "verdict_agreement_pct": 91.7, "severity_within_one_band_pct": 98.3 } ], "substantive_disagreements_rate": "5.0%\u20138.3%" }, "trace_provenance": { "run_id": "run-1", "run_date": "2026-10-19", "consumers_audited": 5, "records_audited": 850, "total_citations": 61174, "b1_retrieved_witness_pct": 77.3, "b2_publisher_match_pct": 99.4, "b2_checked": 42935, "b2_matching": 42683, "b3_t1_allowlist_pct": 96.1, "b6_non_hollow_pct": 100.0, "by_consumer": { "advennt": { "id": "advennt", "name": "Advennt", "domain": "advennt.io", "records": 170, "citations": 2272, "citations_per_record": 13.4, "b1_retrieved_witness_pct": 77.6, "b2_publisher_match_pct": 99.1, "b2_checked": 2272, "b2_matching": 2251, "b3_t1_allowlist_pct": 90.1, "b3_checked": 740, "b3_matching": 667, "b6_non_hollow_pct": 100.0, "review_model": "Expert-Reviewed (Tier 1) + AI-Assembled" }, "crypto": { "id": "crypto", "name": "Crypto", "domain": "cryptoassets.gi", "records": 170, "citations": 2498, "citations_per_record": 14.7, "b1_retrieved_witness_pct": 65.3, "b2_publisher_match_pct": 99.6, "b2_checked": 1610, "b2_matching": 1604, "b3_t1_allowlist_pct": 91.6, "b3_checked": 249, "b3_matching": 228, "b6_non_hollow_pct": 100.0, "review_model": "Multi-Station AI Pipeline + Code Provenance Gates" }, "data-protection": { "id": "data-protection", "name": "Data Protection", "domain": "dataprotection.gi", "records": 170, "citations": 4004, "citations_per_record": 23.6, "b1_retrieved_witness_pct": 80.4, "b2_publisher_match_pct": 99.8, "b2_checked": 2980, "b2_matching": 2975, "b3_t1_allowlist_pct": 94.3, "b3_checked": 300, "b3_matching": 283, "b6_non_hollow_pct": 100.0, "review_model": "Multi-Station AI Pipeline + Code Provenance Gates" }, "financial-integrity": { "id": "financial-integrity", "name": "Financial Integrity", "domain": "sentinel.gi", "records": 170, "citations": 21913, "citations_per_record": 128.9, "b1_retrieved_witness_pct": 74.3, "b2_publisher_match_pct": 99.7, "b2_checked": 14428, "b2_matching": 14386, "b3_t1_allowlist_pct": 97.3, "b3_checked": 3641, "b3_matching": 3541, "b6_non_hollow_pct": 100.0, "review_model": "Multi-Station AI Pipeline + Editor Methodology" }, "world-payments": { "id": "world-payments", "name": "World Payments Monitor", "domain": "payments.gi", "records": 170, "citations": 30487, "citations_per_record": 179.3, "b1_retrieved_witness_pct": 80.0, "b2_publisher_match_pct": 99.2, "b2_checked": 21645, "b2_matching": 21467, "b3_t1_allowlist_pct": 96.3, "b3_checked": 7561, "b3_matching": 7282, "b6_non_hollow_pct": 100.0, "review_model": "Expert-Reviewed (Design Partner) + AI-Assembled" } } }, "question_led_next_bar": { "status": "in_pilot", "description": "Harder bar for question-led answers: deep research on the specific question with a second frontier model challenging it, plus FR-1258 served-surface detail-recall targets (WPM 0.12 -> 0.35, Advennt 0.32 -> 0.50).", "diy_plus_challenge_false_premise_catch": "12/12 (100%)", "served_surface_false_premise_catch": "0/6 (0%)", "diy_plus_challenge_decisive_fact_flips": "2/2 (100%)", "served_surface_decisive_fact_flips": "0/2 (0%)", "fr1258_wpm_detail_recall_target": { "baseline": 0.12, "target": 0.35 }, "fr1258_advennt_detail_recall_target": { "baseline": 0.32, "target": 0.5 } }, "multi_llm_and_hitl_governance": { "ip_boundary_note": "We publish our verification stages, failure gates, and benchmark scores in full. Exact model routing weights, prompt chains, reconciliation heuristics, and calibration harnesses are proprietary Asymmetric Intelligence IP.", "llm_division_of_labour": [ { "stage": "1. Primary-Source Discovery & Retrieval", "what_and_why": "Search-grounded and long-context models locate official statutes, regulatory registers, supervisory guidance, and gazette notices across 170 jurisdictions, capturing candidate provisions with verbatim text spans and primary URLs." }, { "stage": "2. Independent Multi-Model Extraction & Cross-Check", "what_and_why": "Distinct frontier model families independently structure provisions against our fixed domain schema and cross-check whether the cited statutory passage supports the extracted rule\u2014eliminating single-model hallucination and self-agreement bias." }, { "stage": "3. Deterministic Code Verification (Zero-LLM Gate)", "what_and_why": "Non-AI software validates schema conformance, checks HTTP source reachability and verbatim quote-span grounding, enforces Tri-Date separation (last_checked, last_published, last_changed), and routes any ungrounded claim into a visible gap state (not_yet_assessed, stale, confirmed_absent)." }, { "stage": "4. Bounded Synthesis & Change-Diffing", "what_and_why": "Reasoning-optimised models generate role-specific views (MLRO, Legal, Board, Co-Pilot) strictly over the already-verified record atoms\u2014never from open-web recall\u2014so every sentence remains bound to its verified citation." } ], "human_in_the_loop_layers": [ { "layer": "Layer 1 \u2014 Human Legal & Systems Engineering (Development & Gates)", "role": "Lawyers and systems engineers design the domain questions, statutory hierarchies, deterministic rejection gates, and evaluation rubrics. Models cannot alter schemas, relax a gate, or bypass provenance checks." }, { "layer": "Layer 2 \u2014 Human Operational Monitoring, Canary Sweeps & Corrections", "role": "Human operators monitor weekly automated telemetry sweeps, canary benchmark regressions, and source-drift alerts, and adjudicate public withdrawals and corrections in the Corrections Ledger rather than silently overwriting records." }, { "layer": "Layer 3 \u2014 Named Practitioner & Counsel Review (Verified Jurisdictions)", "role": "Where a record carries the Verified mark, a named local practitioner or law firm reviews the structured module layout, can override wording before publication, and signs with firm and individual attribution (capped at <=12% of verified items per firm, with zero cash or pay-to-influence). Machine-reviewed records are explicitly labelled as such." } ], "enterprise_governance_assurances": [ "No customer queries, watchlists, or generated reports are ever used to train foundation models.", "Corpus ingests only public statutory and regulatory instruments (zero personal-data scraping); account data is governed by Gibraltar / UK / EU GDPR.", "Every claim carries item-level primary source links and a logged correction/challenge workflow." ] } }