""" model_exploitation.py Model exploitation vs. equivalence-class size. (Benchmark v1.3) THE QUESTION. theory/ai/world-models-and-vla.md reads model-based RL's best-documented failure mode — the policy exploiting errors of its own learned world model — as the equivalence-class problem in action: a learned model consistent with all training traces is ONE member of the class, and an optimizer planning inside it will preferentially route through the regions where the chosen member diverges from reality. That note flagged the measurable bridge: does model-exploitation rate track equivalence-class size? This module measures it, in the smallest honest setting. THE SETUP. The world is an elementary CA with a hidden rule. The agent's world model is the rule table reconstructed from training data — except that a controlled number u of the 8 neighborhoods was never observed. The consistent-generator class has size exactly 2^u; the agent's model fills the u unseen bits with a guess (one fixed member of the class) and treats the guess as fact, because nothing in the trace marks it as a guess. The agent then PLANS: given a start row, it samples candidate interventions (flip k cells), imagines each candidate's rollout under its model, and executes the argmax-imagined-reward candidate in the real world (reward: number of live cells at horizon T). TWO PREDICTIONS — AND ONE HONEST REVISION FROM THE FIRST RUN. 1. EXPLOITATION GAP. imagined(chosen) − real(chosen) should grow with u. Mechanism: the optimizer's curse (Smith & Winkler 2006) over model error — argmaxing across candidates whose imagined values carry guess-induced error selects positively-biased error. At u = 0 (class size 1) the gap must vanish: the model IS the world. 2. THE SELECTION DECOMPOSITION. The curse predicts a specific shape: across ALL candidates the gap should average near zero (the guesses are symmetric — the model is not biased, only wrong), while the CHOSEN candidate's gap is positive. The wedge between chosen-gap and candidate-mean-gap is the optimizer's curse, isolated. REVISED AFTER RUN 1: the naive "divergence-seeking" prediction — that the chosen plan would *visit* unseen neighborhoods more often than the average candidate — is NOT borne out (usage rates are statistically identical). The refinement matters: the optimizer does not need to steer into the fantasy regions more often; at equal usage it selects the candidates for which the guesses happen to flatter the objective. Exploitation is selection over guess-outcomes, not navigation toward guess-territory — at least in this open-loop setting. The usage metric is retained and reported as the null it produced. WHAT THIS DOES NOT SHOW. A toy bridge, not model-based RL: no learned dynamics network, no closed-loop policy, one-step open-loop planning. The value is that here the equivalence class is EXACT and the mechanism is isolated — if the effect exists anywhere, it must exist here, and its shape here is the null model for the industrial case. Usage:: python model_exploitation.py # console summary python model_exploitation.py --save # also write the figure Related: - theory/ai/world-models-and-vla.md (the flag this answers) - inverse_benchmark.py (v0: the equivalence class, measured) - intervention_experiment.py (v1.1: queries collapse the class) """ from __future__ import annotations import argparse from pathlib import Path import numpy as np # ══════════════════════ CA world ════════════════════════════════════ def rule_bits(rule: int) -> np.ndarray: return np.array([(rule >> k) & 1 for k in range(8)], dtype=np.uint8) def step(row: np.ndarray, bits: np.ndarray) -> np.ndarray: r = row.astype(int) nb = (np.roll(r, 1) << 2) | (r << 1) | np.roll(r, -1) return bits[nb] def rollout_reward(row: np.ndarray, bits: np.ndarray, T: int) -> float: """Reward = live-cell count at horizon (fraction of width).""" r = row.copy() for _ in range(T): r = step(r, bits) return float(r.sum()) / r.size def unseen_usage(row: np.ndarray, bits: np.ndarray, unseen: np.ndarray, T: int) -> float: """Fraction of neighborhood lookups hitting UNSEEN patterns during a rollout under the given table — how much the trajectory leans on guesses.""" r = row.copy() hits = total = 0 for _ in range(T): ri = r.astype(int) nb = (np.roll(ri, 1) << 2) | (ri << 1) | np.roll(ri, -1) hits += int(np.isin(nb, unseen).sum()) total += nb.size r = bits[nb] return hits / total # ══════════════════════ One planning episode ════════════════════════ def episode(true_rule: int, u: int, W: int = 60, T: int = 12, k: int = 4, n_candidates: int = 150, seed: int = 0) -> dict: """Plan inside a class-member model with u guessed bits; execute in reality. Returns imagined/real reward of the chosen plan, the exploitation gap, and unseen-usage of the chosen plan vs. the candidate average (both measured on IMAGINED rollouts — the planner's own view). """ rng = np.random.default_rng(seed) true_bits = rule_bits(true_rule) # Controlled coverage: u neighborhoods were never observed. unseen = rng.choice(8, size=u, replace=False) if u > 0 else np.array([], dtype=int) model_bits = true_bits.copy() if u > 0: # one fixed member of the 2^u consistent class model_bits[unseen] = rng.integers(0, 2, size=u, dtype=np.uint8) row0 = (rng.random(W) < 0.5).astype(np.uint8) # Candidate interventions: flip k cells. cand_positions = [rng.choice(W, size=k, replace=False) for _ in range(n_candidates)] imagined = np.empty(n_candidates) usage = np.empty(n_candidates) for i, pos in enumerate(cand_positions): r = row0.copy() r[pos] ^= 1 imagined[i] = rollout_reward(r, model_bits, T) usage[i] = unseen_usage(r, model_bits, unseen, T) if u > 0 else 0.0 # Real outcome of EVERY candidate — enables the selection decomposition. real_all = np.empty(n_candidates) for i, pos in enumerate(cand_positions): r = row0.copy() r[pos] ^= 1 real_all[i] = rollout_reward(r, true_bits, T) best = int(np.argmax(imagined)) gaps_all = imagined - real_all return { "gap_chosen": float(gaps_all[best]), "gap_cand_mean": float(gaps_all.mean()), "usage_chosen": float(usage[best]), "usage_mean": float(usage.mean()), } # ══════════════════════ Sweep ═══════════════════════════════════════ def run_suite(us=(0, 1, 2, 3, 4, 5), n_rules: int = 14, n_masks: int = 3, n_eps: int = 5, seed: int = 0) -> dict: rng = np.random.default_rng(seed) rules = rng.integers(1, 255, size=n_rules) # skip trivial 0/255 out = {"u": list(us), "gap_chosen": [], "gap_chosen_se": [], "gap_cand": [], "gap_cand_se": [], "usage_chosen": [], "usage_mean": []} for u in us: gc, gm, uc, um = [], [], [], [] s = 0 for rule in rules: for m in range(n_masks): for e in range(n_eps): s += 1 r = episode(int(rule), u, seed=seed * 100000 + s) gc.append(r["gap_chosen"]) gm.append(r["gap_cand_mean"]) uc.append(r["usage_chosen"]) um.append(r["usage_mean"]) gc, gm = np.array(gc), np.array(gm) out["gap_chosen"].append(float(gc.mean())) out["gap_chosen_se"].append(float(gc.std() / np.sqrt(gc.size))) out["gap_cand"].append(float(gm.mean())) out["gap_cand_se"].append(float(gm.std() / np.sqrt(gm.size))) out["usage_chosen"].append(float(np.mean(uc))) out["usage_mean"].append(float(np.mean(um))) return out def print_summary(res: dict) -> None: print("=" * 70) print(" MODEL EXPLOITATION vs EQUIVALENCE-CLASS SIZE (benchmark v1.3)") print("=" * 70) print("\n u = unseen neighborhoods; consistent-generator class size = 2^u") print(" gap = imagined − real reward (fraction of W)\n") print(f" {'u':>2} {'class':>6} {'gap(chosen)':>16} {'gap(cand mean)':>17} " f"{'curse wedge':>12} {'usage ch/mean':>14}") for i, u in enumerate(res["u"]): wedge = res["gap_chosen"][i] - res["gap_cand"][i] print(f" {u:>2} {2**u:>6} {res['gap_chosen'][i]:>10.4f} ±{res['gap_chosen_se'][i]:.4f}" f" {res['gap_cand'][i]:>10.4f} ±{res['gap_cand_se'][i]:.4f}" f" {wedge:>12.4f} {res['usage_chosen'][i]:>7.3f}/{res['usage_mean'][i]:.3f}") print("\nReading: at u=0 both gaps are zero — the model IS the world. As the") print("class grows, the candidate-average gap stays near zero (the guesses") print("are wrong but unbiased) while the CHOSEN plan's gap goes positive:") print("the wedge between the two curves is the optimizer's curse, isolated —") print("argmaxing over guess-contaminated values selects the flattering") print("errors. Note the honest null: unseen-usage of the chosen plan equals") print("the candidate mean. The optimizer does not steer INTO the fantasy") print("regions; at equal exposure it selects the fantasies that pay.") print("Model exploitation is the equivalence class, priced by an argmax.") # ══════════════════════ Figure ══════════════════════════════════════ def _repo_lab_tools() -> Path: return Path(__file__).resolve().parents[2] / "tools" def figure(res: dict, outdir: Path) -> Path: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt us = np.array(res["u"]) fig, (axA, axB) = plt.subplots(1, 2, figsize=(12.5, 5.0)) axA.errorbar(us, res["gap_chosen"], yerr=res["gap_chosen_se"], fmt="o-", color="#d62728", lw=2, capsize=3, label="chosen plan (argmax)") axA.errorbar(us, res["gap_cand"], yerr=res["gap_cand_se"], fmt="s--", color="#7f7f7f", lw=2, capsize=3, label="candidate average") axA.fill_between(us, res["gap_cand"], res["gap_chosen"], color="#d62728", alpha=0.12, label="the optimizer's curse") axA.axhline(0, ls=":", color="gray") axA.set_xlabel("unseen neighborhoods u (class size $2^u$)") axA.set_ylabel("gap (imagined − real reward)") axA.set_title("(a) The selection decomposition: guesses are unbiased on\n" "average — the argmax selects the flattering errors") axA.legend(fontsize=9) axA.grid(alpha=0.3) axB.plot(us, res["usage_chosen"], "s-", color="#9467bd", lw=2, label="chosen plan") axB.plot(us, res["usage_mean"], "o--", color="#7f7f7f", lw=2, label="candidate average") axB.set_xlabel("unseen neighborhoods u") axB.set_ylabel("unseen-neighborhood usage in imagined rollout") axB.set_title("(b) The honest null: no divergence-seeking by usage —\n" "exploitation is selection over guesses at equal exposure") axB.legend() axB.grid(alpha=0.3) fig.suptitle("Model exploitation vs. equivalence-class size (benchmark v1.3) — " "the world-models bridge, measured", fontsize=12) fig.tight_layout(rect=[0, 0, 1, 0.94]) out = outdir / "inverse_benchmark_model_exploitation.png" fig.savefig(out, dpi=200, bbox_inches="tight") plt.close(fig) return out def main() -> None: parser = argparse.ArgumentParser( description="Model exploitation vs equivalence-class size (v1.3).") parser.add_argument("--save", action="store_true") parser.add_argument("-o", "--output", type=str, default=None) args = parser.parse_args() res = run_suite() print_summary(res) if args.save: outdir = Path(args.output) if args.output else _repo_lab_tools() outdir.mkdir(parents=True, exist_ok=True) print(f"\nWriting figure to {outdir} …") print(f" {figure(res, outdir)}") if __name__ == "__main__": main()