`colab.instances()` each time -- both the url and the token expire in about an hour -- and the kernel's
plugin is not importable from a subprocess.
"""
import argparse
import collections
import json
import re
import sys
import urllib.parse
import urllib.request
WINDOW = re.compile(
r"Epoch (\d)/2 \[(\d+)/322\] step=(\d+).*?pref_loss=([\d.]+) margin=([+-][\d.]+)"
)
# the epoch-1 stretch where preference margins collapsed to near-ties (w198-w290)
COLLAPSED = (198, 290)
def contents(host, token, path, text=False, content=True):
q = urllib.parse.urlencode({
"colab-runtime-proxy-token": token,
**({"format": "text"} if text else {}),
with urllib.request.urlopen(req, timeout=40) as r:
return json.loads(r.read())
def census(text):
"""Per-epoch preference-margin health, and the epoch-over-epoch comparison that matters.
A positive margin alone is not evidence of learning: `+0.02` satisfies the sign and decides
nothing. Track magnitude, and compare the *same* window across epochs, since the window order is
deterministic (longest first).
"""
per = collections.defaultdict(list)
by = collections.defaultdict(dict)
for line in text.splitlines():
m = WINDOW.search(line)
if not m:
continue
epoch, win, _step, _pl, margin = m.groups()
per[int(epoch)].append(float(margin))
by[int(win)][int(epoch)] = float(margin)
for epoch in sorted(per):
v = sorted(per[epoch])
near = sum(1 for x in v if x < 5.0)
print(f" epoch {epoch}: n={len(v):3d} median {v[len(v) // 2]:+7.2f} | "
f"margin<5 {near / len(v):4.0%} | violated {sum(1 for x in v if x <= 0)}")
both = [(w, v[1], v[2]) for w, v in sorted(by.items()) if 1 in v and 2 in v]
if not both:
print(" no window seen in both epochs yet")
return
drops = [(w, a, b) for w, a, b in both if b - a < -5.0]
lo, hi = COLLAPSED
stretch = [(w, a, b) for w, a, b in both if lo <= w <= hi]
print(f" windows in both epochs: {len(both)} | margin fell >5: {len(drops)}"
+ (f" -> {[(f'w{w}', f'{a:+.1f}', f'{b:+.1f}') for w, a, b in drops]}" if drops else ""))
if stretch:
print(f" collapsed stretch w{lo}-w{hi} revisited: {len(stretch)} window(s) -> "
+ ", ".join(f"w{w} {a:+.2f}->{b:+.2f}" for w, a, b in stretch))
elif max(w for w, _, _ in both) < lo:
e2_at = max(w for w, v in by.items() if 2 in v)
print(f" collapsed stretch w{lo}-{hi} not yet revisited (epoch 2 is at w{e2_at}): "
"answers whether the anti-slop DPO regime actually moved those pairs")
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--host", required=True)
ap.add_argument("--token", required=True)
ap.add_argument("--lines", type=int, default=4)
ap.add_argument("--dir", default="content/swift15/checkpoints/agent_persona_lora_v1")
ap.add_argument("--census", action="store_true", help="preference-margin health per epoch")
a = ap.parse_args(argv)
log = contents(a.host, a.token, f"{a.dir}/train.log", text=True)
lines = (log.get("content") or "").splitlines()
print(f"train.log {log.get('size', 0):,} B / {len(lines)} lines")
for line in lines[-a.lines:]:
print(" " + line)
if a.census:
census("\n".join(lines))
for item in sorted((contents(a.host, a.token, a.dir).get("content") or []), key=lambda x: x["name"]):
if item["name"].endswith((".pt", ".bin", ".pid")):
print(f" {item['name']:24s} {item.get('size', 0):>12,} {item['last_modified'][11:19]}")
return 0