Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 103 additions & 0 deletions benchmark/runtime/session_observation.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,109 @@ def observe_codex_session_tail(
}


def collect_context_policy_telemetry(
runtime_root: Path | None = None,
*,
goal_id: str | None = None,
context_dir: Path | None = None,
usage: dict | None = None,
) -> dict:
"""Collect {KeepRaw, FoldToReceipt, EpochReset}, prefetch, quiet-turn, and cold/warm KV telemetry."""
operator_counts = {"KeepRaw": 0, "FoldToReceipt": 0, "EpochReset": 0}
quiet_turn_skips = 0
prefetch_latency_ms = 0
prefetch_bytes = 0
prefetch_bytes_saved = 0
round_trips_saved = 0
seen_receipts: set[str] = set()

def _consume_prefetch(entry: object) -> None:
nonlocal prefetch_latency_ms, prefetch_bytes, prefetch_bytes_saved, round_trips_saved
if not isinstance(entry, dict):
return
receipt_id = entry.get("receipt_sha256")
if isinstance(receipt_id, str) and receipt_id:
if receipt_id in seen_receipts:
return
seen_receipts.add(receipt_id)
op = entry.get("operator")
if op in operator_counts:
operator_counts[op] += 1
if isinstance(entry.get("latency_ms"), int) and entry["latency_ms"] >= 0:
prefetch_latency_ms += entry["latency_ms"]
if isinstance(entry.get("retained_bytes"), int) and entry["retained_bytes"] >= 0:
prefetch_bytes += entry["retained_bytes"]
if isinstance(entry.get("bytes_saved"), int) and entry["bytes_saved"] >= 0:
prefetch_bytes_saved += entry["bytes_saved"]
if isinstance(entry.get("round_trips_saved"), int) and entry["round_trips_saved"] >= 0:
round_trips_saved += entry["round_trips_saved"]

def _consume_quiet(entry: object) -> None:
nonlocal quiet_turn_skips
if isinstance(entry, dict) and entry.get("skipped") is True:
quiet_turn_skips += 1

if runtime_root is not None and goal_id:
turns_dir = Path(runtime_root) / "goals" / goal_id / "turns"
if turns_dir.is_dir():
for path in sorted(turns_dir.glob("*.json")):
try:
journal = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if isinstance(journal, dict):
_consume_prefetch(journal.get("epoch_prefetch"))
_consume_quiet(journal.get("quiet_turn"))

if context_dir is not None:
session_file = Path(context_dir) / "session.json"
if session_file.is_file():
try:
session_data = json.loads(session_file.read_text(encoding="utf-8"))
except (OSError, ValueError):
session_data = {}
if isinstance(session_data, dict):
_consume_prefetch(session_data.get("epoch_prefetch"))
_consume_quiet(session_data.get("quiet_turn"))

cold = {"requests": 0, "uncached_prefix_kv_tokens": 0, "cached_prefix_kv_tokens": 0}
warm = {"requests": 0, "uncached_prefix_kv_tokens": 0, "cached_prefix_kv_tokens": 0}
if isinstance(usage, dict):
coverage = usage.get("request_coverage")
requests = coverage.get("requests") if isinstance(coverage, dict) else None
if isinstance(requests, list):
for req in requests:
if not isinstance(req, dict):
continue
inp = req.get("input_tokens")
cached = req.get("cached_input_tokens")
if not isinstance(inp, int) or not isinstance(cached, int) or inp < cached:
continue
uncached = inp - cached
bucket = cold if (cached == 0 or req.get("cache_regime") == "cold") else warm
bucket["requests"] += 1
bucket["uncached_prefix_kv_tokens"] += uncached
bucket["cached_prefix_kv_tokens"] += cached
elif isinstance(usage.get("input_tokens"), int) and isinstance(usage.get("cached_input_tokens"), int):
cached = usage["cached_input_tokens"]
uncached = max(0, usage["input_tokens"] - cached)
bucket = cold if cached == 0 else warm
bucket["requests"] = int(usage.get("completed_turns") or 1)
bucket["uncached_prefix_kv_tokens"] = uncached
bucket["cached_prefix_kv_tokens"] = cached

return {
"schema_version": "loopx_context_policy_telemetry_v0",
"operator_counts": operator_counts,
"quiet_turn_skips": quiet_turn_skips,
"prefetch_latency_ms": prefetch_latency_ms,
"prefetch_bytes": prefetch_bytes,
"prefetch_bytes_saved": prefetch_bytes_saved,
"round_trips_saved": round_trips_saved,
"prefix_kv": {"cold": cold, "warm": warm},
}


def main(argv: list[str] | None = None) -> int:
"""Print one bounded observation for an explicitly selected rollout."""
parser = argparse.ArgumentParser(
Expand Down
31 changes: 21 additions & 10 deletions loopx/capabilities/benchmark_toolkit/experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
build_benchmark_factorial_contrasts,
build_benchmark_metric_delta,
)
from .token_capacity import (
CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS,
normalize_operator_counts,
)

Payload = dict[str, object]
PayloadView = Mapping[str, object]
Expand Down Expand Up @@ -204,25 +208,26 @@ def _countability(value: object) -> dict[str, bool]:
return result


def _effort(value: object) -> dict[str, float | int]:
def _effort(value: object) -> Payload:
if value in (None, {}):
return {}
if not isinstance(value, Mapping):
raise TypeError("effort must be an object")
view = _as_map(value)
int_fields = (
"duration_ms",
"agent_steps",
"goal_turns",
"token_count",
*CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS,
)
_reject_unknown_fields(
view,
allowed={
"duration_ms",
"agent_steps",
"goal_turns",
"token_count",
"estimated_cost_usd",
},
allowed={*int_fields, "estimated_cost_usd", "operator_counts"},
field="effort",
)
result: dict[str, float | int] = {}
for field in ("duration_ms", "agent_steps", "goal_turns", "token_count"):
result: Payload = {}
for field in int_fields:
if field in view:
result[field] = _non_negative_int(view[field], field=f"effort.{field}")
if "estimated_cost_usd" in view:
Expand All @@ -231,6 +236,12 @@ def _effort(value: object) -> dict[str, float | int]:
field="effort.estimated_cost_usd",
non_negative=True,
)
if "operator_counts" in view:
result["operator_counts"] = normalize_operator_counts(
view["operator_counts"],
field="effort.operator_counts",
validate_int=lambda v, f: _non_negative_int(v, field=f),
)
return result


Expand Down
2 changes: 2 additions & 0 deletions loopx/capabilities/benchmark_toolkit/study_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION,
build_benchmark_runtime_observation,
)
from .token_capacity import aggregate_context_policy_effort

Payload = dict[str, object]
PayloadView = Mapping[str, object]
Expand Down Expand Up @@ -1007,6 +1008,7 @@ def _effort_aggregate(rows: Sequence[PayloadView]) -> Payload:
"mean": statistics.fmean(values) if values else None,
"median": statistics.median(values) if values else None,
}
output.update(aggregate_context_policy_effort(rows))
return output


Expand Down
149 changes: 148 additions & 1 deletion loopx/capabilities/benchmark_toolkit/token_capacity.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from __future__ import annotations

import json
from collections.abc import Mapping
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime, timedelta
from pathlib import Path
import statistics
from typing import Any

from ...domain_state import default_domain_state_file_path
Expand Down Expand Up @@ -38,6 +39,73 @@
"evaluator_reserve_tokens_per_minute",
"cooldown_seconds",
}
_CACHE_REGIMES = {"cold", "warm"}
_CONTEXT_POLICY_OPERATORS = ("KeepRaw", "FoldToReceipt", "EpochReset")
CONTEXT_POLICY_OPERATORS = _CONTEXT_POLICY_OPERATORS
CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS = (
"cached_prefix_kv_tokens",
"uncached_prefix_kv_tokens",
"cold_uncached_prefix_kv_tokens",
"cold_cached_prefix_kv_tokens",
"warm_uncached_prefix_kv_tokens",
"warm_cached_prefix_kv_tokens",
"prefetch_latency_ms",
"prefetch_bytes",
"prefetch_bytes_saved",
"quiet_turn_skips",
)


def normalize_operator_counts(
value: object,
*,
field: str,
validate_int: Callable[[object, str], int],
) -> dict[str, int]:
"""Validate optional context-policy operator counts cleanly across benchmark contracts."""
if not isinstance(value, Mapping):
raise TypeError(f"{field} must be an object")
_reject_unknown_fields(
value,
allowed=set(_CONTEXT_POLICY_OPERATORS),
field=field,
)
return {
op: validate_int(value[op], f"{field}.{op}")
for op in _CONTEXT_POLICY_OPERATORS
if op in value
}


def aggregate_context_policy_effort(rows: Sequence[Mapping[str, object]]) -> dict[str, object]:
"""Aggregate optional context-policy effort telemetry and operator counts when present."""
output: dict[str, object] = {}
for field in CONTEXT_POLICY_EFFORT_TELEMETRY_FIELDS:
values = [
float(val)
for row in rows
if isinstance(effort := row.get("effort"), Mapping)
and isinstance(val := effort.get(field), (int, float))
and not isinstance(val, bool)
]
if values:
output[field] = {
"denominator": len(values),
"mean": statistics.fmean(values),
"median": statistics.median(values),
}
operator_rows = [
effort["operator_counts"]
for row in rows
if isinstance(effort := row.get("effort"), Mapping)
and isinstance(effort.get("operator_counts"), Mapping)
]
if operator_rows:
output["operator_counts"] = {
op: sum(int(ops.get(op, 0)) for ops in operator_rows)
for op in _CONTEXT_POLICY_OPERATORS
}
return output
_SAMPLE_FIELDS = {
"schema_version",
"sample_id",
Expand All @@ -47,6 +115,12 @@
"uncached_input_tokens",
"cached_input_tokens",
"output_tokens",
"cache_regime",
"operator_counts",
"quiet_turn_skips",
"prefetch_latency_ms",
"prefetch_bytes",
"prefetch_bytes_saved",
}
_RESERVATION_FIELDS = {
"run_id",
Expand Down Expand Up @@ -196,6 +270,27 @@ def normalize_benchmark_token_usage_sample(
raise ValueError("unmetered samples cannot claim token counts")
elif sample["request_count"] == 0 and token_total != 0:
raise ValueError("token counts require at least one observed request")
if "cache_regime" in value and value["cache_regime"] is not None:
regime = _token(value.get("cache_regime"), field="cache_regime")
if regime not in _CACHE_REGIMES:
raise ValueError("cache_regime must be cold or warm")
sample["cache_regime"] = regime
if "operator_counts" in value and value["operator_counts"] is not None:
sample["operator_counts"] = normalize_operator_counts(
value["operator_counts"],
field="operator_counts",
validate_int=lambda v, f: _bounded_token_int(v, field=f),
)
for telemetry_field in (
"quiet_turn_skips",
"prefetch_latency_ms",
"prefetch_bytes",
"prefetch_bytes_saved",
):
if telemetry_field in value and value[telemetry_field] is not None:
sample[telemetry_field] = _bounded_token_int(
value[telemetry_field], field=telemetry_field
)
return sample


Expand Down Expand Up @@ -350,6 +445,57 @@ def _usage_projection(
+ totals["cached_input_tokens"]
+ totals["output_tokens"]
)
telemetry: dict[str, Any] = {}
has_regime = any("cache_regime" in item for item in window)
if has_regime:
prefix_kv: dict[str, dict[str, int]] = {}
for regime in ("cold", "warm"):
subset = [item for item in window if item.get("cache_regime") == regime]
prefix_kv[regime] = {
"request_count": sum(item["request_count"] for item in subset),
"uncached_prefix_kv_tokens": sum(
item["uncached_input_tokens"] for item in subset
),
"cached_prefix_kv_tokens": sum(
item["cached_input_tokens"] for item in subset
),
}
telemetry["prefix_kv"] = prefix_kv
has_policy = any(
any(
key in item
for key in (
"operator_counts",
"quiet_turn_skips",
"prefetch_latency_ms",
"prefetch_bytes",
"prefetch_bytes_saved",
)
)
for item in window
)
if has_policy:
op_counts = {
op: sum(
int((item.get("operator_counts") or {}).get(op, 0)) for item in window
)
for op in _CONTEXT_POLICY_OPERATORS
}
telemetry["context_policy"] = {
"operator_counts": op_counts,
"quiet_turn_skips": sum(
int(item.get("quiet_turn_skips", 0)) for item in window
),
"prefetch_latency_ms": sum(
int(item.get("prefetch_latency_ms", 0)) for item in window
),
"prefetch_bytes": sum(
int(item.get("prefetch_bytes", 0)) for item in window
),
"prefetch_bytes_saved": sum(
int(item.get("prefetch_bytes_saved", 0)) for item in window
),
}
return {
"window_seconds": int(_WINDOW.total_seconds()),
"window_started_at": (datetime.fromisoformat(observed_at) - _WINDOW)
Expand All @@ -360,6 +506,7 @@ def _usage_projection(
"reason_codes": reason_codes,
"sample_count": len(window),
**totals,
**telemetry,
}


Expand Down
11 changes: 11 additions & 0 deletions loopx/cli_commands/turn_registration.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,17 @@ def register_turn_commands(
action="store_true",
help="Invoke the host and commit validated writeback/quota effects.",
)
run_once.add_argument(
"--epoch-prefetch-json",
type=json.loads,
default=None,
help="Opt-in Stage-then-Swap EpochReset prefetch configuration JSON (default-off).",
)
run_once.add_argument(
"--quiet-turn",
action="store_true",
help="Opt-in Quiet-Turn zero-tax skip metadata evaluation when Turn state is unchanged (default-off).",
)
run_once.add_argument("--limit", type=int, default=5)


Expand Down
Loading