{ "schema_version": "improvement-system.v1", "revised_at": "2026-08-24", "principle": "Exactly four closed improvement loops share one experiment control plane.", "loops": [ { "id": "activation", "name": "Qualified visit → first durable habit", "promise": "Help the right agent reach a useful first brief quickly, then choose WebBrief again.", "input": "A qualified visitor or agent with a current-information task.", "steps": [ "Observe the acquisition surface and successful install/setup action.", "Join authentication to the first successful brief using a pseudonymous journey identifier.", "Measure time to first brief and a second successful brief within seven days.", "Turn the largest measured drop-off into one bounded onboarding or message experiment." ], "output": "More users who experience the core value and form a repeatable research habit.", "reinvestment": "Retained users create better activation cohorts and clearer evidence about which promise and setup path work.", "primaryMetric": { "id": "activation.first_brief_10m_rate", "name": "First brief within 10 minutes", "definition": "Unique authenticated actors completing a first brief within 10 minutes / newly authenticated actors.", "direction": "increase", "sourceOfTruth": "product_warehouse", "unit": "ratio" }, "leadingMetrics": [ { "id": "activation.install_copy_rate", "name": "Install command copy rate", "definition": "Homepage sessions with a successful install-command copy / homepage sessions.", "direction": "increase", "sourceOfTruth": "product_warehouse", "unit": "ratio" }, { "id": "activation.second_brief_7d_rate", "name": "Second brief within seven days", "definition": "Activated actors completing another successful brief within seven days / activated actors.", "direction": "increase", "sourceOfTruth": "product_warehouse", "unit": "ratio" } ], "guardrails": [ { "id": "activation.setup_error_rate", "name": "Setup error rate", "definition": "Setup attempts ending in a typed error / setup attempts.", "direction": "decrease", "sourceOfTruth": "product_warehouse", "unit": "ratio" } ], "cadence": "Review weekly; run at most one activation experiment per overlapping surface.", "automation": "The weekly review ranks statistically supported funnel breaks and prepares an experiment candidate.", "humanGate": "A product owner approves the hypothesis, power calculation, copy, and rollback before exposure begins.", "runtimeHandoff": "The separately deployed auth/CLI runtime must emit authenticated, first-brief, and seven-day return events using the shared event schema." }, { "id": "quality", "name": "Production failure → stronger evaluation", "promise": "Make every correction or material gap improve the next brief without collecting private prose.", "input": "A privacy-safe failure reason, correction signal, material gap, or low-confidence evaluation case.", "steps": [ "Bucket failures by typed reason, query class, source class, and contract check—never raw query text.", "Promote representative, redacted cases into a versioned evaluation corpus.", "Test one retrieval, synthesis, or validation change against the fixed corpus and a production holdout.", "Ship only when groundedness improves without worsening latency, completion, or context size." ], "output": "More complete, faithful briefs and fewer corrective research calls.", "reinvestment": "Higher-quality briefs create cleaner failure signals, more reuse, and a harder evaluation corpus.", "primaryMetric": { "id": "quality.accepted_brief_rate", "name": "Accepted brief rate", "definition": "Successful briefs with no correction, material-gap escalation, or replacement call in the decision window / successful briefs.", "direction": "increase", "sourceOfTruth": "product_warehouse", "unit": "ratio" }, "leadingMetrics": [ { "id": "quality.citation_complete_rate", "name": "Citation-complete rate", "definition": "Briefs labeled complete whose factual claims all resolve to citations / briefs labeled complete.", "direction": "increase", "sourceOfTruth": "evaluation_store", "target": 1, "unit": "ratio" }, { "id": "quality.unnecessary_repeat_call_rate", "name": "Unnecessary repeat-call rate", "definition": "Complete briefs followed by another broad research call for the same task / complete briefs.", "direction": "decrease", "sourceOfTruth": "product_warehouse", "unit": "ratio" } ], "guardrails": [ { "id": "quality.false_complete_rate", "name": "False-complete rate", "definition": "Briefs labeled complete that fail human citation or coverage audit / audited complete briefs.", "direction": "decrease", "sourceOfTruth": "evaluation_store", "target": 0, "unit": "ratio" } ], "cadence": "Triage failure buckets daily; promote cases and review evaluation movement weekly.", "automation": "Typed failures are clustered and ranked by affected successful-brief volume; regressions automatically block rollout.", "humanGate": "A reviewer approves redaction, expected answers, rubric changes, and any model-judged result before it becomes a release gate.", "runtimeHandoff": "The research runtime must emit contract results, correction/deepen reason codes, query class, and evaluation version without query or brief content." }, { "id": "performance", "name": "Trace bottleneck → faster successful brief", "promise": "Make WebBrief faster and cheaper while holding the answer contract constant.", "input": "Stage-level traces for successful, partial, cached, failed, and timed-out briefs.", "steps": [ "Aggregate planning, search, fetch, synthesis, validation, cache, and provider timing by mode.", "Rank the stage contributing most to p95 latency or cost per successful brief.", "Test bounded concurrency, early stopping, routing, caching, or model changes behind a canary.", "Roll out only when latency or cost improves and quality guardrails remain intact." ], "output": "Faster, more reliable successful briefs at sustainable unit economics.", "reinvestment": "Faster success creates more representative traces and makes agent use feel cheap enough to become habitual.", "primaryMetric": { "id": "performance.successful_brief_p95_ms", "name": "Successful brief p95 latency", "definition": "95th percentile end-to-end milliseconds for non-cached successful quick briefs.", "direction": "decrease", "sourceOfTruth": "trace_store", "target": 12000, "unit": "milliseconds" }, "leadingMetrics": [ { "id": "performance.cost_per_success_usd", "name": "Cost per successful brief", "definition": "Search, fetch, and model cost / successful briefs, segmented by mode and cache state.", "direction": "decrease", "sourceOfTruth": "trace_store", "unit": "usd" }, { "id": "performance.cache_saved_ms_p50", "name": "Median cache time saved", "definition": "Median reported milliseconds avoided by a valid evidence or synthesis cache hit.", "direction": "increase", "sourceOfTruth": "trace_store", "unit": "milliseconds" } ], "guardrails": [ { "id": "performance.partial_or_failed_rate", "name": "Partial or failed rate", "definition": "Partial, failed, or timed-out briefs / started briefs.", "direction": "decrease", "sourceOfTruth": "trace_store", "unit": "ratio" }, { "id": "performance.quality_guardrail_rate", "name": "Quality acceptance rate", "definition": "Accepted briefs in the performance variant / completed briefs in that variant.", "direction": "increase", "sourceOfTruth": "evaluation_store", "unit": "ratio" } ], "cadence": "Watch service health continuously; choose one bottleneck experiment in the weekly review.", "automation": "Trace aggregation identifies the slowest material stage and pauses canaries on timeout, failure, or quality regression.", "humanGate": "An engineer approves provider, cache-retention, cost, and rollback changes before a canary expands.", "runtimeHandoff": "The API and research workers must propagate one W3C trace ID and emit per-stage duration, cache state, cost, mode, and typed outcome." }, { "id": "discovery", "name": "Useful artifact → retained user → better artifact", "promise": "Turn opted-in, high-quality research into genuinely useful public pages that attract the right agent users.", "input": "An opted-in or editorially commissioned brief plus an evidenced search or comparison gap.", "steps": [ "Select only briefs that pass quality, privacy, usefulness, freshness, and duplication gates.", "Publish a human-reviewed canonical artifact with citations, dates, correction history, and a relevant setup path.", "Measure discovery, qualified activation, first brief, and seven-day return by artifact cohort.", "Refresh, consolidate, or retire weak artifacts and reinvest winning topics into the editorial queue." ], "output": "Compounding, citable distribution that creates retained—not merely visiting—users.", "reinvestment": "Retained-user cohorts reveal which topics and artifact formats deserve another high-quality page.", "primaryMetric": { "id": "discovery.retained_activations_per_1000_visits", "name": "Retained activations per 1,000 artifact visits", "definition": "Artifact-attributed users with a first brief and a second brief within seven days / qualified artifact sessions × 1,000.", "direction": "increase", "sourceOfTruth": "product_warehouse", "unit": "count" }, "leadingMetrics": [ { "id": "discovery.qualified_organic_visits", "name": "Qualified organic visits", "definition": "Non-bot organic sessions landing on an artifact and reaching its evidence or setup section.", "direction": "increase", "sourceOfTruth": "search_console", "unit": "count" }, { "id": "discovery.indexed_artifact_rate", "name": "Indexed artifact rate", "definition": "Canonical public artifacts indexed by the target search engine / submitted artifacts.", "direction": "increase", "sourceOfTruth": "search_console", "unit": "ratio" } ], "guardrails": [ { "id": "discovery.stale_or_corrected_rate", "name": "Stale or materially corrected artifact rate", "definition": "Public artifacts failing freshness review or receiving a material correction / reviewed artifacts.", "direction": "decrease", "sourceOfTruth": "evaluation_store", "unit": "ratio" } ], "cadence": "Review discovery weekly; audit freshness monthly and immediately after a material correction.", "automation": "Search and product aggregates rank topic gaps; the publisher updates sitemap metadata and prepares IndexNow submissions after approval.", "humanGate": "An editor confirms publication rights, privacy, source quality, non-duplication, and customer usefulness before indexing.", "runtimeHandoff": "The publishing and auth runtimes must preserve artifact attribution through first and second brief without exposing the original private query." } ], "experiments": [ { "id": "activation-setup-path-v1", "loop_id": "activation", "name": "Match setup path to agent intent", "status": "proposed", "hypothesis": "Giving agent-led visitors one copyable setup path while preserving the direct CLI path will increase first briefs within 10 minutes, not just button clicks.", "primary_metric_id": "activation.first_brief_10m_rate", "guardrail_metric_ids": [ "activation.setup_error_rate" ] }, { "id": "quality-failure-bucket-v1", "loop_id": "quality", "name": "Failure-bucket source planning", "status": "proposed", "hypothesis": "Routing each typed query class through an evidence plan learned from its largest failure bucket will increase accepted briefs and reduce unnecessary repeat calls.", "primary_metric_id": "quality.accepted_brief_rate", "guardrail_metric_ids": [ "quality.false_complete_rate", "quality.unnecessary_repeat_call_rate" ] }, { "id": "performance-bounded-parallelism-v1", "loop_id": "performance", "name": "Bounded parallel evidence path", "status": "proposed", "hypothesis": "Running independent source adapters concurrently and stopping when material facets are covered will reduce quick-brief p95 latency without lowering accepted-brief rate.", "primary_metric_id": "performance.successful_brief_p95_ms", "guardrail_metric_ids": [ "performance.partial_or_failed_rate", "performance.quality_guardrail_rate" ] }, { "id": "discovery-artifact-handoff-v1", "loop_id": "discovery", "name": "Intent-matched artifact handoff", "status": "proposed", "hypothesis": "Giving qualified public-artifact visitors an intent-matched first-brief handoff will create more seven-day retained activations than a generic product CTA.", "primary_metric_id": "discovery.retained_activations_per_1000_visits", "guardrail_metric_ids": [ "discovery.stale_or_corrected_rate" ] } ], "contracts": { "event": "/schemas/loop-event.v1.json", "snapshot": "/schemas/loop-snapshot.v1.json" } }