TemplateProjects

WikiSkill loop

Watch an agent teach itself. The WikiSkill loop (Google Research) starts from a blank slate on a real spreadsheet task: fail, write a note, have a checker verify it, then sit an exam on a new sheet.

check.pyPython

"""Mechanical grader for the leaderboard task. No model judgment anywhere.

Usage:
  python3 check.py <case 1|2|3> <output.xlsx>

Compares the answer range of the produced workbook against the gold answer file
for that case, cell by cell (adapted from SpreadsheetBench's evaluation.py, MIT).
Exit code 0 = PASS, 1 = FAIL (prints per-cell diffs).
"""
import datetime
import json
import os
import re
import sys

import openpyxl

ROOT = os.path.dirname(os.path.abspath(__file__))
TASK_DIR = f"{ROOT}/tasks/leaderboard"
TASK = json.load(open(f"{TASK_DIR}/task.json"))


def _tv(v):
    if isinstance(v, (int, float)):
        return round(float(v), 2)
    if isinstance(v, datetime.time):
        return str(v)[:-3]
    if isinstance(v, datetime.datetime):
        d = v - datetime.datetime(1899, 12, 30)
        return round(d.days + d.seconds / 86400.0, 0)
    if isinstance(v, str):
        try:
            return round(float(v), 2)
        except ValueError:
            return v
    return v


def _cmp(v1, v2):
    v1, v2 = _tv(v1), _tv(v2)
    if (v1 in ("", None)) and (v2 in ("", None)):
        return True
    if type(v1) is not type(v2):
        return False
    return v1 == v2


def cells_of(range_str):
    out = []
    m = re.match(r"^([A-Z]+)(\d+):([A-Z]+)(\d+)$", range_str)
    c1, r1, c2, r2 = m.groups()

    def cn(s):
        n = 0
        for ch in s:
            n = n * 26 + ord(ch) - 64
        return n

    for c in range(cn(c1), cn(c2) + 1):
        name, n = "", c
        while n > 0:
            n, r = divmod(n - 1, 26)
            name = chr(65 + r) + name
        for row in range(int(r1), int(r2) + 1):
            out.append(f"{name}{row}")
    return out


def main():
    case, out_path = sys.argv[1], sys.argv[2]
    tid = TASK["id"]
    gt_path = f"{TASK_DIR}/{case}_{tid}_answer.xlsx"
    if not os.path.exists(out_path):
        print("FAIL: output file not found")
        sys.exit(1)
    wb_gt = openpyxl.load_workbook(gt_path, data_only=True)
    wb_p = openpyxl.load_workbook(out_path, data_only=True)
    sheet = TASK.get("answer_sheet") or wb_gt.sheetnames[0]
    if sheet not in wb_p.sheetnames:
        print(f"FAIL: worksheet '{sheet}' not found in output")
        sys.exit(1)
    ws_gt, ws_p = wb_gt[sheet], wb_p[sheet]
    bad = []
    for cell in cells_of(TASK["answer_position"]):
        if not _cmp(ws_gt[cell].value, ws_p[cell].value):
            bad.append(f"  {cell}: expected {ws_gt[cell].value!r}, got {ws_p[cell].value!r}")
    if bad:
        print(f"FAIL: {len(bad)} wrong cell(s)")
        print("\n".join(bad[:15]))
        sys.exit(1)
    print("PASS: all cells in", TASK["answer_position"], "match")


if __name__ == "__main__":
    main()

This workspace starts from zero: no learned notes, nothing pre-baked. Say "go" and I'll run one full self-improvement loop in front of you: attempt the spreadsheet task and get graded by a script; if I fail, I study why, write myself a note, have a mechanical checker verify the note on a sheet I never learned from, then take a final exam on a third sheet I've never seen. Everything lands as files you can open, and comparison.md ends with the first attempt and the exam side by side. Say "reset" anytime to wipe the learning and run it again.

What’s inside

Starter files

check.pylocked · 2.6 KB

"""Mechanical grader for the leaderboard task. No model judgment anywhere.

Usage:
  python3 check.py <case 1|2|3> <output.xlsx>

Compares the answer range of the produced workbook against the gold answer file
for that case, cell by cell (adapted from SpreadsheetBench's evaluation.py, MIT).
Exit code 0 = PASS, 1 = FAIL (prints per-cell diffs).
"""
import datetime
import json
import os
import re
import sys

import openpyxl

ROOT = os.path.dirname(os.path.abspath(__file__))
TASK_DIR = f"{ROOT}/tasks/leaderboard"
TASK = json.load(open(f"{TASK_DIR}/task.json"))


def _tv(v):
    if isinstance(v, (int, float)):
        return round(float(v), 2)
    if isinstance(v, datetime.time):
        return str(v)[:-3]
    if isinstance(v, datetime.datetime):
        d = v - datetime.datetime(1899, 12, 30)
        return round(d.days + d.seconds / 86400.0, 0)
    if isinstance(v, str):
        try:
            return round(float(v), 2)
        except ValueError:
            return v
    return v


def _cmp(v1, v2):
    v1, v2 = _tv(v1), _tv(v2)
    if (v1 in ("", None)) and (v2 in ("", None)):
        return True
    if type(v1) is not type(v2):
        return False
    return v1 == v2


def cells_of(range_str):
    out = []
    m = re.match(r"^([A-Z]+)(\d+):([A-Z]+)(\d+)$", range_str)
    c1, r1, c2, r2 = m.groups()

    def cn(s):
        n = 0
        for ch in s:
            n = n * 26 + ord(ch) - 64
        return n

    for c in range(cn(c1), cn(c2) + 1):
        name, n = "", c
        while n > 0:
            n, r = divmod(n - 1, 26)
            name = chr(65 + r) + name
        for row in range(int(r1), int(r2) + 1):
            out.append(f"{name}{row}")
    return out


def main():
    case, out_path = sys.argv[1], sys.argv[2]
    tid = TASK["id"]
    gt_path = f"{TASK_DIR}/{case}_{tid}_answer.xlsx"
    if not os.path.exists(out_path):
        print("FAIL: output file not found")
        sys.exit(1)
    wb_gt = openpyxl.load_workbook(gt_path, data_only=True)
    wb_p = openpyxl.load_workbook(out_path, data_only=True)
    sheet = TASK.get("answer_sheet") or wb_gt.sheetnames[0]
    if sheet not in wb_p.sheetnames:
        print(f"FAIL: worksheet '{sheet}' not found in output")
        sys.exit(1)
    ws_gt, ws_p = wb_gt[sheet], wb_p[sheet]
    bad = []
    for cell in cells_of(TASK["answer_position"]):
        if not _cmp(ws_gt[cell].value, ws_p[cell].value):
            bad.append(f"  {cell}: expected {ws_gt[cell].value!r}, got {ws_p[cell].value!r}")
    if bad:
        print(f"FAIL: {len(bad)} wrong cell(s)")
        print("\n".join(bad[:15]))
        sys.exit(1)
    print("PASS: all cells in", TASK["answer_position"], "match")


if __name__ == "__main__":
    main()

tasks/leaderboard/task.jsonlocked · 990 B

{
  "id": "leaderboard",
  "instruction": "The 'Leaderboard' sheet lists our 15 salespeople: column B is each person's monthly sales ($k), column C their monthly sales target ($k), and column D the difference, sales minus target ($k; negative means they missed target). Fill column E with each person's leaderboard points based on the difference in column D: points should vary proportionately between -5 and +5, so the best performer gets +5, the worst performer gets -5, and someone who landed exactly on target (difference 0) gets exactly 0 points, with everyone else scored according to the ratio between those numbers.",
  "spreadsheet_path": "tasks/leaderboard",
  "instruction_type": "Cell-Level Manipulation",
  "answer_position": "E2:E16",
  "answer_sheet": "Leaderboard",
  "data_position": "A1:E16",
  "source": "SpreadsheetBench task 42526 (values verbatim; labels added)",
  "split": {
    "train": [
      1
    ],
    "val": [
      2
    ],
    "test": [
      3
    ]
  }
}

Libraries and docs

AGENTS.mdlocked · 3.2 KB

# WikiSkill workspace: loop protocol

This workspace runs the WikiSkill self-improvement loop (arXiv 2608.27454) from a
blank slate, live, on one real spreadsheet task. Nothing is pre-learned: `wiki/`,
`skills/`, and the logs do not exist until a run creates them.

## The task

`tasks/leaderboard/task.json` holds the instruction. Three spreadsheets exist
(`1_` = learning sheet, `2_` = checking sheet, `3_` = exam sheet), each as an
`_input.xlsx` / `_answer.xlsx` pair with the same layout and different numbers.
Answers are literal values in the range named by `answer_position`.

## Integrity rules (never break these)

- **Never open any `*_answer.xlsx`** and never quote its contents. Grading happens
  ONLY through `python3 check.py <case> <output.xlsx>`, which is mechanical.
- **When solving a sheet (a rollout), use ONLY the task instruction, that sheet's
  `_input.xlsx`, and `skills/*/SKILL.md` if it exists.** Never consult `wiki/`,
  `runs/`, chat history of other attempts, or this file's later sections while
  solving. Do not carry insights between rollouts in your head; if it isn't in a
  skill file, the next rollout doesn't know it.
- Never edit `check.py`, `tasks/`, or this file.
- Do not pre-analyze the task or speculate about likely mistakes in chat before or
  during a run; let the attempts and the grader speak.
- Run Python through `/workspace/.sundial/wikiskill-venv/bin/python3` (falls back
  to `python3`; setup installs openpyxl into it).

## Visitor commands

**"go"** — run one full loop, all in this single reply, narrating each step briefly:
1. **Attempt.** Solve the learning sheet (case 1) with whatever `skills/` contains
   (nothing, on a fresh workspace). Save the solver code as `runs/attempt-1.py`
   and its output sheet, grade with check.py, and report the verdict verbatim.
2. **Learn.** If it failed: study your failed code next to the grader's cell
   diffs. Write the root cause into `wiki/patterns/<name>.md` (create the folder;
   append-only, never delete knowledge) and distill ONE general skill into
   `skills/<slug>/SKILL.md`: procedural rules, no copied answer values.
3. **Verify.** Solve the checking sheet (case 2) twice, fresh rollouts with the
   skill included. Both must pass to keep the skill; otherwise delete the skill,
   keep the wiki page, record the failure evidence in `skill-impact.md`, and go
   back to step 2 (at most 3 tries per "go").
4. **Exam.** Solve the exam sheet (case 3) once with the kept skill; grade it.
5. **Report.** Write `comparison.md` from THIS run's own artifacts: a table of
   every employee with sales, target, difference, correct points, the step-1
   attempt's value, and the exam attempt's value, each marked ✓/✗ (read the gold
   values only via a script, never quote answer files in chat), then both grader
   verdicts verbatim. Honesty rule: report whatever actually happened, including
   a passing first attempt or a failing exam; runs vary.

**"reset"** — return to blank slate: delete `wiki/`, `skills/`, `runs/`,
`comparison.md`, `skill-impact.md`. Confirm with the user first.

**"explain"** — after a run exists, walk through its artifacts (the wiki page, the
skill, comparison.md). Before any run, describe only the task and the procedure
above; never predict what the model will get wrong.

README.md1.9 KB

# Watch an agent teach itself a spreadsheet

This workspace is a live run of **WikiSkill**
([arXiv 2608.27454](https://arxiv.org/abs/2608.27454), Google Research): a loop where
an AI model improves by studying its own failed attempts, writing what it learns into
a note, and keeping the note only after a mechanical checker proves it helps.

Nothing here is pre-learned. Say **"go"** and the loop runs from zero in front of you:

1. The model attempts the task and gets graded by a script.
2. If it failed, it studies the failure and writes a note to itself.
3. A checker tests the note on a spreadsheet the model never learned from; the note
   is kept only if it passes.
4. The model takes a final exam on a third spreadsheet it has never seen, and
   `comparison.md` shows every answer, first attempt vs final, side by side.

Every step lands as files you can open: the model's code in `runs/`, its
self-diagnosis in `wiki/`, the note in `skills/`, the verdicts in `comparison.md`.

## The task

Fifteen salespeople, each with monthly sales, a personal sales target, and the
difference between them. Convert each difference into leaderboard points from -5 to
+5: best month +5, worst -5, exactly on target exactly 0, everyone else proportional.
The task is a real question from an Excel help forum, preserved in SpreadsheetBench,
a public benchmark of real spreadsheet problems (task 42526, numbers unchanged).
Grading is a script comparing cells against the correct answers; no AI judges
anything, and the model never sees the answer files.

## Commands

- **"go"** — run one full loop (attempt, learn, verify, exam), a few minutes.
- **"reset"** — wipe the learned state back to zero.
- **"explain"** — after a run, walk through what the model wrote and why it worked.

How it went when we ran it, plus the standalone code:
[sundial-org/wikiskill-spreadsheet](https://github.com/sundial-org/wikiskill-spreadsheet).

tasks/leaderboard/1_leaderboard_answer.xlsxprebuilt · locked · 5.4 KBReplaced by the first recompile.
Binary file, seeded as-is.
tasks/leaderboard/1_leaderboard_input.xlsxprebuilt · locked · 5.3 KBReplaced by the first recompile.
Binary file, seeded as-is.
tasks/leaderboard/2_leaderboard_answer.xlsxprebuilt · locked · 5.4 KBReplaced by the first recompile.
Binary file, seeded as-is.
tasks/leaderboard/2_leaderboard_input.xlsxprebuilt · locked · 5.3 KBReplaced by the first recompile.
Binary file, seeded as-is.
tasks/leaderboard/3_leaderboard_answer.xlsxprebuilt · locked · 5.4 KBReplaced by the first recompile.
Binary file, seeded as-is.
tasks/leaderboard/3_leaderboard_input.xlsxprebuilt · locked · 5.3 KBReplaced by the first recompile.
Binary file, seeded as-is.
Sandbox setup

setup.sh456 BRuns as bash setup.sh when the workspace opens.

#!/bin/bash
# Idempotent sandbox bootstrap: a persistent venv with openpyxl for the grader
# and rollout code. Survives sandbox idle-down (lives on the /workspace volume).
set -e
VENV=/workspace/.sundial/wikiskill-venv
mkdir -p /workspace/.sundial
exec 9>/workspace/.sundial/wikiskill-setup.lock
flock 9
if [ ! -x "$VENV/bin/python3" ]; then
  python3 -m venv "$VENV"
fi
"$VENV/bin/pip" install --quiet 'openpyxl>=3.1'
echo "wikiskill venv ready at $VENV"