{
  "schemaVersion": "1.0.0",
  "id": "website-generation-2026-11-06",
  "title": "Make me a website: effort comparison",
  "date": "2026-11-06",
  "taskType": "website-generation",
  "description": "GPT-6.1 Sol and Claude Opus 5.5 at Low, Medium, High, and Extra High. One fresh static website per setting.",
  "protocol": {
    "version": "1.0.0",
    "prompt": "Machine,\n\nPls make website,\n\nall responsive like,\nw/ BIG pictures ooo,\nuse my fav fonts,\nalso fancy menus with whooosh on,\nload fast pls\n\nThanks,\nHuman\n\nPS no bugs :)",
    "instructionsTemplate": "Website generation benchmark. Work independently; do not delegate or invoke other models. You have a fresh empty output directory: {outputDirectory}\nCreate the website requested below. Since subject and favorite fonts are unspecified, choose them yourself without asking questions. Deliver a static site with index.html at that directory; HTML/CSS/JS and external image/font URLs are allowed. No package installs, build pipeline, servers, publishing, or edits outside that directory. Use filesystem/shell tools to write files. Make the result complete, then stop. Do not read other contenders, benchmark files, prior runs, or repository source. You may verify your own work locally but do not call preview tools or launch browsers. This harness fixes static-site delivery for all models; you are being timed from task start to final answer. In your final answer state completion and any assumptions briefly.\n\nExact user prompt transcribed from their image:\nMachine,\n\nPls make website,\n\nall responsive like,\nw/ BIG pictures ooo,\nuse my fav fonts,\nalso fancy menus with whooosh on,\nload fast pls\n\nThanks,\nHuman\n\nPS no bugs :)",
    "execution": "Eight fresh T3 child sessions, sequential; Promptfoo grades captured browser artifacts without new inference.",
    "timingDefinition": "Completed T3 provider-turn wall time; excludes parent orchestration gaps and post-run browser verification.",
    "controls": [
      "Same static-site instructions except output directory.",
      "Codex Priority tier held constant; Claude Fast explicitly off.",
      "Fresh session and empty output directory for every setting.",
      "Contenders could verify locally; browser use, delegation, publishing, package installs, and edits outside output directory were forbidden."
    ],
    "limitations": [
      "One sample per setting; uncontrolled provider load and cache state.",
      "Subject and favorite fonts unspecified; compliance smoke checks are not visual-design accuracy.",
      "Different scopes chosen by the models; no matched feature budget.",
      "Local browser load observations are not production mobile performance.",
      "Controls were prompt instructions rather than filesystem restrictions; deviations are noted per run.",
      "Generated websites may depend on live external images and fonts.",
      "Raw T3 traces and machine-specific paths are not included in this dataset."
    ]
  },
  "rubricVersion": "1.0.0",
  "metrics": [
    {
      "id": "tool-wall-ms",
      "label": "Tool wall time",
      "description": "Union of recorded tool intervals, not tool CPU time.",
      "category": "measurement",
      "valueType": "number",
      "unit": "ms",
      "direction": "lower",
      "minimum": 0
    },
    {
      "id": "non-tool-wall-ms",
      "label": "Other provider wall time",
      "description": "Generation, reasoning, transport, and runtime overhead outside recorded tools.",
      "category": "measurement",
      "valueType": "number",
      "unit": "ms",
      "direction": "lower",
      "minimum": 0
    },
    {
      "id": "first-tool-after-ms",
      "label": "Time until first tool",
      "description": "Provider start to first recorded tool call; not first-token latency.",
      "category": "measurement",
      "valueType": "number",
      "unit": "ms",
      "direction": "lower",
      "minimum": 0
    },
    {
      "id": "local-load-ms",
      "label": "Observed local load",
      "description": "One unthrottled localhost navigation; external assets and cache are uncontrolled.",
      "category": "measurement",
      "valueType": "number",
      "unit": "ms",
      "direction": "lower",
      "minimum": 0
    },
    {
      "id": "site-code-bytes",
      "label": "Site code size",
      "description": "Total HTML/CSS/JavaScript bytes; excludes image assets.",
      "category": "measurement",
      "valueType": "number",
      "unit": "bytes",
      "direction": "lower",
      "minimum": 0
    },
    {
      "id": "renders",
      "label": "Renders",
      "description": "Body contains substantive page content; screenshots inspected separately.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "responsive",
      "label": "Responsive",
      "description": "No page-wide horizontal overflow at 390, 768, and 1440 CSS pixels.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "large-picture",
      "label": "Large Picture",
      "description": "At least one loaded image is at least half the desktop viewport width and 240px high.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "typography",
      "label": "Typography",
      "description": "Computed fonts show a deliberate font choice; favorite-font preference is unspecified.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "menu-works",
      "label": "Menu Works",
      "description": "Observed menu opens and closes.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "menu-motion",
      "label": "Menu Motion",
      "description": "Menu or descendant has nonzero transition/animation duration.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "script-checks",
      "label": "Script Checks",
      "description": "Inline/local JS parses and no errors were observed during tested interactions.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    },
    {
      "id": "anchor-targets",
      "label": "Anchor Targets",
      "description": "Nonempty internal hash links have section targets or browser-verified dialog actions.",
      "category": "check",
      "valueType": "boolean",
      "direction": "higher"
    }
  ],
  "viewers": [
    "timing-comparison",
    "screenshot-comparison",
    "website-preview"
  ],
  "tags": [
    "silly-tests",
    "websites",
    "effort"
  ]
}