@codecademy/gamut
72.3.072.3.1-alpha.f273fd.0
agent-tools/skills/gamut-style-utilities-workspace/iteration-1/benchmark.json+
agent-tools/skills/gamut-style-utilities-workspace/iteration-1/benchmark.jsonNew file+295
Index: package/agent-tools/skills/gamut-style-utilities-workspace/iteration-1/benchmark.json
===================================================================
--- package/agent-tools/skills/gamut-style-utilities-workspace/iteration-1/benchmark.json
+++ package/agent-tools/skills/gamut-style-utilities-workspace/iteration-1/benchmark.json
@@ -0,0 +1,295 @@
+{
+ "metadata": {
+ "skill_name": "gamut-style-utilities",
+ "skill_path": "<path/to/skill>",
+ "executor_model": "<model-name>",
+ "analyzer_model": "<model-name>",
+ "timestamp": "2026-07-27T21:02:51Z",
+ "evals_run": [1, 2, 3],
+ "runs_per_configuration": 3
+ },
+ "runs": [
+ {
+ "eval_id": 1,
+ "configuration": "with_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 0.75,
+ "passed": 3,
+ "failed": 1,
+ "total": 4,
+ "time_seconds": 134.773,
+ "tokens": 36516,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses variant() with a variants map keyed by status, not manual branching",
+ "passed": true,
+ "evidence": "variant({ prop: 'status', defaultVariant: 'info', variants: { success, warning, info } }) \u2014 correct."
+ },
+ {
+ "text": "status prop type is derived from the variant definition via StyleProps",
+ "passed": true,
+ "evidence": "StyleProps<typeof statusBadgeVariants> used for the styled component and StatusBadgeProps."
+ },
+ {
+ "text": "Colors used for each status are semantic ColorMode tokens (e.g. background-success/feedback-success, background-warning/feedback-warning) rather than raw hex literals",
+ "passed": false,
+ "evidence": "Uses raw palette pairs green-0/green-900 (success) and yellow-0/yellow-900 (warning) instead of the semantic, ColorMode-adaptive background-success/feedback-success and background-warning/feedback-warning tokens that actually exist in the theme. These raw pairs are fixed and will not adapt correctly between light/dark mode the way the semantic tokens would. (info falling back to blue-0/blue-800 is reasonable since no semantic 'info' token exists.)"
+ },
+ {
+ "text": "Does not wrap an existing Gamut component in styled() \u2014 this is a new primitive",
+ "passed": true,
+ "evidence": "Built as styled.span, not wrapping Badge/Tag/any existing Gamut component."
+ }
+ ],
+ "notes": []
+ },
+ {
+ "eval_id": 2,
+ "configuration": "with_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 1.0,
+ "passed": 4,
+ "failed": 0,
+ "total": 4,
+ "time_seconds": 101.165,
+ "tokens": 38222,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses states() rather than variant() since the flags are independent, not mutually exclusive",
+ "passed": true,
+ "evidence": "states({ base, selected, disabled }) \u2014 correct, with a comment explicitly explaining why states() over variant()."
+ },
+ {
+ "text": "Does not hand-write a separate combined selected+disabled branch",
+ "passed": true,
+ "evidence": "Only base/selected/disabled blocks exist; both apply simultaneously via states()'s layering, no combined branch."
+ },
+ {
+ "text": "Props typed via StyleProps<typeof theStates>",
+ "passed": true,
+ "evidence": "ChipStyleProps = StyleProps<typeof chipStates>."
+ },
+ {
+ "text": "Colors are semantic tokens, not raw hex",
+ "passed": true,
+ "evidence": "background, border-primary, text, background-hover, background-selected, primary \u2014 all semantic tokens."
+ }
+ ],
+ "notes": []
+ },
+ {
+ "eval_id": 3,
+ "configuration": "with_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 1.0,
+ "passed": 4,
+ "failed": 0,
+ "total": 4,
+ "time_seconds": 68.737,
+ "tokens": 25159,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses useTheme() to read a real theme color token in JS",
+ "passed": true,
+ "evidence": "useTheme() from @emotion/react, theme.colors[strokeColor]."
+ },
+ {
+ "text": "The token path read genuinely exists in the theme",
+ "passed": true,
+ "evidence": "Default strokeColor='primary', a real token."
+ },
+ {
+ "text": "Does not try to apply CSS-based styling for the stroke value",
+ "passed": true,
+ "evidence": "No css()/variant()/states() used anywhere; the path/stroke are plain SVG attributes."
+ },
+ {
+ "text": "Rest of the component still follows normal library styling conventions",
+ "passed": true,
+ "evidence": "Simple functional component, typed props, no unrelated inline-style abuse."
+ }
+ ],
+ "notes": []
+ },
+ {
+ "eval_id": 1,
+ "configuration": "without_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 1.0,
+ "passed": 4,
+ "failed": 0,
+ "total": 4,
+ "time_seconds": 458.34,
+ "tokens": 59561,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses variant() with a variants map keyed by status, not manual branching",
+ "passed": true,
+ "evidence": "variant({ prop: 'status', base: {...}, variants: { success, warning, info } }) \u2014 correct."
+ },
+ {
+ "text": "status prop type is derived from the variant definition via StyleProps",
+ "passed": true,
+ "evidence": "StyleProps<typeof statusBadgeVariants> used."
+ },
+ {
+ "text": "Colors used for each status are semantic ColorMode tokens (e.g. background-success/feedback-success, background-warning/feedback-warning) rather than raw hex literals",
+ "passed": true,
+ "evidence": "Uses background-success/feedback-success and background-warning/feedback-warning exactly, with an explanatory comment noting this keeps the badge in sync with dark/light color modes. info falls back to blue-100/blue-500 with a comment explaining no dedicated info token exists \u2014 same reasonable exception as the with_skill run."
+ },
+ {
+ "text": "Does not wrap an existing Gamut component in styled() \u2014 this is a new primitive",
+ "passed": true,
+ "evidence": "Built as styled.span, not wrapping an existing component."
+ }
+ ],
+ "notes": []
+ },
+ {
+ "eval_id": 2,
+ "configuration": "without_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 1.0,
+ "passed": 4,
+ "failed": 0,
+ "total": 4,
+ "time_seconds": 166.97,
+ "tokens": 50209,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses states() rather than variant() since the flags are independent, not mutually exclusive",
+ "passed": true,
+ "evidence": "system.states({ selected, disabled }) with a comment explicitly contrasting it against variant()."
+ },
+ {
+ "text": "Does not hand-write a separate combined selected+disabled branch",
+ "passed": true,
+ "evidence": "Only selected/disabled blocks; both apply simultaneously, no combined branch."
+ },
+ {
+ "text": "Props typed via StyleProps<typeof theStates>",
+ "passed": true,
+ "evidence": "ChipStateProps = StyleProps<typeof chipStates>, composed into ChipProps."
+ },
+ {
+ "text": "Colors are semantic tokens, not raw hex",
+ "passed": true,
+ "evidence": "background, background-hover, background-selected, primary, border-tertiary, text \u2014 all semantic tokens."
+ }
+ ],
+ "notes": []
+ },
+ {
+ "eval_id": 3,
+ "configuration": "without_skill",
+ "run_number": 1,
+ "result": {
+ "pass_rate": 1.0,
+ "passed": 4,
+ "failed": 0,
+ "total": 4,
+ "time_seconds": 153.012,
+ "tokens": 40686,
+ "tool_calls": 0,
+ "errors": 0
+ },
+ "expectations": [
+ {
+ "text": "Uses useTheme() to read a real theme color token in JS",
+ "passed": true,
+ "evidence": "useTheme() from @emotion/react, theme.colors[color]."
+ },
+ {
+ "text": "The token path read genuinely exists in the theme",
+ "passed": true,
+ "evidence": "Default color='primary', typed as the real Colors type imported from @codecademy/gamut-styles, a real token."
+ },
+ {
+ "text": "Does not try to apply CSS-based styling for the stroke value",
+ "passed": true,
+ "evidence": "No css()/variant()/states() used; plain SVG attributes."
+ },
+ {
+ "text": "Rest of the component still follows normal library styling conventions",
+ "passed": true,
+ "evidence": "Strongly-typed props, memoized path calculation, cites real precedent (RadialProgress) for the pattern."
+ }
+ ],
+ "notes": []
+ }
+ ],
+ "run_summary": {
+ "with_skill": {
+ "pass_rate": {
+ "mean": 0.9167,
+ "stddev": 0.1443,
+ "min": 0.75,
+ "max": 1.0
+ },
+ "time_seconds": {
+ "mean": 101.5583,
+ "stddev": 33.0198,
+ "min": 68.737,
+ "max": 134.773
+ },
+ "tokens": {
+ "mean": 33299.0,
+ "stddev": 7100.8668,
+ "min": 25159,
+ "max": 38222
+ }
+ },
+ "without_skill": {
+ "pass_rate": {
+ "mean": 1.0,
+ "stddev": 0.0,
+ "min": 1.0,
+ "max": 1.0
+ },
+ "time_seconds": {
+ "mean": 259.4407,
+ "stddev": 172.3932,
+ "min": 153.012,
+ "max": 458.34
+ },
+ "tokens": {
+ "mean": 50152.0,
+ "stddev": 9437.6291,
+ "min": 40686,
+ "max": 59561
+ }
+ },
+ "delta": {
+ "pass_rate": "-0.08",
+ "time_seconds": "-157.9",
+ "tokens": "-16853"
+ }
+ },
+ "notes": [
+ "without_skill actually beats with_skill on pass rate this iteration (1.0 vs 0.917), driven entirely by badge-variant-status: with_skill used raw, non-adaptive palette pairs (green-0/green-900, yellow-0/yellow-900) for the success/warning variant colors, while without_skill discovered and used the real semantic, ColorMode-adaptive tokens (background-success/feedback-success, background-warning/feedback-warning) that already exist in the theme for exactly this purpose.",
+ "Likely root cause: the skill (gamut-style-utilities) own css() example shows css({ bg: 'navy-400', p: 4 }) as its first, unlabeled example, immediately next to a second example explicitly commented '// Semantic color (adapts to color mode)'. The first example reads as the default path with no caveat that it is a deliberately-fixed color choice, and likely primed the with_skill agent toward reaching for a similarly convenient raw palette pair instead of checking whether semantic feedback tokens already existed for this use case. This is the same underlying color-token-discipline gap already found and partially fixed in gamut-system-props (dark-mode-adaptive color selection).",
+ "chip-independent-states and sparkline-theme-color both tied at 1.0/1.0 -- with generous repo-exploration access, the baseline independently found the correct states()-vs-variant() distinction and the useTheme() escape hatch by studying real precedent (Tag/Menu/ButtonBase for states(), RadialProgress for useTheme()).",
+ "with_skill is dramatically cheaper and faster here: 101.6s / 33,299 tokens mean vs. without_skill's 259.4s / 50,152 tokens -- the skill saved roughly 60% of the wall-clock time and tokens by giving the agent the css()/variant()/states()/useTheme() decision up front instead of making it discover the right API by grepping the whole component library. That efficiency gain came with a real quality cost on the one eval where the skill's own example modeled the anti-pattern.",
+ "Net read: this looks like a genuine, fixable skill gap rather than noise -- the fix that worked for gamut-system-props (call out the semantic-vs-fixed color choice explicitly, right next to the example that risks being copied uncritically) should transfer directly here."
+ ]
+}