Separate clean and grid screenshot artifacts

runwave/agent/src/agent-player.js

@@ -23,6 +23,13 @@ function latestScreenshot(response) {
   return (capture && capture.path) || body.screenshot || null;
 }
 
+function latestAgentScreenshot(response) {
+  const body = responseBody(response);
+  const captures = Array.isArray(body.captures) ? body.captures : [];
+  const capture = captures.length ? captures[captures.length - 1] : null;
+  return (capture && (capture.gridPath || capture.path)) || body.gridScreenshot || body.screenshot || null;
+}
+
 function responseState(response) {
   const body = responseBody(response);
   return body.endState || body.state || {};
@@ -193,9 +200,10 @@ async function runAgenticPlaytest({ job, initialResponse, runAction, outputDir,
     const remainingMs = maxMs - elapsedMs - STOP_RESERVE_MS;
     if (remainingMs <= 0) break;
 
-    const screenshot = latestScreenshot(lastResponse);
+    const screenshot = latestAgentScreenshot(lastResponse);
+    const cleanScreenshot = latestScreenshot(lastResponse);
     const state = responseState(lastResponse);
-    recorder.observation({ step, elapsedMs, screenshot, state });
+    recorder.observation({ step, elapsedMs, screenshot, cleanScreenshot, state });
 
     let sequence;
     let model;
@@ -284,7 +292,7 @@ async function runAgenticPlaytest({ job, initialResponse, runAction, outputDir,
         error: sequence.error,
       });
 
-      const failedResult = failedActionResult({ screenshot, state, error: sequence.error });
+      const failedResult = failedActionResult({ screenshot: cleanScreenshot, state, error: sequence.error });
       history.push({
         step,
         summary: sequence.summary,
@@ -361,13 +369,13 @@ async function runAgenticPlaytest({ job, initialResponse, runAction, outputDir,
         cursorMoves: buckets.cursorMoves,
         viewMoves: buckets.viewMoves,
         failedAction: true,
-        result: failedActionResult({ screenshot, state, error: message }),
+        result: failedActionResult({ screenshot: cleanScreenshot, state, error: message }),
       });
       continue;
     }
 
     lastResponse = stepResponse;
-    const result = postSequenceResult(lastResponse, screenshot);
+    const result = postSequenceResult(lastResponse, cleanScreenshot);
     history.push({
       step,
       summary: sequence.summary,
@@ -411,6 +419,7 @@ module.exports = {
   actionsWithinDuration,
   failedActionAfterInvalidJson,
   isRecoverableModelSequenceError,
+  latestAgentScreenshot,
   latestScreenshot,
   postSequenceResult,
   responseState,

runwave/agent/test/agent.test.js

@@ -8,8 +8,14 @@ const test = require('node:test');
 const { PNG } = require('pngjs');
 
 const { normalizeSequence } = require('../src/action-parser');
-const { actionsWithinDuration, failedActionAfterInvalidJson, runAgenticPlaytest } = require('../src/agent-player');
-const { chatCompletion, parseJsonResponse } = require('../src/model-client');
+const {
+  actionsWithinDuration,
+  failedActionAfterInvalidJson,
+  latestAgentScreenshot,
+  latestScreenshot,
+  runAgenticPlaytest,
+} = require('../src/agent-player');
+const { chatCompletion, dataUrl, parseJsonResponse } = require('../src/model-client');
 const { buildPlaytesterPrompt, compactHistory, sequenceSchemaGuide } = require('../src/prompt');
 const { drawMarkGridOnScreenshot } = require('../../controller/src/grid-overlay');
 const { gridSafeSampleRatio, randomPointInCells } = require('../../protocol/src/mark-grid');
@@ -60,6 +66,26 @@ test('normalizes model sequences into controller steps', () => {
   assert.equal(sequence.previousSequenceOutcome, 'Enter opened the menu.');
 });
 
+test('agent screenshot selection separates clean evidence from grid targeting images', () => {
+  const response = {
+    screenshot: '/tmp/initial-clean.png',
+    gridScreenshot: '/tmp/initial-grid.png',
+    captures: [
+      {
+        path: '/tmp/capture-clean.png',
+        gridPath: '/tmp/capture-grid.png',
+      },
+    ],
+  };
+
+  assert.equal(latestScreenshot(response), '/tmp/capture-clean.png');
+  assert.equal(latestAgentScreenshot(response), '/tmp/capture-grid.png');
+  assert.equal(latestScreenshot({ screenshot: '/tmp/clean.png' }), '/tmp/clean.png');
+  assert.equal(latestAgentScreenshot({ screenshot: '/tmp/clean.png' }), '/tmp/clean.png');
+  assert.equal(latestScreenshot({ gridScreenshot: '/tmp/grid-only.png' }), null);
+  assert.equal(latestAgentScreenshot({ gridScreenshot: '/tmp/grid-only.png' }), '/tmp/grid-only.png');
+});
+
 test('normalizes grid-cell model actions into concrete pointer events', () => {
   const sequence = normalizeSequence(
     {
@@ -483,6 +509,69 @@ test('agent playtest loop calls model and executes returned sequence', async ()
   assert.equal(fs.existsSync(path.join(dir, 'agent', 'agent-summary.json')), true);
 });
 
+test('agent loop sends grid screenshots to the model and records clean screenshots as evidence', async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-agent-grid-split-test-'));
+  const cleanInitial = path.join(dir, 'initial.png');
+  const gridInitial = path.join(dir, 'initial.grid.png');
+  const cleanAfter = path.join(dir, 'after.png');
+  const gridAfter = path.join(dir, 'after.grid.png');
+  fs.writeFileSync(cleanInitial, 'clean initial screenshot bytes');
+  fs.writeFileSync(gridInitial, 'grid initial screenshot bytes');
+  fs.writeFileSync(cleanAfter, 'clean after screenshot bytes');
+  fs.writeFileSync(gridAfter, 'grid after screenshot bytes');
+
+  const modelScreenshots = [];
+  const result = await runAgenticPlaytest({
+    job: {
+      playtestDurationMs: 7000,
+      agentMinPlaytestMs: 0,
+      viewport: { width: 640, height: 360 },
+    },
+    initialResponse: {
+      screenshot: cleanInitial,
+      gridScreenshot: gridInitial,
+      state: { screen: 'initial' },
+    },
+    outputDir: path.join(dir, 'agent'),
+    modelClient: async ({ messages }) => {
+      const image = messages[0].content.find((item) => item.type === 'image_url');
+      modelScreenshots.push(image.image_url.url);
+      return {
+        model: 'fake-model',
+        usage: { total_tokens: 1 },
+        json: {
+          summary: 'start screen is visible',
+          actions: [{ type: 'key', start: 0, end: 500, key: 'Enter' }],
+          should_stop: true,
+        },
+      };
+    },
+    runAction: async () => ({
+      ok: true,
+      captures: [{ path: cleanAfter, gridPath: gridAfter }],
+      endState: { screen: 'after' },
+    }),
+  });
+
+  const promptLog = fs
+    .readFileSync(path.join(dir, 'agent', 'agent-prompts.jsonl'), 'utf8')
+    .trim()
+    .split('\n')
+    .map((line) => JSON.parse(line));
+  const observationLog = fs
+    .readFileSync(path.join(dir, 'agent', 'agent-observations.jsonl'), 'utf8')
+    .trim()
+    .split('\n')
+    .map((line) => JSON.parse(line));
+
+  assert.equal(modelScreenshots[0], dataUrl(gridInitial));
+  assert.equal(promptLog[0].screenshot, gridInitial);
+  assert.equal(observationLog[0].screenshot, gridInitial);
+  assert.equal(observationLog[0].cleanScreenshot, cleanInitial);
+  assert.equal(result.history[0].result.screenshot, cleanAfter);
+  assert.equal(result.history[0].result.screenshotChanged, true);
+});
+
 test('agent loop records invalid JSON as a failed action and retries with the same screenshot', async () => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-agent-json-failed-action-test-'));
   const screenshot = path.join(dir, 'screen.png');

runwave/controller/README.md

@@ -109,8 +109,11 @@ runwave-controller '{
 }'
 ```
 
-Screenshots include a 16x16 red mark grid by default. Overlay column labels are
-shown in the top/bottom margins and overlay row labels are shown in the
+Screenshots returned in `screenshot` and `captures[].path` are clean game
+screenshots. Responses also include playtester-only grid images in
+`gridScreenshot` and `captures[].gridPath`.
+The grid image contains a 16x16 red mark grid by default. Overlay column labels
+are shown in the top/bottom margins and overlay row labels are shown in the
 left/right margins. Pointer actions may use overlay row/column grid objects
 instead of exact pixels.
 

runwave/controller/src/action-handler.js

@@ -24,6 +24,14 @@ function withVerboseLog(runtime, payload) {
   };
 }
 
+function screenshotFields(artifact) {
+  if (!artifact || typeof artifact !== 'object') return {};
+  return {
+    screenshot: artifact.path,
+    ...(artifact.gridPath ? { gridScreenshot: artifact.gridPath } : {}),
+  };
+}
+
 async function writeStateResponse(runtime, input) {
   const { output, paths } = runtime;
   const payload = await runtime.profiler.time('action.state.build_payload', { action_name: input.action_name }, async () =>
@@ -46,15 +54,16 @@ async function writeScreenshotResponse(runtime, input) {
   const outputDir = runtime.profiler.timeSync('action.screenshot.action_dir', { action_name: input.action_name }, () =>
     output.actionDir(input.action_name)
   );
+  const screenshot = await runtime.profiler.time('action.screenshot.capture', { action_name: input.action_name }, () =>
+    browser.screenshotArtifact(outputDir, input.name || 'screenshot')
+  );
   const payload = withVerboseLog(runtime, {
     ok: true,
     action: 'screenshot',
     action_name: input.action_name,
     session_id: runtime.sessionId,
     sessionDir: paths.runDir,
-    screenshot: await runtime.profiler.time('action.screenshot.capture', { action_name: input.action_name }, () =>
-      browser.screenshot(outputDir, input.name || 'screenshot')
-    ),
+    ...screenshotFields(screenshot),
     state: await readState(runtime, input),
   });
   return runtime.profiler.timeSync('action.screenshot.write_response', { action_name: input.action_name }, () =>
@@ -73,16 +82,17 @@ async function writeNavigateResponse(runtime, input) {
     browser.navigate(isReset ? { url: browser.launchUrl } : input)
   );
   if (isReset) runtime.stepIndex = 0;
+  const screenshot = await runtime.profiler.time('action.navigate.screenshot', { action_name: input.action_name }, () =>
+    browser.screenshotArtifact(outputDir, '000-after-navigate')
+  );
 
   const payload = withVerboseLog(runtime, {
     ok: true,
     action: input.action,
     action_name: input.action_name,
     session_id: runtime.sessionId,
     sessionDir: paths.runDir,
-    screenshot: await runtime.profiler.time('action.navigate.screenshot', { action_name: input.action_name }, () =>
-      browser.screenshot(outputDir, '000-after-navigate')
-    ),
+    ...screenshotFields(screenshot),
     state: await readState(runtime, input),
   });
   return runtime.profiler.timeSync('action.navigate.write_response', { action_name: input.action_name }, () =>
@@ -128,7 +138,7 @@ async function writeStopResponse(runtime, input) {
     input.finalScreenshot === false || config.finalScreenshot === false
       ? null
       : await runtime.profiler.time('action.stop.screenshot', { action_name: input.action_name }, () =>
-          browser.screenshot(outputDir, '999-final')
+          browser.screenshotArtifact(outputDir, '999-final')
         );
   const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close());
   const video = typeof recording === 'string' ? recording : recording.video;
@@ -148,7 +158,7 @@ async function writeStopResponse(runtime, input) {
   if (recording && typeof recording === 'object') {
     if (recording.audioVideo) payload.audioVideo = recording.audioVideo;
   }
-  if (screenshot) payload.screenshot = screenshot;
+  Object.assign(payload, screenshotFields(screenshot));
   const response = runtime.profiler.timeSync('action.stop.write_response', { action_name: input.action_name }, () =>
     output.response(input.action_name, payload)
   );

runwave/controller/src/browser-session.js

@@ -55,6 +55,10 @@ function launchHeadless(config = {}) {
   return isRecording(config) ? false : config.headless !== false;
 }
 
+function gridScreenshotPath(file) {
+  return file.toLowerCase().endsWith('.png') ? file.slice(0, -4) + '.grid.png' : `${file}.grid.png`;
+}
+
 async function pageViewportVideoSource(page, env = process.env) {
   const display = env.DISPLAY || ':0';
   const metrics = await page.evaluate(() => ({
@@ -199,7 +203,7 @@ class BrowserSession {
     await this.time('browser.navigate.wait_after_load', { waitAfterLoad }, () => sleep(waitAfterLoad));
   }
 
-  async screenshot(outputDir, name) {
+  async screenshotArtifact(outputDir, name) {
     const fileName = `${safeName(name || `capture-${timestamp()}`)}.png`;
     const file = path.join(outputDir, fileName);
     await this.time('browser.screenshot.capture', { file, fullPage: Boolean(this.config.fullPageScreenshots) }, () =>
@@ -209,10 +213,16 @@ class BrowserSession {
         scale: 'css',
       })
     );
-    if (this.config.gridScreenshots !== false) {
-      this.timeSync('browser.screenshot.grid_overlay', { file }, () => drawGridOnScreenshot(file, this.config));
-    }
-    return file;
+    const gridFile = gridScreenshotPath(file);
+    this.timeSync('browser.screenshot.grid_copy', { file, gridFile }, () => fs.copyFileSync(file, gridFile));
+    this.timeSync('browser.screenshot.grid_overlay', { file: gridFile }, () => drawGridOnScreenshot(gridFile, this.config));
+    const artifact = { path: file, gridPath: gridFile };
+    return artifact;
+  }
+
+  async screenshot(outputDir, name) {
+    const artifact = await this.screenshotArtifact(outputDir, name);
+    return artifact.path;
   }
 
   async keyDown(key) {

runwave/controller/src/daemon.js

@@ -102,9 +102,11 @@ async function main() {
       state: await profiler.time('daemon.start_response.state', () => browser.state()),
     };
     if (config.initialScreenshot !== false) {
-      payload.screenshot = await profiler.time('daemon.start_response.screenshot', () =>
-        browser.screenshot(outputDir, '000-initial')
+      const screenshot = await profiler.time('daemon.start_response.screenshot', () =>
+        browser.screenshotArtifact(outputDir, '000-initial')
       );
+      payload.screenshot = screenshot.path;
+      if (screenshot.gridPath) payload.gridScreenshot = screenshot.gridPath;
     }
     if (!payload.verboseLog) delete payload.verboseLog;
     const startResponse = profiler.timeSync('daemon.start_response.write_output', () =>

runwave/controller/src/protocol.js

@@ -145,7 +145,6 @@ function startSessionConfig(input, options = {}) {
       captureIntervalMs: optionalNumber(input.captureIntervalMs, 1000),
       finalScreenshot: input.finalScreenshot !== false,
       fullPageScreenshots: Boolean(input.fullPageScreenshots),
-      gridScreenshots: input.gridScreenshots !== false,
       markGridRows: optionalNumber(input.markGridRows ?? input.gridRows, DEFAULT_MARK_GRID.rows),
       markGridCols: optionalNumber(input.markGridCols ?? input.gridCols, DEFAULT_MARK_GRID.cols),
     },

runwave/controller/src/step-executor.js

@@ -15,13 +15,17 @@ async function waitUntil(startedAt, offsetMs, profiler, fields = {}) {
 }
 
 async function captureAt({ browser, outputDir, prefix, stateExpression, at, profiler }) {
+  const name = `${prefix}-${String(at).padStart(5, '0')}ms`;
+  const captureScreenshot = () => browser.screenshotArtifact
+    ? browser.screenshotArtifact(outputDir, name)
+    : Promise.resolve(browser.screenshot(outputDir, name)).then((path) => ({ path }));
+  const screenshot = profiler
+    ? await profiler.time('timeline.capture.screenshot', { at }, captureScreenshot)
+    : await captureScreenshot();
   return {
     at,
-    path: profiler
-      ? await profiler.time('timeline.capture.screenshot', { at }, () =>
-          browser.screenshot(outputDir, `${prefix}-${String(at).padStart(5, '0')}ms`)
-        )
-      : await browser.screenshot(outputDir, `${prefix}-${String(at).padStart(5, '0')}ms`),
+    path: screenshot.path,
+    ...(screenshot.gridPath ? { gridPath: screenshot.gridPath } : {}),
     state: profiler
       ? await profiler.time('timeline.capture.state', { at }, () => browser.state(stateExpression))
       : await browser.state(stateExpression),