Initial import from garrytan/gstack@026751e (main snapshot via local relay)

Source: https://github.com/garrytan/gstack/commit/026751e

test/skill-e2e-setup-gbrain-remote.test.ts

@@ -0,0 +1,223 @@
+// E2E: /setup-gbrain Path 4 (Remote MCP) happy path via Agent SDK.
+//
+// Drives the skill against a stub HTTP MCP server and a stubbed `claude`
+// binary that records `claude mcp add` calls. Asserts:
+//   - The verify helper succeeds (no AUTH/MALFORMED/NETWORK error in output)
+//   - The skill calls `claude mcp add --transport http` with the bearer
+//   - The token NEVER appears in the CLAUDE.md block the skill writes
+//   - The wrote_findings_before_asking failure mode is NOT triggered
+//
+// Cost: ~$0.30-$0.50 per run. Gate-tier (EVALS=1 EVALS_TIER=gate).
+//
+// See setup-gbrain/SKILL.md.tmpl Step 4 (Path 4) for the contract under test.
+
+import { describe, test, expect } from 'bun:test';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import * as http from 'http';
+import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner';
+
+// Periodic-tier: the model's interpretation of "follow Path 4 only" is
+// non-deterministic (it sometimes skips Step 8 CLAUDE.md write, sometimes
+// shortcuts past the verify helper). The deterministic gate coverage for
+// Path 4 lives in test/setup-gbrain-path4-structure.test.ts (free, <200ms).
+const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
+const describeE2E = shouldRun ? describe : describe.skip;
+
+// Spin up a stub MCP server that responds to initialize + tools/list.
+function startStubMcpServer(opts: { failWithStatus?: number; failBody?: string } = {}): Promise<{ url: string; close: () => Promise<void> }> {
+  return new Promise((resolve) => {
+    const server = http.createServer((req, res) => {
+      if (req.method !== 'POST' || !(req.url ?? '').endsWith('/mcp')) {
+        res.statusCode = 404;
+        res.end();
+        return;
+      }
+      let body = '';
+      req.on('data', (c) => (body += c));
+      req.on('end', () => {
+        if (opts.failWithStatus) {
+          res.statusCode = opts.failWithStatus;
+          res.setHeader('Content-Type', 'application/json');
+          res.end(opts.failBody ?? JSON.stringify({ error: 'fail' }));
+          return;
+        }
+        const reqJson = (() => {
+          try { return JSON.parse(body); } catch { return {} as any; }
+        })();
+        let respBody: any;
+        if (reqJson.method === 'initialize') {
+          respBody = {
+            result: {
+              protocolVersion: '2024-11-05',
+              capabilities: { tools: {} },
+              serverInfo: { name: 'gbrain', version: '0.27.1' },
+            },
+            jsonrpc: '2.0',
+            id: reqJson.id,
+          };
+        } else if (reqJson.method === 'tools/list') {
+          respBody = { result: { tools: [{ name: 'search' }, { name: 'put_page' }] }, jsonrpc: '2.0', id: reqJson.id };
+        } else {
+          respBody = { error: { code: -32601, message: 'unknown method' }, jsonrpc: '2.0', id: reqJson.id };
+        }
+        // SSE-shape since the verify helper supports both, and many MCP
+        // servers (including wintermute) wrap responses as SSE.
+        res.statusCode = 200;
+        res.setHeader('Content-Type', 'text/event-stream');
+        res.end(`event: message\ndata: ${JSON.stringify(respBody)}\n\n`);
+      });
+    });
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address();
+      if (!addr || typeof addr === 'string') throw new Error('no address');
+      resolve({
+        url: `http://127.0.0.1:${addr.port}/mcp`,
+        close: () => new Promise((r) => server.close(() => r())),
+      });
+    });
+  });
+}
+
+// Stubbed `claude` binary: intercepts `mcp add` and `mcp list` commands so
+// the skill's Step 5a registration appears to succeed, while we record
+// every invocation for assertions.
+function makeFakeClaude(fakeBinDir: string): string {
+  const claudeJsonPath = path.join(fakeBinDir, 'claude.json');
+  const callLog = path.join(fakeBinDir, 'claude-calls.log');
+  const script = `#!/bin/bash
+echo "claude $@" >> "${callLog}"
+case "$1 $2" in
+  "mcp add")
+    # Just record the call; pretend it succeeded.
+    exit 0
+    ;;
+  "mcp list")
+    echo "gbrain: http://127.0.0.1:0/mcp (HTTP) - ✓ Connected"
+    exit 0
+    ;;
+  "mcp remove")
+    exit 0
+    ;;
+  "mcp get")
+    # First few calls return "no entry"; after mcp add fires, return success.
+    if [ -f "${claudeJsonPath}" ]; then
+      cat "${claudeJsonPath}"
+      exit 0
+    fi
+    exit 1
+    ;;
+esac
+exit 0
+`;
+  fs.writeFileSync(path.join(fakeBinDir, 'claude'), script, { mode: 0o755 });
+  return callLog;
+}
+
+describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
+  test('verifies, registers HTTP MCP, never writes token to CLAUDE.md', async () => {
+    const stubServer = await startStubMcpServer();
+    const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-gbrain-remote-'));
+    const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-gbrain-remote-bin-'));
+    const callLog = makeFakeClaude(fakeBinDir);
+
+    // The skill writes CLAUDE.md in cwd. Use gstackHome as cwd so we
+    // can inspect it after the run.
+    fs.writeFileSync(path.join(gstackHome, 'CLAUDE.md'), '# Test project\n');
+
+    const SECRET_TOKEN = 'gbrain_TEST_TOKEN_THAT_MUST_NEVER_LEAK_84613';
+    const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
+    const binary = resolveClaudeBinary();
+
+    // Ambient env mutations. Restored in finally.
+    const orig = {
+      gstackHome: process.env.GSTACK_HOME,
+      pathEnv: process.env.PATH,
+      mcpToken: process.env.GBRAIN_MCP_TOKEN,
+    };
+    process.env.GSTACK_HOME = gstackHome;
+    process.env.PATH = `${fakeBinDir}:${path.join(path.resolve(import.meta.dir, '..'), 'bin')}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
+    process.env.GBRAIN_MCP_TOKEN = SECRET_TOKEN;
+
+    let modelTextOutput = '';
+
+    try {
+      const skillPath = path.resolve(import.meta.dir, '..', 'setup-gbrain', 'SKILL.md');
+      const result = await runAgentSdkTest({
+        systemPrompt: { type: 'preset', preset: 'claude_code' },
+        userPrompt:
+          `Read the skill file at ${skillPath} and follow Path 4 (Remote MCP) only. ` +
+          `Use this MCP URL: ${stubServer.url}. ` +
+          `The bearer token is already in the GBRAIN_MCP_TOKEN env var (do not echo it). ` +
+          `Skip the privacy gate — answer "Decline" if the preamble fires. ` +
+          `Skip the artifacts-repo provisioning step (Step 7) — answer "No thanks". ` +
+          `Skip per-remote policy (Step 6) — answer "skip-for-now". ` +
+          `Walk through Steps 4a, 4b, 4c, 5a, 8, 10 ONLY.`,
+        workingDirectory: gstackHome,
+        maxTurns: 25,
+        allowedTools: ['Read', 'Grep', 'Glob', 'Bash', 'Write', 'Edit'],
+        ...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
+        canUseTool: async (toolName, input) => {
+          if (toolName === 'AskUserQuestion') {
+            askUserQuestions.push({ input });
+            const q = (input.questions as Array<{
+              question: string;
+              options: Array<{ label: string }>;
+            }>)[0];
+            // Auto-decline / skip everything except the path-pick (which the
+            // user-prompt already directed to Path 4).
+            const decline =
+              q.options.find((o) => /skip|decline|no thanks|local/i.test(o.label)) ?? q.options[q.options.length - 1]!;
+            return {
+              behavior: 'allow',
+              updatedInput: {
+                questions: input.questions,
+                answers: { [q.question]: decline.label },
+              },
+            };
+          }
+          return passThroughNonAskUserQuestion(toolName, input);
+        },
+      });
+
+      modelTextOutput = JSON.stringify(result);
+
+      // Assertion 1: no classified failure surfaced.
+      // Match the literal verify-helper field shape (avoid false-positives
+      // from parent session's "needs-auth" MCP server discovery markers).
+      // We can't deterministically force the model to invoke the verify
+      // helper through user-prompt alone, so the bound here is "if verify
+      // ran and emitted an error class, it wasn't NETWORK / AUTH / MALFORMED."
+      expect(modelTextOutput).not.toMatch(/"error_class"\s*:\s*"NETWORK"/);
+      expect(modelTextOutput).not.toMatch(/"error_class"\s*:\s*"AUTH"/);
+      expect(modelTextOutput).not.toMatch(/"error_class"\s*:\s*"MALFORMED"/);
+
+      // Assertion 2: claude mcp add was called with --transport http.
+      const calls = fs.existsSync(callLog) ? fs.readFileSync(callLog, 'utf-8') : '';
+      expect(calls).toMatch(/mcp add.*--transport http/);
+
+      // Assertion 3: the secret token NEVER appears in the final CLAUDE.md.
+      const claudeMd = fs.readFileSync(path.join(gstackHome, 'CLAUDE.md'), 'utf-8');
+      expect(claudeMd).not.toContain(SECRET_TOKEN);
+
+      // Assertion 4: CLAUDE.md got the remote-http block.
+      expect(claudeMd).toMatch(/Mode: remote-http/);
+
+      // Assertion 5: classifier — the model didn't write findings before
+      // asking. The Path 4 prose has 5 STOP gates; if any of them got
+      // skipped, that's the wrote_findings_before_asking pattern.
+      const wroteBefore = /## GSTACK REVIEW REPORT|critical_gaps/i.test(modelTextOutput);
+      // Setup-gbrain doesn't have a review report contract, so this is
+      // a structural shape check, not a hard failure mode.
+      expect(wroteBefore).toBe(false);
+    } finally {
+      if (orig.gstackHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = orig.gstackHome;
+      if (orig.pathEnv === undefined) delete process.env.PATH; else process.env.PATH = orig.pathEnv;
+      if (orig.mcpToken === undefined) delete process.env.GBRAIN_MCP_TOKEN; else process.env.GBRAIN_MCP_TOKEN = orig.mcpToken;
+      await stubServer.close();
+      fs.rmSync(gstackHome, { recursive: true, force: true });
+      fs.rmSync(fakeBinDir, { recursive: true, force: true });
+    }
+  }, 240_000);
+});