Initial MVP: Claude Permission Observability CLI tool

2026-06-16 00:30:52 +00:00
commit 50a2f9058c
14 changed files with 1701 additions and 0 deletions
@@ -0,0 +1,267 @@
+import { test } from 'node:test';
+import assert from 'node:assert';
+import { writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, readdirSync, statSync } from 'fs';
+import { resolve } from 'path';
+import { evaluateAction, getEffectiveRules } from '../rules.js';
+import { loadManifest, validateManifest, createDefaultManifest } from '../config.js';
+import { Auditor } from '../auditor.js';
+import { Reporter, findLatestAuditTrail } from '../reporter.js';
+import type { PermissionManifest, AgentAction } from '../types.js';
+
+const TEST_DIR = resolve('/tmp/cpo-test-' + Date.now());
+
+function setup(): void {
+  if (existsSync(TEST_DIR)) {
+    rmSync(TEST_DIR, { recursive: true });
+  }
+  mkdirSync(TEST_DIR, { recursive: true });
+}
+
+function teardown(): void {
+  if (existsSync(TEST_DIR)) {
+    rmSync(TEST_DIR, { recursive: true });
+  }
+}
+
+const TEST_MANIFEST: PermissionManifest = {
+  name: 'test-manifest',
+  version: '1.0.0',
+  rules: [
+    { type: 'file', pattern: 'src/**/*.ts', action: 'allow' },
+    { type: 'file', pattern: '**/*.env*', action: 'deny' },
+    { type: 'command', pattern: 'git push', action: 'warn' },
+    { type: 'command', pattern: 'rm -rf /', action: 'deny' },
+  ],
+  defaultAction: 'warn',
+  compression: { enabled: true, maxTrailSize: 10, retentionDays: 7 },
+  sensitivePatterns: ['.env', 'secret'],
+};
+
+// --- Config Tests ---
+
+test('validateManifest accepts valid manifest', () => {
+  assert.doesNotThrow(() => validateManifest(TEST_MANIFEST));
+});
+
+test('validateManifest rejects empty name', () => {
+  assert.throws(() => validateManifest({ ...TEST_MANIFEST, name: '' }), /name/);
+});
+
+test('validateManifest rejects invalid rule type', () => {
+  const bad = {
+    ...TEST_MANIFEST,
+    rules: [{ type: 'invalid', pattern: 'x', action: 'allow' }],
+  };
+  assert.throws(() => validateManifest(bad as PermissionManifest), /Invalid rule type/);
+});
+
+test('validateManifest rejects invalid action', () => {
+  const bad = {
+    ...TEST_MANIFEST,
+    rules: [{ type: 'file', pattern: 'x', action: 'invalid' }],
+  };
+  assert.throws(() => validateManifest(bad as PermissionManifest), /Invalid rule action/);
+});
+
+test('loadManifest reads and parses JSON file', () => {
+  setup();
+  const path = resolve(TEST_DIR, 'manifest.json');
+  writeFileSync(path, JSON.stringify(TEST_MANIFEST));
+  const loaded = loadManifest(path);
+  assert.equal(loaded.name, 'test-manifest');
+  assert.equal(loaded.rules.length, 4);
+  teardown();
+});
+
+test('loadManifest throws on missing file', () => {
+  assert.throws(() => loadManifest('/nonexistent/manifest.json'), /not found/);
+});
+
+// --- Rules Tests ---
+
+test('evaluateAction allows matching allow rule', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'file', 'src/main.ts', 'fs.write', 'agent-1');
+  assert.equal(action.resolvedAction, 'allow');
+  assert.equal(action.matchedRule, 'src/**/*.ts');
+});
+
+test('evaluateAction denies matching deny rule', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'file', '.env.local', 'fs.read', 'agent-1');
+  assert.equal(action.resolvedAction, 'deny');
+  assert.equal(action.matchedRule, '**/*.env*');
+});
+
+test('evaluateAction warns on default for non-matching paths', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'file', 'README.md', 'fs.read', 'agent-1');
+  assert.equal(action.resolvedAction, 'warn');
+});
+
+test('evaluateAction detects sensitive patterns as bypass', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'file', 'config/secrets.json', 'fs.read', 'agent-1');
+  assert.equal(action.type, 'bypass_attempt');
+  assert.equal(action.resolvedAction, 'warn');
+  assert.ok(action.matchedRule?.includes('sensitive-pattern'));
+});
+
+test('evaluateAction matches command patterns', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'command', 'git push origin main', 'git.push', 'agent-1');
+  assert.equal(action.resolvedAction, 'warn');
+  assert.equal(action.matchedRule, 'git push');
+});
+
+test('evaluateAction denies dangerous commands', () => {
+  const action = evaluateAction(TEST_MANIFEST, 'command', 'rm -rf /', 'rm', 'agent-1');
+  assert.equal(action.resolvedAction, 'deny');
+});
+
+test('getEffectiveRules returns all rules when no type filter', () => {
+  const rules = getEffectiveRules(TEST_MANIFEST);
+  assert.equal(rules.length, 4);
+});
+
+test('getEffectiveRules filters by type', () => {
+  const rules = getEffectiveRules(TEST_MANIFEST, 'command');
+  assert.equal(rules.length, 2);
+  assert.equal(rules[0].type, 'command');
+});
+
+// --- Auditor Tests ---
+
+test('Auditor logs actions and produces summary', () => {
+  setup();
+  const auditor = new Auditor(TEST_MANIFEST, TEST_DIR);
+  auditor.log(makeAction('allow', 'file', 'src/main.ts'));
+  auditor.log(makeAction('deny', 'file', '.env'));
+  auditor.log(makeAction('warn', 'command', 'git push'));
+
+  const trail = auditor.getTrail();
+  assert.equal(trail.summary.total, 3);
+  assert.equal(trail.summary.allowed, 1);
+  assert.equal(trail.summary.denied, 1);
+  assert.equal(trail.summary.warned, 1);
+
+  auditor.close();
+  teardown();
+});
+
+test('Auditor writes JSONL to file', () => {
+  setup();
+  const auditor = new Auditor(TEST_MANIFEST, TEST_DIR);
+  auditor.log(makeAction('allow', 'file', 'src/main.ts'));
+  auditor.close();
+
+  const files = existsSync(TEST_DIR) ? readdirSync(TEST_DIR) : [];
+  const jsonl = files.find((f: string) => f.endsWith('.jsonl'));
+  assert.ok(jsonl, 'JSONL file should exist');
+  teardown();
+});
+
+test('Auditor.loadFromFile reads back trail', () => {
+  setup();
+  const auditor = new Auditor(TEST_MANIFEST, TEST_DIR);
+  auditor.log(makeAction('deny', 'file', '.env'));
+  auditor.close();
+
+  const files = readdirSync(TEST_DIR).filter((f: string) => f.endsWith('.jsonl'));
+  const path = resolve(TEST_DIR, files[0]);
+  const loaded = Auditor.loadFromFile(path);
+  assert.equal(loaded.summary.total, 1);
+  assert.equal(loaded.summary.denied, 1);
+  teardown();
+});
+
+test('Auditor counts bypass attempts', () => {
+  setup();
+  const auditor = new Auditor(TEST_MANIFEST, TEST_DIR);
+  auditor.log(makeAction('deny', 'file', '.env'));
+  auditor.log(makeAction('warn', 'bypass_attempt', 'secrets.txt'));
+  assert.equal(auditor.getBypassCount(), 2);
+  teardown();
+});
+
+// --- Reporter Tests ---
+
+test('Reporter generates violation report', () => {
+  const trail: import('../types.js').AuditTrail = {
+    manifest: 'test',
+    startedAt: new Date(),
+    actions: [
+      makeAction('allow', 'file', 'src/main.ts'),
+      makeAction('deny', 'file', '.env'),
+      makeAction('deny', 'command', 'rm -rf /'),
+    ],
+    summary: { total: 3, allowed: 1, denied: 2, warned: 0, bypassAttempts: 2 },
+  };
+
+  const reporter = new Reporter();
+  const report = reporter.generateViolationReport(trail);
+  assert.equal(report.violations.length, 2);
+  assert.ok(report.riskScore > 0);
+  assert.equal(report.topViolatedRules.length, 2);
+});
+
+test('Reporter formatConsoleReport includes headers and data', () => {
+  const trail: import('../types.js').AuditTrail = {
+    manifest: 'test',
+    startedAt: new Date(),
+    actions: [],
+    summary: { total: 0, allowed: 0, denied: 0, warned: 0, bypassAttempts: 0 },
+  };
+
+  const reporter = new Reporter();
+  const report = reporter.generateViolationReport(trail);
+  const output = reporter.formatConsoleReport(report);
+  assert.ok(output.includes('SAFE'));
+  assert.ok(output.includes('No violations'));
+});
+
+test('Reporter formatJsonReport produces valid JSON', () => {
+  const trail: import('../types.js').AuditTrail = {
+    manifest: 'test',
+    startedAt: new Date(),
+    actions: [makeAction('deny', 'file', '.env')],
+    summary: { total: 1, allowed: 0, denied: 1, warned: 0, bypassAttempts: 1 },
+  };
+
+  const reporter = new Reporter();
+  const report = reporter.generateViolationReport(trail);
+  const json = reporter.formatJsonReport(report);
+  const parsed = JSON.parse(json);
+  assert.equal(parsed.violations.length, 1);
+  assert.equal(parsed.riskScore, report.riskScore);
+});
+
+test('findLatestAuditTrail returns most recent file', async () => {
+  setup();
+  writeFileSync(resolve(TEST_DIR, 'audit-1000.jsonl'), '');
+  await new Promise(r => setTimeout(r, 10));
+  writeFileSync(resolve(TEST_DIR, 'audit-2000.jsonl'), '');
+  await new Promise(r => setTimeout(r, 10));
+  writeFileSync(resolve(TEST_DIR, 'audit-3000.jsonl'), '');
+
+  const latest = findLatestAuditTrail(TEST_DIR);
+  assert.ok(latest?.includes('audit-3000'));
+  teardown();
+});
+
+test('findLatestAuditTrail returns null for empty dir', () => {
+  setup();
+  const result = findLatestAuditTrail(TEST_DIR);
+  assert.equal(result, null);
+  teardown();
+});
+
+// --- Helper ---
+
+function makeAction(resolved: 'allow' | 'deny' | 'warn', type: AgentAction['type'], target: string): AgentAction {
+  return {
+    id: `test-${Date.now()}-${Math.random()}`,
+    timestamp: new Date(),
+    agentId: 'test-agent',
+    type,
+    action: `test.${type}`,
+    target,
+    resolvedAction: resolved,
+    matchedRule: target,
+  };
+}