From cdf1b03779abe28b9ea47f8060e1336107d6b0a3 Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Tue, 12 May 2026 02:59:52 -0400 Subject: [PATCH] docs: add data-backed harness adapter scorecard (#1785) * docs: add data-backed harness adapter scorecard * fix: normalize adapter matrix line endings * test: avoid doubled CRLF simulation --- docs/ECC-2.0-GA-ROADMAP.md | 15 +- .../harness-adapter-compliance.md | 40 +- package.json | 2 + scripts/harness-adapter-compliance.js | 149 ++++++ scripts/lib/harness-adapter-compliance.js | 446 ++++++++++++++++++ tests/docs/harness-adapter-compliance.test.js | 60 ++- tests/scripts/npm-publish-surface.test.js | 1 + 7 files changed, 688 insertions(+), 25 deletions(-) create mode 100644 scripts/harness-adapter-compliance.js create mode 100644 scripts/lib/harness-adapter-compliance.js diff --git a/docs/ECC-2.0-GA-ROADMAP.md b/docs/ECC-2.0-GA-ROADMAP.md index a3e5430f..d7989e4b 100644 --- a/docs/ECC-2.0-GA-ROADMAP.md +++ b/docs/ECC-2.0-GA-ROADMAP.md @@ -24,6 +24,9 @@ As of 2026-05-12: OpenCode, Cursor, Gemini, Zed-adjacent, dmux, Orca, Superset, Ghast, and terminal-only support to install paths, verification commands, and risk notes. +- `npm run harness:adapters -- --check` validates that the public adapter + matrix still matches the source data in + `scripts/lib/harness-adapter-compliance.js`. - AgentShield PR #53 reduced two context-rule false positives and closed the remaining AgentShield issues. - ECC PR #1778 recovered the useful stale #1413 network/homelab architect-agent @@ -179,14 +182,12 @@ Acceptance: ## Next Engineering Slices -1. Move the harness adapter compliance matrix from Markdown to a data-backed - validator. -2. Add the release/name/plugin publication checklist with evidence fields. -3. Start AgentShield enterprise policy schema and SARIF implementation in the +1. Add the release/name/plugin publication checklist with evidence fields. +2. Start AgentShield enterprise policy schema and SARIF implementation in the AgentShield repo. -4. Audit ECC Tools billing and check-run surfaces before any native GitHub +3. Audit ECC Tools billing and check-run surfaces before any native GitHub payments announcement. -5. Inventory `_legacy-documents-*` and map useful artifacts to landed, +4. Inventory `_legacy-documents-*` and map useful artifacts to landed, milestone-tracked, salvage, or archive states. -6. Build the stale-PR salvage ledger from closed cleanup batches, then port +5. Build the stale-PR salvage ledger from closed cleanup batches, then port useful pieces in small attributed maintainer PRs. diff --git a/docs/architecture/harness-adapter-compliance.md b/docs/architecture/harness-adapter-compliance.md index 55bd794e..d557fa51 100644 --- a/docs/architecture/harness-adapter-compliance.md +++ b/docs/architecture/harness-adapter-compliance.md @@ -29,19 +29,25 @@ or platform limits. ## Matrix +The matrix below is rendered from +`scripts/lib/harness-adapter-compliance.js` and verified by +`npm run harness:adapters -- --check`. + + | Harness or runtime | State | Supported assets | Unsupported or different surfaces | Install or onramp | Verification command | Risk notes | | --- | --- | --- | --- | --- | --- | --- | -| Claude Code | Native | Claude plugin assets, skills, commands, hooks, MCP config, local rules, statusline-oriented workflows | Claude-native hooks do not imply parity in other harnesses | `./install.sh --profile minimal --target claude` or Claude plugin install | `npm run harness:audit -- --format json` and `node scripts/session-inspect.js --list-adapters` | Avoid loading every skill by default; keep hooks opt-in and inspectable. | -| Codex | Instruction-backed | `AGENTS.md`, Codex plugin metadata, skills, MCP reference config, command patterns | Native hook enforcement and Claude slash-command semantics are not equivalent | `./install.sh --profile minimal --target codex` plus repo-local `AGENTS.md` review | `npm run harness:audit -- --format json` | Treat hooks as policy text unless a native Codex hook surface exists. | -| OpenCode | Adapter-backed | OpenCode package/plugin metadata, shared skills, MCP config, event adapter patterns | Event names, plugin packaging, and command dispatch differ from Claude Code | OpenCode package or plugin surface from this repo | `node tests/scripts/build-opencode.test.js` and `npm run harness:audit -- --format json` | Keep hook logic in shared scripts and adapt only event shape at the edge. | -| Cursor | Adapter-backed | Cursor rules, project-local skills, hook adapter, shared scripts | Cursor hook events and rule loading differ from Claude Code | `./install.sh --profile minimal --target cursor` | `node tests/lib/install-targets.test.js` and `npm run harness:audit -- --format json` | Cursor adapters must preserve existing project rules and avoid silent overwrite. | -| Gemini | Instruction-backed | Gemini project-local instructions, shared skills, rules, compatibility docs | No full ECC hook parity; ecosystem ports must document drift from upstream ECC | `./install.sh --profile minimal --target gemini` | `node tests/lib/install-targets.test.js` | Treat Gemini ports as ecosystem adapters until validated end to end inside Gemini CLI. | -| Zed-adjacent workflows | Instruction-backed | Shared skills, `AGENTS.md` style project instructions, verification loops | Zed agent surfaces vary; no first-party ECC installer is shipped today | Manual copy from shared ECC sources until adapter requirements settle | `npm run harness:audit -- --format json` | Do not claim native Zed support before a real adapter and verification path exist. | -| dmux | Adapter-backed | Session snapshots, tmux/worktree orchestration status, handoff exports | dmux is an orchestration runtime, not an install target for skills/rules | `node scripts/session-inspect.js --list-adapters` and dmux session target inspection | `node tests/lib/session-adapters.test.js` | Treat dmux events as session/runtime signals, not as a replacement for repo validation. | -| Orca | Reference-only | Worktree lifecycle, review state, notification, and provider-identity design pressure | No ECC installer or direct adapter today | Use as a comparison target for worktree/session state requirements | `npm run observability:ready` | Do not import product-specific assumptions; convert lessons into ECC event fields. | -| Superset | Reference-only | Workspace presets, parallel-agent review loops, worktree isolation design pressure | No ECC installer or direct adapter today | Use as a comparison target for workspace preset taxonomy | `npm run observability:ready` | Keep ECC portable; do not require a desktop workspace to get basic value. | -| Ghast | Reference-only | Terminal-native pane grouping, cwd grouping, search, notifications | No ECC installer or direct adapter today | Use as a comparison target for terminal-first session grouping | `node scripts/session-inspect.js --list-adapters` | Preserve terminal ergonomics before adding visual UI assumptions. | -| Terminal-only | Native | Skills, rules, commands, scripts, harness audit, observability readiness, handoffs | No external UI, no automatic session control unless scripts are run explicitly | Clone repo, run commands directly, use minimal profile for project installs | `npm run harness:audit -- --format json` and `npm run observability:ready` | This is the fallback contract; every higher-level adapter should degrade to it. | +| Claude Code | Native | Claude plugin assets; skills; commands; hooks; MCP config; local rules; statusline-oriented workflows | Claude-native hooks do not imply parity in other harnesses | `./install.sh --profile minimal --target claude`; Claude plugin install | `npm run harness:audit -- --format json`; `node scripts/session-inspect.js --list-adapters` | Avoid loading every skill by default; keep hooks opt-in and inspectable. | +| Codex | Instruction-backed | `AGENTS.md`; Codex plugin metadata; skills; MCP reference config; command patterns | Native hook enforcement and Claude slash-command semantics are not equivalent | `./install.sh --profile minimal --target codex`; repo-local `AGENTS.md` review | `npm run harness:audit -- --format json` | Treat hooks as policy text unless a native Codex hook surface exists. | +| OpenCode | Adapter-backed | OpenCode package/plugin metadata; shared skills; MCP config; event adapter patterns | Event names, plugin packaging, and command dispatch differ from Claude Code | OpenCode package or plugin surface from this repo | `node tests/scripts/build-opencode.test.js`; `npm run harness:audit -- --format json` | Keep hook logic in shared scripts and adapt only event shape at the edge. | +| Cursor | Adapter-backed | Cursor rules; project-local skills; hook adapter; shared scripts | Cursor hook events and rule loading differ from Claude Code | `./install.sh --profile minimal --target cursor` | `node tests/lib/install-targets.test.js`; `npm run harness:audit -- --format json` | Cursor adapters must preserve existing project rules and avoid silent overwrite. | +| Gemini | Instruction-backed | Gemini project-local instructions; shared skills; rules; compatibility docs | No full ECC hook parity; ecosystem ports must document drift from upstream ECC | `./install.sh --profile minimal --target gemini` | `node tests/lib/install-targets.test.js` | Treat Gemini ports as ecosystem adapters until validated end to end inside Gemini CLI. | +| Zed-adjacent workflows | Instruction-backed | shared skills; `AGENTS.md` style project instructions; verification loops | Zed agent surfaces vary; no first-party ECC installer is shipped today | Manual copy from shared ECC sources until adapter requirements settle | `npm run harness:audit -- --format json` | Do not claim native Zed support before a real adapter and verification path exist. | +| dmux | Adapter-backed | session snapshots; tmux/worktree orchestration status; handoff exports | dmux is an orchestration runtime, not an install target for skills/rules | `node scripts/session-inspect.js --list-adapters`; dmux session target inspection | `node tests/lib/session-adapters.test.js` | Treat dmux events as session/runtime signals, not as a replacement for repo validation. | +| Orca | Reference-only | worktree lifecycle; review state; notification; provider-identity design pressure | No ECC installer or direct adapter today | Use as a comparison target for worktree/session state requirements | `npm run observability:ready` | Do not import product-specific assumptions; convert lessons into ECC event fields. | +| Superset | Reference-only | workspace presets; parallel-agent review loops; worktree isolation design pressure | No ECC installer or direct adapter today | Use as a comparison target for workspace preset taxonomy | `npm run observability:ready` | Keep ECC portable; do not require a desktop workspace to get basic value. | +| Ghast | Reference-only | terminal-native pane grouping; cwd grouping; search; notifications | No ECC installer or direct adapter today | Use as a comparison target for terminal-first session grouping | `node scripts/session-inspect.js --list-adapters` | Preserve terminal ergonomics before adding visual UI assumptions. | +| Terminal-only | Native | skills; rules; commands; scripts; harness audit; observability readiness; handoffs | No external UI, no automatic session control unless scripts are run explicitly | Clone repo; run commands directly; use minimal profile for project installs | `npm run harness:audit -- --format json`; `npm run observability:ready` | This is the fallback contract; every higher-level adapter should degrade to it. | + ## Scorecard Onramp @@ -49,6 +55,7 @@ Use this sequence before asking ECC to make a team or repo setup more autonomous: ```bash +npm run harness:adapters -- --check npm run harness:audit -- --format json npm run observability:ready node scripts/session-inspect.js --list-adapters @@ -57,6 +64,8 @@ node scripts/loop-status.js --json --write-dir .ecc/loop-status Read the result as a setup scorecard, not a product badge: +- `harness:adapters -- --check` proves this public matrix still matches the + adapter source data and required evidence fields. - `harness:audit` scores tool coverage, context efficiency, quality gates, memory persistence, eval coverage, security guardrails, and cost efficiency. - `observability:ready` proves the repo still exposes the local status, @@ -66,10 +75,9 @@ Read the result as a setup scorecard, not a product badge: - `loop-status --json` creates a machine-readable handoff/status payload for longer autonomous runs. -## Evidence Fields For Future Data Matrix +## Data-Backed Scorecard Contract -When this matrix moves from Markdown to a data-backed validator, each adapter -record should expose: +Each adapter record exposes: - `id` - `state` @@ -82,8 +90,8 @@ record should expose: - `owner` - `source_docs` -The validator should fail if a public adapter claim has no install path, -verification command, or risk note. +The validator fails if a public adapter claim has no install path, +verification command, risk note, owner, source doc, or verification date. ## Operating Rules diff --git a/package.json b/package.json index a56b27be..84715ac4 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "scripts/doctor.js", "scripts/ecc.js", "scripts/gemini-adapt-agents.js", + "scripts/harness-adapter-compliance.js", "scripts/harness-audit.js", "scripts/observability-readiness.js", "scripts/hooks/", @@ -276,6 +277,7 @@ "catalog:check": "node scripts/ci/catalog.js --text", "catalog:sync": "node scripts/ci/catalog.js --write --text", "lint": "eslint . && markdownlint '**/*.md' --ignore node_modules", + "harness:adapters": "node scripts/harness-adapter-compliance.js", "harness:audit": "node scripts/harness-audit.js", "observability:ready": "node scripts/observability-readiness.js", "claw": "node scripts/claw.js", diff --git a/scripts/harness-adapter-compliance.js b/scripts/harness-adapter-compliance.js new file mode 100644 index 00000000..f2e5f9eb --- /dev/null +++ b/scripts/harness-adapter-compliance.js @@ -0,0 +1,149 @@ +#!/usr/bin/env node +'use strict'; + +const path = require('path'); +const { + ADAPTER_RECORDS, + renderMarkdownTable, + validateAdapterRecords, + validateDocumentation, +} = require('./lib/harness-adapter-compliance'); + +function parseArgs(argv) { + const args = argv.slice(2); + const parsed = { + check: false, + format: 'text', + help: false, + root: process.cwd(), + }; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + + if (arg === '--help' || arg === '-h') { + parsed.help = true; + continue; + } + + if (arg === '--check') { + parsed.check = true; + continue; + } + + if (arg === '--format') { + parsed.format = String(args[index + 1] || '').toLowerCase(); + index += 1; + continue; + } + + if (arg.startsWith('--format=')) { + parsed.format = arg.slice('--format='.length).toLowerCase(); + continue; + } + + if (arg === '--root') { + parsed.root = path.resolve(args[index + 1] || process.cwd()); + index += 1; + continue; + } + + if (arg.startsWith('--root=')) { + parsed.root = path.resolve(arg.slice('--root='.length)); + continue; + } + + throw new Error(`Unknown argument: ${arg}`); + } + + if (!['text', 'json', 'markdown'].includes(parsed.format)) { + throw new Error(`Invalid format: ${parsed.format}. Use text, json, or markdown.`); + } + + parsed.root = path.resolve(parsed.root); + return parsed; +} + +function printHelp() { + console.log([ + 'Usage: node scripts/harness-adapter-compliance.js [options]', + '', + 'Validate or render the ECC harness adapter compliance scorecard.', + '', + 'Options:', + ' --check Fail if adapter records or docs are out of sync', + ' --format ', + ' --root Repository root, defaults to cwd', + ' -h, --help Show this help', + ].join('\n')); +} + +function buildPayload(root) { + const recordErrors = validateAdapterRecords(); + const documentationErrors = validateDocumentation({ repoRoot: root }); + + return { + schema_version: 'ecc.harness-adapter-compliance.v1', + generated_from: 'scripts/lib/harness-adapter-compliance.js', + adapter_count: ADAPTER_RECORDS.length, + valid: recordErrors.length === 0 && documentationErrors.length === 0, + errors: [...recordErrors, ...documentationErrors], + adapters: ADAPTER_RECORDS, + }; +} + +function renderText(payload) { + const lines = [ + `Harness Adapter Compliance: ${payload.valid ? 'PASS' : 'FAIL'}`, + `Adapters: ${payload.adapter_count}`, + ]; + + if (payload.errors.length > 0) { + lines.push('Errors:'); + for (const error of payload.errors) { + lines.push(`- ${error}`); + } + } + + return lines.join('\n'); +} + +function main() { + let parsed; + + try { + parsed = parseArgs(process.argv); + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + + if (parsed.help) { + printHelp(); + return; + } + + const payload = buildPayload(parsed.root); + + if (parsed.format === 'json') { + console.log(JSON.stringify(payload, null, 2)); + } else if (parsed.format === 'markdown') { + console.log(renderMarkdownTable()); + } else { + console.log(renderText(payload)); + } + + if (parsed.check && !payload.valid) { + process.exit(1); + } +} + +if (require.main === module) { + main(); +} + +module.exports = { + buildPayload, + parseArgs, +}; + diff --git a/scripts/lib/harness-adapter-compliance.js b/scripts/lib/harness-adapter-compliance.js new file mode 100644 index 00000000..22f09cb6 --- /dev/null +++ b/scripts/lib/harness-adapter-compliance.js @@ -0,0 +1,446 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const MATRIX_BLOCK_START = ''; +const MATRIX_BLOCK_END = ''; + +const COMPLIANCE_STATES = Object.freeze({ + Native: 'ECC can install or verify the surface directly for this harness.', + 'Adapter-backed': 'ECC has a thin adapter, plugin, or package surface, but parity differs by harness.', + 'Instruction-backed': 'ECC can provide the guidance and files, but the harness does not expose the runtime hook/session surface ECC needs for enforcement.', + 'Reference-only': 'The tool is useful as a design pressure or external runtime, but ECC does not yet ship a direct installer or adapter for it.', +}); + +const REQUIRED_FIELDS = Object.freeze([ + 'id', + 'harness', + 'state', + 'supported_assets', + 'unsupported_surfaces', + 'install_or_onramp', + 'verification_commands', + 'risk_notes', + 'last_verified_at', + 'owner', + 'source_docs', +]); + +function freezeRecord(record) { + return Object.freeze({ + ...record, + supported_assets: Object.freeze(record.supported_assets.slice()), + unsupported_surfaces: Object.freeze(record.unsupported_surfaces.slice()), + install_or_onramp: Object.freeze(record.install_or_onramp.slice()), + verification_commands: Object.freeze(record.verification_commands.slice()), + risk_notes: Object.freeze(record.risk_notes.slice()), + source_docs: Object.freeze(record.source_docs.slice()), + }); +} + +const ADAPTER_RECORDS = Object.freeze([ + { + id: 'claude-code', + harness: 'Claude Code', + state: 'Native', + supported_assets: [ + 'Claude plugin assets', + 'skills', + 'commands', + 'hooks', + 'MCP config', + 'local rules', + 'statusline-oriented workflows', + ], + unsupported_surfaces: ['Claude-native hooks do not imply parity in other harnesses'], + install_or_onramp: [ + '`./install.sh --profile minimal --target claude`', + 'Claude plugin install', + ], + verification_commands: [ + '`npm run harness:audit -- --format json`', + '`node scripts/session-inspect.js --list-adapters`', + ], + risk_notes: ['Avoid loading every skill by default; keep hooks opt-in and inspectable.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + '.claude-plugin/plugin.json', + 'docs/architecture/cross-harness.md', + 'scripts/lib/install-targets/claude-home.js', + ], + }, + { + id: 'codex', + harness: 'Codex', + state: 'Instruction-backed', + supported_assets: [ + '`AGENTS.md`', + 'Codex plugin metadata', + 'skills', + 'MCP reference config', + 'command patterns', + ], + unsupported_surfaces: ['Native hook enforcement and Claude slash-command semantics are not equivalent'], + install_or_onramp: [ + '`./install.sh --profile minimal --target codex`', + 'repo-local `AGENTS.md` review', + ], + verification_commands: ['`npm run harness:audit -- --format json`'], + risk_notes: ['Treat hooks as policy text unless a native Codex hook surface exists.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + '.codex-plugin/plugin.json', + 'AGENTS.md', + 'scripts/lib/install-targets/codex-home.js', + ], + }, + { + id: 'opencode', + harness: 'OpenCode', + state: 'Adapter-backed', + supported_assets: [ + 'OpenCode package/plugin metadata', + 'shared skills', + 'MCP config', + 'event adapter patterns', + ], + unsupported_surfaces: ['Event names, plugin packaging, and command dispatch differ from Claude Code'], + install_or_onramp: ['OpenCode package or plugin surface from this repo'], + verification_commands: [ + '`node tests/scripts/build-opencode.test.js`', + '`npm run harness:audit -- --format json`', + ], + risk_notes: ['Keep hook logic in shared scripts and adapt only event shape at the edge.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + '.opencode/package.json', + '.opencode/plugins/ecc-hooks.ts', + 'scripts/build-opencode.js', + ], + }, + { + id: 'cursor', + harness: 'Cursor', + state: 'Adapter-backed', + supported_assets: [ + 'Cursor rules', + 'project-local skills', + 'hook adapter', + 'shared scripts', + ], + unsupported_surfaces: ['Cursor hook events and rule loading differ from Claude Code'], + install_or_onramp: ['`./install.sh --profile minimal --target cursor`'], + verification_commands: [ + '`node tests/lib/install-targets.test.js`', + '`npm run harness:audit -- --format json`', + ], + risk_notes: ['Cursor adapters must preserve existing project rules and avoid silent overwrite.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + '.cursor/', + 'scripts/lib/install-targets/cursor-project.js', + 'tests/lib/install-targets.test.js', + ], + }, + { + id: 'gemini', + harness: 'Gemini', + state: 'Instruction-backed', + supported_assets: [ + 'Gemini project-local instructions', + 'shared skills', + 'rules', + 'compatibility docs', + ], + unsupported_surfaces: ['No full ECC hook parity; ecosystem ports must document drift from upstream ECC'], + install_or_onramp: ['`./install.sh --profile minimal --target gemini`'], + verification_commands: ['`node tests/lib/install-targets.test.js`'], + risk_notes: ['Treat Gemini ports as ecosystem adapters until validated end to end inside Gemini CLI.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + '.gemini/', + 'scripts/lib/install-targets/gemini-project.js', + 'tests/lib/install-targets.test.js', + ], + }, + { + id: 'zed-adjacent', + harness: 'Zed-adjacent workflows', + state: 'Instruction-backed', + supported_assets: [ + 'shared skills', + '`AGENTS.md` style project instructions', + 'verification loops', + ], + unsupported_surfaces: ['Zed agent surfaces vary; no first-party ECC installer is shipped today'], + install_or_onramp: ['Manual copy from shared ECC sources until adapter requirements settle'], + verification_commands: ['`npm run harness:audit -- --format json`'], + risk_notes: ['Do not claim native Zed support before a real adapter and verification path exist.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + 'AGENTS.md', + 'docs/architecture/cross-harness.md', + ], + }, + { + id: 'dmux', + harness: 'dmux', + state: 'Adapter-backed', + supported_assets: [ + 'session snapshots', + 'tmux/worktree orchestration status', + 'handoff exports', + ], + unsupported_surfaces: ['dmux is an orchestration runtime, not an install target for skills/rules'], + install_or_onramp: [ + '`node scripts/session-inspect.js --list-adapters`', + 'dmux session target inspection', + ], + verification_commands: ['`node tests/lib/session-adapters.test.js`'], + risk_notes: ['Treat dmux events as session/runtime signals, not as a replacement for repo validation.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + 'scripts/lib/session-adapters/dmux-tmux.js', + 'scripts/orchestration-status.js', + 'tests/lib/session-adapters.test.js', + ], + }, + { + id: 'orca', + harness: 'Orca', + state: 'Reference-only', + supported_assets: [ + 'worktree lifecycle', + 'review state', + 'notification', + 'provider-identity design pressure', + ], + unsupported_surfaces: ['No ECC installer or direct adapter today'], + install_or_onramp: ['Use as a comparison target for worktree/session state requirements'], + verification_commands: ['`npm run observability:ready`'], + risk_notes: ['Do not import product-specific assumptions; convert lessons into ECC event fields.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: ['docs/architecture/cross-harness.md'], + }, + { + id: 'superset', + harness: 'Superset', + state: 'Reference-only', + supported_assets: [ + 'workspace presets', + 'parallel-agent review loops', + 'worktree isolation design pressure', + ], + unsupported_surfaces: ['No ECC installer or direct adapter today'], + install_or_onramp: ['Use as a comparison target for workspace preset taxonomy'], + verification_commands: ['`npm run observability:ready`'], + risk_notes: ['Keep ECC portable; do not require a desktop workspace to get basic value.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: ['docs/architecture/cross-harness.md'], + }, + { + id: 'ghast', + harness: 'Ghast', + state: 'Reference-only', + supported_assets: [ + 'terminal-native pane grouping', + 'cwd grouping', + 'search', + 'notifications', + ], + unsupported_surfaces: ['No ECC installer or direct adapter today'], + install_or_onramp: ['Use as a comparison target for terminal-first session grouping'], + verification_commands: ['`node scripts/session-inspect.js --list-adapters`'], + risk_notes: ['Preserve terminal ergonomics before adding visual UI assumptions.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: ['docs/architecture/cross-harness.md'], + }, + { + id: 'terminal-only', + harness: 'Terminal-only', + state: 'Native', + supported_assets: [ + 'skills', + 'rules', + 'commands', + 'scripts', + 'harness audit', + 'observability readiness', + 'handoffs', + ], + unsupported_surfaces: ['No external UI, no automatic session control unless scripts are run explicitly'], + install_or_onramp: [ + 'Clone repo', + 'run commands directly', + 'use minimal profile for project installs', + ], + verification_commands: [ + '`npm run harness:audit -- --format json`', + '`npm run observability:ready`', + ], + risk_notes: ['This is the fallback contract; every higher-level adapter should degrade to it.'], + last_verified_at: '2026-05-12', + owner: 'ECC maintainers', + source_docs: [ + 'scripts/harness-audit.js', + 'scripts/observability-readiness.js', + 'docs/architecture/observability-readiness.md', + ], + }, +].map(freezeRecord)); + +function toTextList(value) { + return Array.isArray(value) ? value.join('; ') : String(value || ''); +} + +function escapeMarkdownCell(value) { + return toTextList(value).replace(/\|/g, '\\|').trim(); +} + +function renderMarkdownTable(records = ADAPTER_RECORDS) { + const lines = [ + '| Harness or runtime | State | Supported assets | Unsupported or different surfaces | Install or onramp | Verification command | Risk notes |', + '| --- | --- | --- | --- | --- | --- | --- |', + ]; + + for (const record of records) { + lines.push([ + record.harness, + record.state, + record.supported_assets, + record.unsupported_surfaces, + record.install_or_onramp, + record.verification_commands, + record.risk_notes, + ].map(escapeMarkdownCell).join(' | ').replace(/^/, '| ').replace(/$/, ' |')); + } + + return lines.join('\n'); +} + +function renderStateTable() { + const lines = [ + '| State | Meaning |', + '| --- | --- |', + ]; + + for (const [state, meaning] of Object.entries(COMPLIANCE_STATES)) { + lines.push(`| ${escapeMarkdownCell(state)} | ${escapeMarkdownCell(meaning)} |`); + } + + return lines.join('\n'); +} + +function validateAdapterRecords(records = ADAPTER_RECORDS) { + const errors = []; + const ids = new Set(); + + records.forEach((record, index) => { + const label = record?.id || `record[${index}]`; + + for (const field of REQUIRED_FIELDS) { + if (!Object.prototype.hasOwnProperty.call(record, field)) { + errors.push(`${label}: missing required field ${field}`); + } + } + + if (typeof record.id !== 'string' || !/^[a-z0-9-]+$/.test(record.id)) { + errors.push(`${label}: id must be a lowercase slug`); + } else if (ids.has(record.id)) { + errors.push(`${label}: duplicate id`); + } else { + ids.add(record.id); + } + + if (!Object.prototype.hasOwnProperty.call(COMPLIANCE_STATES, record.state)) { + errors.push(`${label}: unknown state ${record.state}`); + } + + for (const field of [ + 'supported_assets', + 'unsupported_surfaces', + 'install_or_onramp', + 'verification_commands', + 'risk_notes', + 'source_docs', + ]) { + if (!Array.isArray(record[field]) || record[field].length === 0) { + errors.push(`${label}: ${field} must be a non-empty array`); + continue; + } + + record[field].forEach((value, valueIndex) => { + if (typeof value !== 'string' || !value.trim()) { + errors.push(`${label}: ${field}[${valueIndex}] must be a non-empty string`); + } + }); + } + + if (typeof record.harness !== 'string' || !record.harness.trim()) { + errors.push(`${label}: harness must be a non-empty string`); + } + + if (typeof record.owner !== 'string' || !record.owner.trim()) { + errors.push(`${label}: owner must be a non-empty string`); + } + + if (typeof record.last_verified_at !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(record.last_verified_at)) { + errors.push(`${label}: last_verified_at must be YYYY-MM-DD`); + } + }); + + return errors; +} + +function extractMatrixBlock(markdown) { + const normalized = String(markdown).replace(/\r\n/g, '\n'); + const start = normalized.indexOf(MATRIX_BLOCK_START); + const end = normalized.indexOf(MATRIX_BLOCK_END); + + if (start < 0 || end < 0 || end <= start) { + return null; + } + + return normalized.slice(start + MATRIX_BLOCK_START.length, end).trim(); +} + +function validateDocumentation(options = {}) { + const repoRoot = options.repoRoot || path.resolve(__dirname, '..', '..'); + const docPath = options.docPath || path.join(repoRoot, 'docs', 'architecture', 'harness-adapter-compliance.md'); + const errors = []; + const source = fs.readFileSync(docPath, 'utf8'); + const actual = extractMatrixBlock(source); + const expected = renderMarkdownTable(); + + if (actual === null) { + errors.push(`missing matrix block markers in ${path.relative(repoRoot, docPath)}`); + } else if (actual !== expected) { + errors.push(`matrix block in ${path.relative(repoRoot, docPath)} is not generated from adapter records`); + } + + return errors; +} + +module.exports = { + ADAPTER_RECORDS, + COMPLIANCE_STATES, + MATRIX_BLOCK_END, + MATRIX_BLOCK_START, + REQUIRED_FIELDS, + extractMatrixBlock, + renderMarkdownTable, + renderStateTable, + validateAdapterRecords, + validateDocumentation, +}; diff --git a/tests/docs/harness-adapter-compliance.test.js b/tests/docs/harness-adapter-compliance.test.js index a1dc0c62..7a140f73 100644 --- a/tests/docs/harness-adapter-compliance.test.js +++ b/tests/docs/harness-adapter-compliance.test.js @@ -1,10 +1,18 @@ 'use strict'; const assert = require('assert'); +const { execFileSync } = require('child_process'); const fs = require('fs'); const path = require('path'); +const { + ADAPTER_RECORDS, + extractMatrixBlock, + renderMarkdownTable, + validateAdapterRecords, +} = require('../../scripts/lib/harness-adapter-compliance'); const repoRoot = path.resolve(__dirname, '..', '..'); +const scriptPath = path.join(repoRoot, 'scripts', 'harness-adapter-compliance.js'); let passed = 0; let failed = 0; @@ -46,6 +54,29 @@ test('adapter compliance matrix covers the required harness surfaces', () => { } }); +test('adapter compliance source data validates required evidence fields', () => { + assert.deepStrictEqual(validateAdapterRecords(), []); + + for (const record of ADAPTER_RECORDS) { + assert.ok(record.install_or_onramp.length > 0, `${record.id} needs an install or onramp`); + assert.ok(record.verification_commands.length > 0, `${record.id} needs verification commands`); + assert.ok(record.risk_notes.length > 0, `${record.id} needs risk notes`); + assert.ok(record.source_docs.length > 0, `${record.id} needs source docs`); + } +}); + +test('adapter compliance matrix is generated from source data', () => { + const source = read('docs/architecture/harness-adapter-compliance.md'); + assert.strictEqual(extractMatrixBlock(source), renderMarkdownTable()); +}); + +test('adapter compliance matrix extraction tolerates Windows line endings', () => { + const source = read('docs/architecture/harness-adapter-compliance.md') + .replace(/\r\n/g, '\n') + .replace(/\n/g, '\r\n'); + assert.strictEqual(extractMatrixBlock(source), renderMarkdownTable()); +}); + test('adapter compliance matrix includes the required evidence columns', () => { const source = read('docs/architecture/harness-adapter-compliance.md'); for (const heading of [ @@ -62,6 +93,7 @@ test('adapter compliance matrix includes the required evidence columns', () => { test('scorecard onramp names the local verification commands', () => { const source = read('docs/architecture/harness-adapter-compliance.md'); for (const command of [ + 'npm run harness:adapters -- --check', 'npm run harness:audit -- --format json', 'npm run observability:ready', 'node scripts/session-inspect.js --list-adapters', @@ -71,15 +103,39 @@ test('scorecard onramp names the local verification commands', () => { } }); +test('adapter compliance CLI check passes against the committed doc', () => { + const output = execFileSync('node', [scriptPath, '--check'], { + cwd: repoRoot, + encoding: 'utf8', + }); + + assert.ok(output.includes('Harness Adapter Compliance: PASS')); + assert.ok(output.includes(`Adapters: ${ADAPTER_RECORDS.length}`)); +}); + +test('adapter compliance CLI emits machine-readable scorecard data', () => { + const output = execFileSync('node', [scriptPath, '--format=json'], { + cwd: repoRoot, + encoding: 'utf8', + }); + const parsed = JSON.parse(output); + + assert.strictEqual(parsed.schema_version, 'ecc.harness-adapter-compliance.v1'); + assert.strictEqual(parsed.valid, true); + assert.strictEqual(parsed.adapter_count, ADAPTER_RECORDS.length); + assert.ok(parsed.adapters.some(record => record.id === 'terminal-only')); +}); + test('cross-harness architecture links to the adapter compliance matrix', () => { const source = read('docs/architecture/cross-harness.md'); assert.ok(source.includes('harness-adapter-compliance.md')); }); -test('GA roadmap records the matrix as current evidence and points to data-backed validation next', () => { +test('GA roadmap records the matrix and validator as current evidence', () => { const source = read('docs/ECC-2.0-GA-ROADMAP.md'); assert.ok(source.includes('docs/architecture/harness-adapter-compliance.md')); - assert.ok(source.includes('data-backed')); + assert.ok(source.includes('npm run harness:adapters -- --check')); + assert.ok(source.includes('scripts/lib/harness-adapter-compliance.js')); }); if (failed > 0) { diff --git a/tests/scripts/npm-publish-surface.test.js b/tests/scripts/npm-publish-surface.test.js index 68f9e0cc..18852106 100644 --- a/tests/scripts/npm-publish-surface.test.js +++ b/tests/scripts/npm-publish-surface.test.js @@ -56,6 +56,7 @@ function buildExpectedPublishPaths(repoRoot) { "scripts/observability-readiness.js", "scripts/skill-create-output.js", "scripts/repair.js", + "scripts/harness-adapter-compliance.js", "scripts/harness-audit.js", "scripts/session-inspect.js", "scripts/uninstall.js",