From 45a9bcf29572ee13508b1433ed2ddcbe63a8892b Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Wed, 29 Apr 2026 18:38:48 -0400 Subject: [PATCH] test: lift harness manifest branch coverage --- tests/lib/install-manifests.test.js | 262 ++++++++++++++++++++++++++++ tests/scripts/harness-audit.test.js | 151 +++++++++++++++- 2 files changed, 410 insertions(+), 3 deletions(-) diff --git a/tests/lib/install-manifests.test.js b/tests/lib/install-manifests.test.js index d8a79060..10c97ad1 100644 --- a/tests/lib/install-manifests.test.js +++ b/tests/lib/install-manifests.test.js @@ -8,6 +8,7 @@ const os = require('os'); const path = require('path'); const { + getInstallComponent, loadInstallManifests, listInstallComponents, listLegacyCompatibilityLanguages, @@ -45,6 +46,24 @@ function writeJson(filePath, value) { fs.writeFileSync(filePath, JSON.stringify(value, null, 2)); } +function writeManifestSet(repoRoot, options = {}) { + writeJson(path.join(repoRoot, 'manifests', 'install-modules.json'), { + version: options.modulesVersion || 1, + modules: options.modules || [], + }); + writeJson(path.join(repoRoot, 'manifests', 'install-profiles.json'), { + version: options.profilesVersion || 1, + profiles: options.profiles || {}, + }); + + if (Object.prototype.hasOwnProperty.call(options, 'components')) { + writeJson(path.join(repoRoot, 'manifests', 'install-components.json'), { + version: options.componentsVersion || 1, + components: options.components, + }); + } +} + function runTests() { console.log('\n=== Testing install-manifests.js ===\n'); @@ -80,6 +99,43 @@ function runTests() { 'Should include capability:security'); })) passed++; else failed++; + if (test('gets install component details and validates component IDs', () => { + const component = getInstallComponent(' lang:typescript '); + + assert.strictEqual(component.id, 'lang:typescript'); + assert.strictEqual(component.family, 'language'); + assert.ok(component.moduleIds.length > 0, 'Should expose component module IDs'); + assert.strictEqual(component.moduleCount, component.moduleIds.length); + assert.strictEqual(component.modules.length, component.moduleIds.length); + assert.ok(component.modules.every(module => component.moduleIds.includes(module.id))); + assert.ok(Array.isArray(component.targets)); + + assert.throws( + () => getInstallComponent(''), + /An install component ID is required/ + ); + assert.throws( + () => getInstallComponent('lang:missing'), + /Unknown install component: lang:missing/ + ); + })) passed++; else failed++; + + if (test('validates install component filters', () => { + const claudeComponents = listInstallComponents({ family: 'capability', target: 'claude' }); + assert.ok(claudeComponents.length > 0, 'Should list Claude capability components'); + assert.ok(claudeComponents.every(component => component.family === 'capability')); + assert.ok(claudeComponents.every(component => component.targets.includes('claude'))); + + assert.throws( + () => listInstallComponents({ family: 'unknown' }), + /Unknown component family: unknown/ + ); + assert.throws( + () => listInstallComponents({ target: 'unknown-target' }), + /Unknown install target: unknown-target/ + ); + })) passed++; else failed++; + if (test('labels continuous-learning as a legacy v1 install surface', () => { const components = listInstallComponents({ family: 'skill' }); const component = components.find(entry => entry.id === 'skill:continuous-learning'); @@ -172,6 +228,10 @@ function runTests() { () => validateInstallModuleIds(['ghost-module']), /Unknown install module: ghost-module/ ); + assert.throws( + () => validateInstallModuleIds(['ghost-one', 'ghost-two']), + /Unknown install modules: ghost-one, ghost-two/ + ); })) passed++; else failed++; if (test('resolves legacy compatibility selections into manifest module IDs', () => { @@ -251,6 +311,25 @@ function runTests() { }), /Unknown legacy language: brainfuck/ ); + assert.throws( + () => resolveLegacyCompatibilitySelection({ + legacyLanguages: [], + }), + /No legacy languages were provided/ + ); + assert.throws( + () => resolveLegacyCompatibilitySelection({ + target: 'not-a-target', + legacyLanguages: ['typescript'], + }), + /Unknown install target: not-a-target/ + ); + assert.throws( + () => resolveLegacyCompatibilitySelection({ + legacyLanguages: ['brainfuck', 'whitespace'], + }), + /Unknown legacy languages: brainfuck, whitespace/ + ); })) passed++; else failed++; if (test('resolves included and excluded user-facing components', () => { @@ -293,6 +372,61 @@ function runTests() { ); })) passed++; else failed++; + if (test('rejects empty, unknown, and fully excluded install selections', () => { + const repoRoot = createTestRepo(); + try { + writeManifestSet(repoRoot, { + modules: [ + { + id: 'core', + kind: 'rules', + description: 'Core', + paths: ['rules/core.md'], + targets: ['claude'], + dependencies: [], + defaultInstall: true, + cost: 'light', + stability: 'stable' + } + ], + profiles: { + core: { description: 'Core', modules: ['core'] } + }, + components: [ + { + id: 'capability:core', + family: 'capability', + description: 'Core', + modules: ['core'] + } + ], + }); + + assert.throws( + () => resolveInstallPlan({ repoRoot }), + /No install profile, module IDs, or included component IDs were provided/ + ); + assert.throws( + () => resolveInstallPlan({ repoRoot, moduleIds: ['missing'] }), + /Unknown install module: missing/ + ); + assert.throws( + () => resolveInstallPlan({ repoRoot, includeComponentIds: ['capability:missing'] }), + /Unknown install component: capability:missing/ + ); + assert.throws( + () => resolveInstallPlan({ + repoRoot, + profileId: 'core', + excludeComponentIds: ['capability:core'], + }), + /Selection excludes every requested install module/ + ); + } finally { + cleanupTestRepo(repoRoot); + } + })) passed++; else failed++; + if (test('validates projectRoot and homeDir option types before adapter planning', () => { assert.throws( () => resolveInstallPlan({ profileId: 'core', target: 'cursor', projectRoot: 42 }), @@ -349,6 +483,92 @@ function runTests() { } })) passed++; else failed++; + if (test('rejects missing, malformed, and unsupported manifest fixtures', () => { + const repoRoot = createTestRepo(); + try { + assert.throws( + () => loadInstallManifests({ repoRoot }), + /Install manifests not found/ + ); + + fs.writeFileSync(path.join(repoRoot, 'manifests', 'install-modules.json'), '{ bad json'); + writeJson(path.join(repoRoot, 'manifests', 'install-profiles.json'), { + version: 1, + profiles: {}, + }); + assert.throws( + () => loadInstallManifests({ repoRoot }), + /Failed to read install-modules\.json/ + ); + + writeManifestSet(repoRoot, { + modules: [ + { + id: 'empty-target', + kind: 'rules', + description: 'Empty target', + paths: ['rules/core.md'], + targets: ['claude', ''], + dependencies: [], + defaultInstall: false, + cost: 'light', + stability: 'stable' + } + ], + profiles: {}, + }); + assert.throws( + () => loadInstallManifests({ repoRoot }), + /Install module empty-target has invalid targets/ + ); + + writeManifestSet(repoRoot, { + modules: [ + { + id: 'unsupported-target', + kind: 'rules', + description: 'Unsupported target', + paths: ['rules/core.md'], + targets: ['claude', 'moonbase'], + dependencies: [], + defaultInstall: false, + cost: 'light', + stability: 'stable' + } + ], + profiles: {}, + }); + assert.throws( + () => loadInstallManifests({ repoRoot }), + /Install module unsupported-target has unsupported targets: moonbase/ + ); + + writeManifestSet(repoRoot, { + modules: [ + { + id: 'core', + kind: 'rules', + description: 'Core', + paths: ['rules/core.md'], + targets: ['claude'], + dependencies: [], + defaultInstall: false, + cost: 'light', + stability: 'stable' + } + ], + profiles: { + core: { description: 'Core', modules: ['core'] } + }, + }); + const manifests = loadInstallManifests({ repoRoot }); + assert.deepStrictEqual(manifests.components, []); + assert.strictEqual(manifests.componentsVersion, null); + } finally { + cleanupTestRepo(repoRoot); + } + })) passed++; else failed++; + if (test('fails fast when install manifest module targets is not an array', () => { const repoRoot = createTestRepo(); try { @@ -431,6 +651,48 @@ function runTests() { } })) passed++; else failed++; + if (test('detects circular install dependencies', () => { + const repoRoot = createTestRepo(); + try { + writeManifestSet(repoRoot, { + modules: [ + { + id: 'alpha', + kind: 'skills', + description: 'Alpha', + paths: ['skills/alpha'], + targets: ['claude'], + dependencies: ['beta'], + defaultInstall: false, + cost: 'light', + stability: 'stable' + }, + { + id: 'beta', + kind: 'skills', + description: 'Beta', + paths: ['skills/beta'], + targets: ['claude'], + dependencies: ['alpha'], + defaultInstall: false, + cost: 'light', + stability: 'stable' + } + ], + profiles: { + core: { description: 'Core', modules: ['alpha'] } + }, + }); + + assert.throws( + () => resolveInstallPlan({ repoRoot, profileId: 'core' }), + /Circular install dependency detected at alpha/ + ); + } finally { + cleanupTestRepo(repoRoot); + } + })) passed++; else failed++; + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); process.exit(failed > 0 ? 1 : 0); } diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index ae22e754..0983d1b2 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -6,9 +6,10 @@ const assert = require('assert'); const fs = require('fs'); const os = require('os'); const path = require('path'); -const { execFileSync } = require('child_process'); +const { execFileSync, spawnSync } = require('child_process'); const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'harness-audit.js'); +const { parseArgs } = require(SCRIPT); function createTempDir(prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -18,7 +19,7 @@ function cleanup(dirPath) { fs.rmSync(dirPath, { recursive: true, force: true }); } -function run(args = [], options = {}) { +function buildEnv(options = {}) { const userProfile = options.userProfile || options.homeDir || process.env.USERPROFILE; const env = { ...process.env, @@ -31,9 +32,13 @@ function run(args = [], options = {}) { env.HOME = process.env.HOME; } + return env; +} + +function run(args = [], options = {}) { const stdout = execFileSync('node', [SCRIPT, ...args], { cwd: options.cwd || path.join(__dirname, '..', '..'), - env, + env: buildEnv(options), encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000, @@ -42,6 +47,16 @@ function run(args = [], options = {}) { return stdout; } +function runProcess(args = [], options = {}) { + return spawnSync('node', [SCRIPT, ...args], { + cwd: options.cwd || path.join(__dirname, '..', '..'), + env: buildEnv(options), + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 10000, + }); +} + function test(name, fn) { try { fn(); @@ -60,6 +75,46 @@ function runTests() { let passed = 0; let failed = 0; + if (test('parseArgs accepts supported forms and rejects invalid arguments', () => { + const rootDir = createTempDir('harness-audit-args-root-'); + + try { + assert.strictEqual(parseArgs(['node', 'script', '--help']).help, true); + assert.strictEqual(parseArgs(['node', 'script', '-h']).help, true); + + const spaced = parseArgs(['node', 'script', '--format', 'json', '--scope', 'skills', '--root', rootDir]); + assert.strictEqual(spaced.format, 'json'); + assert.strictEqual(spaced.scope, 'skills'); + assert.strictEqual(spaced.root, path.resolve(rootDir)); + + const equals = parseArgs(['node', 'script', '--format=json', '--scope=hooks', `--root=${rootDir}`]); + assert.strictEqual(equals.format, 'json'); + assert.strictEqual(equals.scope, 'hooks'); + assert.strictEqual(equals.root, path.resolve(rootDir)); + + assert.strictEqual(parseArgs(['node', 'script', 'commands']).scope, 'commands'); + assert.strictEqual(parseArgs(['node', 'script', '--scope']).scope, 'repo'); + assert.throws(() => parseArgs(['node', 'script', '--format', 'xml']), /Invalid format: xml/); + assert.throws(() => parseArgs(['node', 'script', '--scope', 'bad-scope']), /Invalid scope: bad-scope/); + assert.throws(() => parseArgs(['node', 'script', '--unknown']), /Unknown argument: --unknown/); + } finally { + cleanup(rootDir); + } + })) passed++; else failed++; + + if (test('cli help exits cleanly and invalid cli args exit with stderr', () => { + const help = runProcess(['--help']); + assert.strictEqual(help.status, 0); + assert.strictEqual(help.stderr, ''); + assert.ok(help.stdout.includes('Usage: node scripts/harness-audit.js')); + assert.ok(help.stdout.includes('Deterministic harness audit')); + + const invalid = runProcess(['--format', 'xml']); + assert.strictEqual(invalid.status, 1); + assert.strictEqual(invalid.stdout, ''); + assert.ok(invalid.stderr.includes('Error: Invalid format: xml. Use text or json.')); + })) passed++; else failed++; + if (test('json output is deterministic between runs', () => { const first = run(['repo', '--format', 'json']); const second = run(['repo', '--format', 'json']); @@ -103,6 +158,29 @@ function runTests() { assert.ok(output.includes('Top 3 Actions:') || output.includes('Checks:')); })) passed++; else failed++; + if (test('detects repo mode from structural markers when package name differs', () => { + const projectRoot = createTempDir('harness-audit-structural-repo-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'scripts'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.claude-plugin'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'agents'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'skills'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'scripts', 'harness-audit.js'), '#!/usr/bin/env node\n'); + fs.writeFileSync(path.join(projectRoot, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'ecc' }, null, 2)); + fs.writeFileSync( + path.join(projectRoot, 'package.json'), + JSON.stringify({ name: 'forked-harness', scripts: { test: 'node scripts/validate-commands.js && node tests/run-all.js' } }, null, 2) + ); + + const parsed = JSON.parse(run(['--format=json', `--root=${projectRoot}`])); + assert.strictEqual(parsed.target_mode, 'repo'); + assert.strictEqual(parsed.root_dir, path.resolve(projectRoot)); + } finally { + cleanup(projectRoot); + } + })) passed++; else failed++; + if (test('audits consumer projects from cwd instead of the ECC repo root', () => { const homeDir = createTempDir('harness-audit-home-'); const projectRoot = createTempDir('harness-audit-project-'); @@ -141,6 +219,73 @@ function runTests() { } })) passed++; else failed++; + if (test('scores empty consumer projects without plugin or harness signals as failing checks', () => { + const homeDir = createTempDir('harness-audit-empty-home-'); + const projectRoot = createTempDir('harness-audit-empty-project-'); + + try { + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + + assert.strictEqual(parsed.target_mode, 'consumer'); + assert.strictEqual(parsed.overall_score, 0); + assert.ok(parsed.max_score > 0); + assert.strictEqual(parsed.top_actions.length, 3); + assert.ok(parsed.checks.some(check => check.id === 'consumer-plugin-install' && !check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-project-overrides' && !check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-secret-hygiene' && !check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('prints no top actions when consumer checks all pass', () => { + const homeDir = createTempDir('harness-audit-passing-home-'); + const projectRoot = createTempDir('harness-audit-passing-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, '.claude', 'plugins', 'ecc@ecc'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, '.claude', 'plugins', 'ecc@ecc', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + fs.mkdirSync(path.join(projectRoot, '.claude'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.github', 'workflows', 'nested'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'docs', 'adr'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'evals'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'src'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, '.claude', 'hooks.json'), JSON.stringify({ hooks: [] }, null, 2)); + fs.writeFileSync(path.join(projectRoot, '.claude', 'settings.local.json'), JSON.stringify({ local: true }, null, 2)); + fs.writeFileSync(path.join(projectRoot, 'CLAUDE.md'), '# Consumer instructions\n'); + fs.writeFileSync(path.join(projectRoot, 'src', 'app.spec.ts'), 'test placeholder\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'workflows', 'nested', 'ci.yaml'), 'name: ci\n'); + fs.writeFileSync(path.join(projectRoot, 'docs', 'adr', '001.md'), '# Record\n'); + fs.writeFileSync(path.join(projectRoot, 'evals', 'smoke.json'), '{}\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'dependabot.yml'), 'version: 2\n'); + fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'node_modules\n.env.local\n'); + fs.writeFileSync( + path.join(projectRoot, 'package.json'), + JSON.stringify({ name: 'passing-consumer', scripts: {} }, null, 2) + ); + + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.strictEqual(parsed.target_mode, 'consumer'); + assert.strictEqual(parsed.overall_score, parsed.max_score); + + const text = run(['repo'], { cwd: projectRoot, homeDir }); + assert.ok(text.includes(`Harness Audit (repo, consumer): ${parsed.max_score}/${parsed.max_score}`)); + assert.ok(text.includes('Checks: 11 total, 0 failing')); + assert.ok(!text.includes('Top 3 Actions:')); + + const scopedText = run(['agents'], { cwd: projectRoot, homeDir }); + assert.ok(scopedText.includes('Harness Audit (agents, consumer):')); + assert.ok(scopedText.includes('Checks: 1 total, 0 failing')); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + if (test('detects marketplace-installed Claude plugins under home marketplaces/', () => { const homeDir = createTempDir('harness-audit-marketplace-home-'); const projectRoot = createTempDir('harness-audit-marketplace-project-');