From b6b5b6d08ed48ee21156caa31e8c20da9915a8ce Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Tue, 28 Apr 2026 22:14:19 -0400 Subject: [PATCH] test: cover CI catalog validator --- scripts/ci/catalog.js | 150 +++++++++++++++------- tests/ci/catalog.test.js | 265 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 367 insertions(+), 48 deletions(-) create mode 100644 tests/ci/catalog.test.js diff --git a/scripts/ci/catalog.js b/scripts/ci/catalog.js index da08d9ef..4c482d2c 100644 --- a/scripts/ci/catalog.js +++ b/scripts/ci/catalog.js @@ -33,8 +33,8 @@ function normalizePathSegments(relativePath) { return relativePath.split(path.sep).join('/'); } -function listMatchingFiles(relativeDir, matcher) { - const directory = path.join(ROOT, relativeDir); +function listMatchingFiles(root, relativeDir, matcher) { + const directory = path.join(root, relativeDir); if (!fs.existsSync(directory)) { return []; } @@ -45,11 +45,11 @@ function listMatchingFiles(relativeDir, matcher) { .sort(); } -function buildCatalog() { - const agents = listMatchingFiles('agents', entry => entry.isFile() && entry.name.endsWith('.md')); - const commands = listMatchingFiles('commands', entry => entry.isFile() && entry.name.endsWith('.md')); - const skills = listMatchingFiles('skills', entry => ( - entry.isDirectory() && fs.existsSync(path.join(ROOT, 'skills', entry.name, 'SKILL.md')) +function buildCatalog(root = ROOT) { + const agents = listMatchingFiles(root, 'agents', entry => entry.isFile() && entry.name.endsWith('.md')); + const commands = listMatchingFiles(root, 'commands', entry => entry.isFile() && entry.name.endsWith('.md')); + const skills = listMatchingFiles(root, 'skills', entry => ( + entry.isDirectory() && fs.existsSync(path.join(root, 'skills', entry.name, 'SKILL.md')) )).map(skillDir => `${skillDir}/SKILL.md`); return { @@ -540,33 +540,55 @@ function syncZhAgents(content, catalog) { return nextContent; } -const DOCUMENT_SPECS = [ - { - filePath: README_PATH, - parseExpectations: parseReadmeExpectations, - syncContent: syncEnglishReadme, - }, - { - filePath: AGENTS_PATH, - parseExpectations: parseAgentsDocExpectations, - syncContent: syncEnglishAgents, - }, - { - filePath: README_ZH_CN_PATH, - parseExpectations: parseZhRootReadmeExpectations, - syncContent: syncZhRootReadme, - }, - { - filePath: DOCS_ZH_CN_README_PATH, - parseExpectations: parseZhDocsReadmeExpectations, - syncContent: syncZhDocsReadme, - }, - { - filePath: DOCS_ZH_CN_AGENTS_PATH, - parseExpectations: parseZhAgentsDocExpectations, - syncContent: syncZhAgents, - }, -]; +function createDocumentSpecs(paths = {}) { + const { + readmePath = README_PATH, + agentsPath = AGENTS_PATH, + zhRootReadmePath = README_ZH_CN_PATH, + zhDocsReadmePath = DOCS_ZH_CN_README_PATH, + zhDocsAgentsPath = DOCS_ZH_CN_AGENTS_PATH, + } = paths; + + return [ + { + filePath: readmePath, + parseExpectations: parseReadmeExpectations, + syncContent: syncEnglishReadme, + }, + { + filePath: agentsPath, + parseExpectations: parseAgentsDocExpectations, + syncContent: syncEnglishAgents, + }, + { + filePath: zhRootReadmePath, + parseExpectations: parseZhRootReadmeExpectations, + syncContent: syncZhRootReadme, + }, + { + filePath: zhDocsReadmePath, + parseExpectations: parseZhDocsReadmeExpectations, + syncContent: syncZhDocsReadme, + }, + { + filePath: zhDocsAgentsPath, + parseExpectations: parseZhAgentsDocExpectations, + syncContent: syncZhAgents, + }, + ]; +} + +function createDocumentSpecsForRoot(root) { + return createDocumentSpecs({ + readmePath: path.join(root, 'README.md'), + agentsPath: path.join(root, 'AGENTS.md'), + zhRootReadmePath: path.join(root, 'README.zh-CN.md'), + zhDocsReadmePath: path.join(root, 'docs', 'zh-CN', 'README.md'), + zhDocsAgentsPath: path.join(root, 'docs', 'zh-CN', 'AGENTS.md'), + }); +} + +const DOCUMENT_SPECS = createDocumentSpecs(); function renderText(result) { console.log('Catalog counts:'); @@ -608,11 +630,16 @@ function renderMarkdown(result) { } } -function main() { - const catalog = buildCatalog(); +function runCatalogCheck(options = {}) { + const root = options.root || ROOT; + const writeMode = options.writeMode ?? WRITE_MODE; + const documentSpecs = options.documentSpecs || ( + root === ROOT ? DOCUMENT_SPECS : createDocumentSpecsForRoot(root) + ); + const catalog = buildCatalog(root); - if (WRITE_MODE) { - for (const spec of DOCUMENT_SPECS) { + if (writeMode) { + for (const spec of documentSpecs) { const currentContent = readFileOrThrow(spec.filePath); const nextContent = spec.syncContent(currentContent, catalog); if (nextContent !== currentContent) { @@ -621,28 +648,55 @@ function main() { } } - const expectations = DOCUMENT_SPECS.flatMap(spec => ( + const expectations = documentSpecs.flatMap(spec => ( spec.parseExpectations(readFileOrThrow(spec.filePath)) )); const checks = evaluateExpectations(catalog, expectations); - const result = { catalog, checks }; + return { catalog, checks }; +} - if (OUTPUT_MODE === 'json') { +function main(options = {}) { + const outputMode = options.outputMode || OUTPUT_MODE; + const result = runCatalogCheck(options); + + if (outputMode === 'json') { console.log(JSON.stringify(result, null, 2)); - } else if (OUTPUT_MODE === 'md') { + } else if (outputMode === 'md') { renderMarkdown(result); } else { renderText(result); } - if (checks.some(check => !check.ok)) { + if (result.checks.some(check => !check.ok)) { process.exit(1); } } -try { - main(); -} catch (error) { - console.error(`ERROR: ${error.message}`); - process.exit(1); +if (require.main === module) { + try { + main(); + } catch (error) { + console.error(`ERROR: ${error.message}`); + process.exit(1); + } } + +module.exports = { + buildCatalog, + createDocumentSpecs, + createDocumentSpecsForRoot, + evaluateExpectations, + formatExpectation, + main, + parseAgentsDocExpectations, + parseReadmeExpectations, + parseZhAgentsDocExpectations, + parseZhDocsReadmeExpectations, + parseZhRootReadmeExpectations, + runCatalogCheck, + syncEnglishAgents, + syncEnglishReadme, + syncZhAgents, + syncZhDocsReadme, + syncZhRootReadme, +}; diff --git a/tests/ci/catalog.test.js b/tests/ci/catalog.test.js new file mode 100644 index 00000000..98e22a84 --- /dev/null +++ b/tests/ci/catalog.test.js @@ -0,0 +1,265 @@ +/** + * Direct coverage for scripts/ci/catalog.js. + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + buildCatalog, + formatExpectation, + runCatalogCheck, +} = require('../../scripts/ci/catalog'); + +function createTestDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ci-catalog-')); +} + +function cleanupTestDir(testDir) { + fs.rmSync(testDir, { recursive: true, force: true }); +} + +function writeCountedFiles(root, category, count) { + const dir = path.join(root, category); + fs.mkdirSync(dir, { recursive: true }); + + for (let index = 1; index <= count; index += 1) { + if (category === 'skills') { + const skillDir = path.join(dir, `skill-${index}`); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), `# Skill ${index}\n`); + } else { + fs.writeFileSync(path.join(dir, `${category}-${index}.md`), `# ${category} ${index}\n`); + } + } +} + +function writeEnglishReadme(root, counts, options = {}) { + const tableCounts = options.tableCounts || counts; + const parityCounts = options.parityCounts || counts; + const unrelatedSkillsCount = options.unrelatedSkillsCount || 16; + + fs.writeFileSync(path.join(root, 'README.md'), `Access to ${counts.agents} agents, ${counts.skills} skills, and ${counts.commands} commands. +| Feature | Claude Code | Cursor IDE | Codex CLI | OpenCode | +| --- | --- | --- | --- | --- | +| Agents | PASS: ${tableCounts.agents} agents | +| Commands | PASS: ${tableCounts.commands} commands | +| Skills | PASS: ${tableCounts.skills} skills | + +| Feature | Count | Format | +| --- | ---: | --- | +| Skills | ${unrelatedSkillsCount} | .agents/skills/ | + +## Cross-Tool Feature Parity + +| Feature | Claude Code | Cursor IDE | Codex CLI | OpenCode | +| --- | --- | --- | --- | --- | +| **Agents** | ${parityCounts.agents} | Shared (AGENTS.md) | Shared (AGENTS.md) | 12 | +| **Commands** | ${parityCounts.commands} | Shared | Instruction-based | 31 | +| **Skills** | ${parityCounts.skills} | Shared | 10 (native format) | 37 | +`); +} + +function writeEnglishAgents(root, counts, options = {}) { + const plus = options.skillsMinimum ? '+' : ''; + + fs.writeFileSync(path.join(root, 'AGENTS.md'), `This is a production plugin providing ${counts.agents} specialized agents, ${counts.skills}${plus} skills, ${counts.commands} commands. + +\`\`\` +agents/ - ${counts.agents} specialized subagents +skills/ - ${counts.skills}${plus} workflow skills and domain knowledge +commands/ - ${counts.commands} slash commands +\`\`\` +`); +} + +function writeZhRootReadme(root, counts) { + fs.writeFileSync(path.join(root, 'README.zh-CN.md'), `你现在可以使用 ${counts.agents} 个代理、${counts.skills} 个技能和 ${counts.commands} 个命令。\n`); +} + +function writeZhDocsReadme(root, counts, options = {}) { + const tableCounts = options.tableCounts || counts; + const parityCounts = options.parityCounts || counts; + const unrelatedSkillsCount = options.unrelatedSkillsCount || 16; + const dir = path.join(root, 'docs', 'zh-CN'); + fs.mkdirSync(dir, { recursive: true }); + + fs.writeFileSync(path.join(dir, 'README.md'), `你现在可以使用 ${counts.agents} 个智能体、${counts.skills} 项技能和 ${counts.commands} 个命令了。 +| 功能特性 | Claude Code | OpenCode | 状态 | +| --- | --- | --- | --- | +| 智能体 | PASS: ${tableCounts.agents} 个 | +| 命令 | PASS: ${tableCounts.commands} 个 | +| 技能 | PASS: ${tableCounts.skills} 项 | + +| 功能特性 | 数量 | 格式 | +| --- | ---: | --- | +| 技能 | ${unrelatedSkillsCount} | .agents/skills/ | + +## 跨工具功能对等 + +| 功能特性 | Claude Code | Cursor IDE | Codex CLI | OpenCode | +| --- | --- | --- | --- | --- | +| **智能体** | ${parityCounts.agents} | 共享 (AGENTS.md) | 共享 (AGENTS.md) | 12 | +| **命令** | ${parityCounts.commands} | 共享 | 基于指令 | 31 | +| **技能** | ${parityCounts.skills} | 共享 | 10 (原生格式) | 37 | +`); +} + +function writeZhAgents(root, counts, options = {}) { + const plus = options.skillsMinimum ? '+' : ''; + const dir = path.join(root, 'docs', 'zh-CN'); + fs.mkdirSync(dir, { recursive: true }); + + fs.writeFileSync(path.join(dir, 'AGENTS.md'), `这是一个生产就绪的 AI 编码插件，提供 ${counts.agents} 个专业代理、${counts.skills}${plus} 项技能、${counts.commands} 条命令。 + +\`\`\` +agents/ - ${counts.agents} 个专业子代理 +skills/ - ${counts.skills}${plus} 个工作流技能和领域知识 +commands/ - ${counts.commands} 个斜杠命令 +\`\`\` +`); +} + +function writeCatalogFixture(root, options = {}) { + const actualCounts = options.actualCounts || { agents: 1, skills: 1, commands: 1 }; + const documentedCounts = options.documentedCounts || actualCounts; + const skillsMinimum = Boolean(options.skillsMinimum); + const unrelatedSkillsCount = options.unrelatedSkillsCount || 16; + + writeCountedFiles(root, 'agents', actualCounts.agents); + writeCountedFiles(root, 'commands', actualCounts.commands); + writeCountedFiles(root, 'skills', actualCounts.skills); + + fs.writeFileSync(path.join(root, 'agents', 'notes.txt'), 'not counted\n'); + fs.writeFileSync(path.join(root, 'commands', 'notes.txt'), 'not counted\n'); + fs.mkdirSync(path.join(root, 'skills', 'missing-skill-file'), { recursive: true }); + + writeEnglishReadme(root, documentedCounts, { unrelatedSkillsCount }); + writeEnglishAgents(root, documentedCounts, { skillsMinimum }); + writeZhRootReadme(root, documentedCounts); + writeZhDocsReadme(root, documentedCounts, { unrelatedSkillsCount }); + writeZhAgents(root, documentedCounts, { skillsMinimum }); +} + +function test(name, fn) { + try { + fn(); + console.log(` PASS ${name}`); + return true; + } catch (error) { + console.log(` FAIL ${name}`); + console.log(` Error: ${error.message}`); + return false; + } +} + +function runTests() { + console.log('\n=== Testing CI catalog.js ===\n'); + + let passed = 0; + let failed = 0; + + if (test('builds catalog counts from a supplied root', () => { + const testDir = createTestDir(); + try { + writeCatalogFixture(testDir, { + actualCounts: { agents: 2, skills: 1, commands: 3 }, + documentedCounts: { agents: 2, skills: 1, commands: 3 }, + }); + + const catalog = buildCatalog(testDir); + + assert.deepStrictEqual( + { + agents: catalog.agents.count, + skills: catalog.skills.count, + commands: catalog.commands.count, + }, + { agents: 2, skills: 1, commands: 3 } + ); + assert.ok(catalog.agents.files.every(file => file.endsWith('.md'))); + assert.ok(catalog.skills.files.every(file => file.endsWith('/SKILL.md'))); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('reports mismatches from every tracked catalog document', () => { + const testDir = createTestDir(); + try { + writeCatalogFixture(testDir, { + actualCounts: { agents: 1, skills: 1, commands: 1 }, + documentedCounts: { agents: 9, skills: 9, commands: 9 }, + }); + + const result = runCatalogCheck({ root: testDir }); + const formatted = result.checks + .filter(check => !check.ok) + .map(formatExpectation) + .join('\n'); + + assert.ok(formatted.includes('README.md quick-start summary')); + assert.ok(formatted.includes('AGENTS.md summary')); + assert.ok(formatted.includes('README.zh-CN.md quick-start summary')); + assert.ok(formatted.includes('docs/zh-CN/README.md parity table')); + assert.ok(formatted.includes('docs/zh-CN/AGENTS.md project structure')); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('write mode syncs counts while preserving plus suffixes and unrelated tables', () => { + const testDir = createTestDir(); + try { + writeCatalogFixture(testDir, { + actualCounts: { agents: 1, skills: 1, commands: 1 }, + documentedCounts: { agents: 7, skills: 7, commands: 7 }, + skillsMinimum: true, + unrelatedSkillsCount: 42, + }); + + const result = runCatalogCheck({ root: testDir, writeMode: true }); + + assert.strictEqual(result.checks.filter(check => !check.ok).length, 0); + + const readme = fs.readFileSync(path.join(testDir, 'README.md'), 'utf8'); + const agentsDoc = fs.readFileSync(path.join(testDir, 'AGENTS.md'), 'utf8'); + const zhReadme = fs.readFileSync(path.join(testDir, 'docs', 'zh-CN', 'README.md'), 'utf8'); + const zhAgentsDoc = fs.readFileSync(path.join(testDir, 'docs', 'zh-CN', 'AGENTS.md'), 'utf8'); + + assert.ok(readme.includes('Access to 1 agents, 1 skills, and 1 legacy command shims')); + assert.ok(readme.includes('| Skills | 42 | .agents/skills/ |')); + assert.ok(agentsDoc.includes('providing 1 specialized agents, 1+ skills, 1 commands')); + assert.ok(agentsDoc.includes('skills/ - 1+ workflow skills and domain knowledge')); + assert.ok(zhReadme.includes('| 技能 | 42 | .agents/skills/ |')); + assert.ok(zhAgentsDoc.includes('提供 1 个专业代理、1+ 项技能、1 条命令')); + assert.ok(zhAgentsDoc.includes('skills/ - 1+ 个工作流技能和领域知识')); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('throws a clear error for missing tracked documents', () => { + const testDir = createTestDir(); + try { + writeCatalogFixture(testDir); + fs.rmSync(path.join(testDir, 'docs', 'zh-CN', 'AGENTS.md')); + + assert.throws( + () => runCatalogCheck({ root: testDir }), + /Failed to read AGENTS\.md/ + ); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests();