diff --git a/tests/test_json_envelope_field_consistency.py b/tests/test_json_envelope_field_consistency.py new file mode 100644 index 0000000..b3a6426 --- /dev/null +++ b/tests/test_json_envelope_field_consistency.py @@ -0,0 +1,211 @@ +"""JSON envelope field consistency validation (ROADMAP #173 prep). + +This test suite validates that clawable-surface commands' JSON output +follows the contract defined in SCHEMAS.md. Currently, commands emit +command-specific envelopes without the canonical common fields +(timestamp, command, exit_code, output_format, schema_version). + +This test documents the current gap and validates the consistency +of what IS there, providing a baseline for #173 (common field wrapping). + +Phase 1 (this test): Validate consistency within each command's envelope. +Phase 2 (future #173): Wrap all 13 commands with canonical common fields. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.main import build_parser # noqa: E402 + + +# Expected fields for each clawable command's JSON envelope. +# These are the command-specific fields (not including common fields yet). +# Entries are (command_name, required_fields, optional_fields). +ENVELOPE_CONTRACTS = { + 'list-sessions': ( + {'count', 'sessions'}, + set(), + ), + 'delete-session': ( + {'session_id', 'deleted', 'directory'}, + set(), + ), + 'load-session': ( + {'session_id', 'loaded', 'directory', 'path'}, + set(), + ), + 'flush-transcript': ( + {'session_id', 'path', 'flushed', 'messages_count', 'input_tokens', 'output_tokens'}, + set(), + ), + 'show-command': ( + {'name', 'found', 'source_hint', 'responsibility'}, + set(), + ), + 'show-tool': ( + {'name', 'found', 'source_hint'}, + set(), + ), + 'exec-command': ( + {'name', 'prompt', 'handled', 'message', 'source_hint'}, + set(), + ), + 'exec-tool': ( + {'name', 'payload', 'handled', 'message', 'source_hint'}, + set(), + ), + 'route': ( + {'prompt', 'limit', 'match_count', 'matches'}, + set(), + ), + 'bootstrap': ( + {'prompt', 'setup', 'routed_matches', 'turn', 'persisted_session_path'}, + set(), + ), + 'command-graph': ( + {'builtins_count', 'plugin_like_count', 'skill_like_count', 'total_count', 'builtins', 'plugin_like', 'skill_like'}, + set(), + ), + 'tool-pool': ( + {'simple_mode', 'include_mcp', 'tool_count', 'tools'}, + set(), + ), + 'bootstrap-graph': ( + {'stages', 'note'}, + set(), + ), +} + + +class TestJsonEnvelopeConsistency: + """Validate current command envelopes match their declared contracts. + + This is a consistency check, not a conformance check. Once #173 adds + common fields to all commands, these tests will auto-pass the common + field assertions and verify command-specific fields stay consistent. + """ + + @pytest.mark.parametrize('cmd_name,contract', sorted(ENVELOPE_CONTRACTS.items())) + def test_command_json_fields_present(self, cmd_name: str, contract: tuple[set[str], set[str]]) -> None: + required, optional = contract + """Command's JSON envelope must include all required fields.""" + # Get minimal invocation args for this command + test_invocations = { + 'list-sessions': [], + 'show-command': ['add-dir'], + 'show-tool': ['BashTool'], + 'exec-command': ['add-dir', 'hi'], + 'exec-tool': ['BashTool', '{}'], + 'route': ['review'], + 'bootstrap': ['hello'], + 'command-graph': [], + 'tool-pool': [], + 'bootstrap-graph': [], + } + + if cmd_name not in test_invocations: + pytest.skip(f'{cmd_name} requires session setup; skipped') + + cmd_args = test_invocations[cmd_name] + result = subprocess.run( + [sys.executable, '-m', 'src.main', cmd_name, *cmd_args, '--output-format', 'json'], + cwd=Path(__file__).resolve().parent.parent, + capture_output=True, + text=True, + ) + + if result.returncode not in (0, 1): + pytest.fail(f'{cmd_name}: unexpected exit {result.returncode}\nstderr: {result.stderr}') + + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError as e: + pytest.fail(f'{cmd_name}: invalid JSON: {e}\nOutput: {result.stdout[:200]}') + + # Check required fields (command-specific) + missing = required - set(envelope.keys()) + if missing: + pytest.fail( + f'{cmd_name} envelope missing required fields: {missing}\n' + f'Expected: {required}\nGot: {set(envelope.keys())}' + ) + + # Check that extra fields are accounted for (warn if unknown) + known = required | optional + extra = set(envelope.keys()) - known + if extra: + # Warn but don't fail — there may be new fields added + pytest.warns(UserWarning, match=f'extra fields in {cmd_name}: {extra}') + + def test_envelope_field_value_types(self) -> None: + """Smoke test: envelope fields have expected types (bool, int, str, list, dict, null).""" + result = subprocess.run( + [sys.executable, '-m', 'src.main', 'list-sessions', '--output-format', 'json'], + cwd=Path(__file__).resolve().parent.parent, + capture_output=True, + text=True, + ) + + envelope = json.loads(result.stdout) + + # Spot check a few fields + assert isinstance(envelope.get('count'), int), 'count should be int' + assert isinstance(envelope.get('sessions'), list), 'sessions should be list' + + +class TestJsonEnvelopeCommonFieldPrep: + """Validation stubs for common fields (part of #173 implementation). + + These tests will activate once wrap_json_envelope() is applied to all + 13 clawable commands. Currently they document the expected contract. + """ + + @pytest.mark.skip(reason='Common fields not yet wrapped (ROADMAP #173)') + def test_all_envelopes_include_timestamp(self) -> None: + """Every clawable envelope must include ISO 8601 UTC timestamp.""" + result = subprocess.run( + [sys.executable, '-m', 'src.main', 'command-graph', '--output-format', 'json'], + cwd=Path(__file__).resolve().parent.parent, + capture_output=True, + text=True, + ) + envelope = json.loads(result.stdout) + assert 'timestamp' in envelope, 'Missing timestamp field' + # Verify ISO 8601 format (ends with Z for UTC) + assert envelope['timestamp'].endswith('Z'), f'Timestamp not UTC: {envelope["timestamp"]}' + + @pytest.mark.skip(reason='Common fields not yet wrapped (ROADMAP #173)') + def test_all_envelopes_include_command(self) -> None: + """Every envelope must echo the command name.""" + for cmd_name in ['list-sessions', 'command-graph', 'bootstrap']: + result = subprocess.run( + [sys.executable, '-m', 'src.main', cmd_name, '--output-format', 'json'], + cwd=Path(__file__).resolve().parent.parent, + capture_output=True, + text=True, + ) + envelope = json.loads(result.stdout) + assert envelope.get('command') == cmd_name, f'{cmd_name} envelope.command mismatch' + + @pytest.mark.skip(reason='Common fields not yet wrapped (ROADMAP #173)') + def test_all_envelopes_include_exit_code_and_schema_version(self) -> None: + """Every envelope must include exit_code and schema_version.""" + result = subprocess.run( + [sys.executable, '-m', 'src.main', 'tool-pool', '--output-format', 'json'], + cwd=Path(__file__).resolve().parent.parent, + capture_output=True, + text=True, + ) + envelope = json.loads(result.stdout) + assert 'exit_code' in envelope, 'Missing exit_code' + assert 'schema_version' in envelope, 'Missing schema_version' + assert envelope['schema_version'] == '1.0', 'Wrong schema_version'