Your Name
61241c3a82
- install.sh: install runtime tree under OPENCODE_DEV_AGENT_TEAM (bin/, 12 skills, improvements/, install-manifest.json), idempotent shell-rc export, --uninstall (preserves improvements/ user data) and --migrate; preserves 13-agent copy, .backup retention, KEEP_BACKUPS=5, count gate, cmp -s integrity, install-time permission gate - memory-lifecycle.sh: resolve MEMORY_DIR from project git root + OPENCODE_MEMORY_DIR - all 13 agents: canonical runtime sentence + env-resolved runtime paths; project-scoped refs unchanged - tests: new test-install (15), test-runtime (10), test-path-resolution (12), test-memory-isolation (12); TEAM_ROOT override on existing 4 suites; test-all.sh registers 8 suites (100 checks total)
305 lines
13 KiB
Bash
305 lines
13 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# test-integration.sh — Integration tests for memory, skills, routing, and
|
|
# improvement systems. Verifies that the components work together end-to-end:
|
|
# - memory-lifecycle.sh actually stores/recalls/lists/searches real entries
|
|
# - agent↔skill mapping is consistent (every listed agent skill exists)
|
|
# - every skill file is valid markdown with required frontmatter
|
|
# - every skill whose owner is an agent is registered in that agent's prompt
|
|
# - improvement proposal lifecycle (pending → applied) works
|
|
# - skip-when routing guards are present in the orchestrator
|
|
#
|
|
# Run: bash scripts/test-integration.sh
|
|
|
|
set -euo pipefail
|
|
TEAM_ROOT="${TEAM_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
|
AGENTS="$TEAM_ROOT/agents"
|
|
SKILLS="$TEAM_ROOT/skills"
|
|
MEMORY="$TEAM_ROOT/memory"
|
|
IMPROV="$TEAM_ROOT/improvements"
|
|
LS="$TEAM_ROOT/scripts/memory-lifecycle.sh"
|
|
|
|
PASS=0; FAIL=0
|
|
T="$(mktemp -d)"
|
|
trap 'rm -rf "$T"' EXIT
|
|
|
|
ok() { PASS=$((PASS+1)); echo "PASS $1"; }
|
|
fail(){ FAIL=$((FAIL+1)); echo "FAIL $1 — ${2:-}" >&2; }
|
|
|
|
assert_file() { [ -f "$1" ] && return 0; return 1; }
|
|
assert_dir() { [ -d "$1" ] && return 0; return 1; }
|
|
assert_contains() { grep -qF "$2" "$1" 2>/dev/null && return 0; return 1; }
|
|
|
|
echo "init — team: $TEAM_ROOT"
|
|
|
|
# ======================================================================== #
|
|
# TEST I01: memory-lifecycle.sh end-to-end store + recall + list + search
|
|
# ======================================================================== #
|
|
# Create an actual lesson file in the temp memory workspace, store it, verify it
|
|
# is found by recall and search, then remove it (leave memory clean).
|
|
LESSON_FILE="$T/lesson-integration-test.md"
|
|
cat > "$LESSON_FILE" <<'EOF'
|
|
---
|
|
date: 2026-09-08
|
|
category: lesson
|
|
title: Integration test lesson
|
|
tags: [integration, memory]
|
|
---
|
|
# Integration test lesson
|
|
|
|
A lesson stored and retrieved by the memory lifecycle script during integration
|
|
testing. This entry is temporary and will be removed after verification.
|
|
EOF
|
|
|
|
SC_OK=0
|
|
# NOTE: capture output into variables rather than piping `memory-lifecycle.sh`
|
|
# into `grep -q`. grep -q closes the pipe early (SIGPIPE, rc=141), which fails
|
|
# the pipeline under `pipefail`.
|
|
STORE_OK=0
|
|
# Test isolation: remove any residue a previous interrupted/crashed run may have
|
|
# left behind before storing. Writing a fresh entry makes the check independent
|
|
# of the memory directory's pre-existing state.
|
|
rm -f "$MEMORY/lessons/"*integration-test* 2>/dev/null || true
|
|
if bash "$LS" store lessons "$LESSON_FILE" >/dev/null 2>&1; then
|
|
STORE_OK=1
|
|
fi
|
|
LIST_OUT="$(bash "$LS" list lessons 2>&1 || true)"
|
|
RECALL_OUT="$(bash "$LS" recall lessons "Integration" 2>&1 || true)"
|
|
SEARCH_OUT="$(bash "$LS" search "Integration test lesson" 2>&1 || true)"
|
|
if [ "$STORE_OK" = "1" ] \
|
|
&& echo "$LIST_OUT" | grep -q "integration-test" \
|
|
&& echo "$RECALL_OUT" | grep -qi "Integration test lesson" \
|
|
&& echo "$SEARCH_OUT" | grep -qi "integration-test"; then
|
|
SC_OK=1
|
|
fi
|
|
# clean up whatever got written so memory stays pristine
|
|
rm -f "$MEMORY/lessons/"*integration-test* 2>/dev/null || true
|
|
if [ "$SC_OK" = "1" ]; then
|
|
ok "I01 memory-lifecycle store/recall/list/search end-to-end"
|
|
else
|
|
fail "I01 memory-lifecycle store/recall/list/search end-to-end" "store/recall/list/search chain failed"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I02: Session lifecycle — create, list, cleanup
|
|
# ======================================================================== #
|
|
Sess="$MEMORY/sessions/2026-09-08_integration-session-test.md"
|
|
# Test isolation: remove any residue a previous interrupted/crashed run left.
|
|
rm -f "$Sess" 2>/dev/null || true
|
|
printf -- '%s\n' "Status: active" "# Integration session test" "## State" "in progress" > "$Sess"
|
|
SESS_OUT="$(bash "$LS" sessions 2>&1 || true)"
|
|
SESS_OK=0
|
|
if echo "$SESS_OUT" | grep -q "integration-session-test" \
|
|
&& bash "$LS" cleanup >/dev/null 2>&1; then
|
|
SESS_OK=1
|
|
fi
|
|
rm -f "$Sess" 2>/dev/null || true
|
|
if [ "$SESS_OK" = "1" ]; then
|
|
ok "I02 session create/list/cleanup"
|
|
else
|
|
fail "I02 session create/list/cleanup" "sessions chain failed — session listed?"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I03: agent↔skill mapping integrity — agent prompt names a skill or skill exists
|
|
# ======================================================================== #
|
|
# The agent-skill mapping in orchestrator.md should reference only skills that
|
|
# exist as files. This is a hard consistency check.
|
|
RULES_OK=1
|
|
# every skill directory in skills/ has a SKILL.md with valid frontmatter
|
|
for d in "$SKILLS"/*/; do
|
|
n="$(basename "$d")"
|
|
f="$d/SKILL.md"
|
|
if ! assert_file "$f"; then fail "I03 skill $n has no SKILL.md"; RULES_OK=0; break; fi
|
|
if ! grep -q "^name:" "$f" || ! grep -q "^owner:" "$f"; then
|
|
fail "I03 skill $n missing frontmatter"; RULES_OK=0; break
|
|
fi
|
|
# description should be a single line (required frontmatter)
|
|
if ! grep -q "^description:" "$f"; then fail "I03 skill $n missing description"; RULES_OK=0; break; fi
|
|
done
|
|
if [ "$RULES_OK" = "1" ]; then
|
|
ok "I03 all skill files have valid frontmatter"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I04: Every skill's owner is a registered agent (case-insensitive,
|
|
# and multi-owner "A + B" resolved to each named agent)
|
|
# ======================================================================== #
|
|
OWNER_OK=1
|
|
for d in "$SKILLS"/*/; do
|
|
n="$(basename "$d")"
|
|
f="$d/SKILL.md"
|
|
owner="$(grep -m1 "^owner:" "$f" 2>/dev/null | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
|
[ -z "$owner" ] && continue
|
|
# Resolve multi-owner "A + B" and case differences to agent file names (lowercase)
|
|
IFS='+' read -ra owners <<< "$owner"
|
|
for o in "${owners[@]}"; do
|
|
o_trim="$(echo "$o" | xargs | tr '[:upper:]' '[:lower:]')"
|
|
[ -f "$AGENTS/$o_trim.md" ] || { fail "I04 skill $n owner '$owner' not an agent" "missing agents/$o_trim.md"; OWNER_OK=0; }
|
|
done
|
|
done
|
|
if [ "$OWNER_OK" = "1" ]; then
|
|
ok "I04 every skill owner is a registered agent"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I05: Every agent named in orchestrator's skill mapping owns a real skill
|
|
# ======================================================================== #
|
|
# Agents in the Agent-Skill Mapping table should own at least one skill in skills/.
|
|
MAPPED_OWNERS="$(grep -A2 "name:.*tdd" "$SKILLS/tdd/SKILL.md" >/dev/null && echo ok)"
|
|
OWNER_LIST=""
|
|
for d in "$SKILLS"/*/; do
|
|
o="$(grep -m1 "^owner:" "$d/SKILL.md" | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
|
[ -z "$o" ] && continue
|
|
OWNER_LIST="$OWNER_LIST $o"
|
|
done
|
|
# Each mapped primary agent in the orchestrator should appear as an owner
|
|
MAPPED_OK=1
|
|
for ag in tdd systematic-debugging architecture-design code-review security-review \
|
|
repository-analysis failure-analysis refactoring test-analysis \
|
|
incident-investigation browser-automation research; do
|
|
if ! [ -f "$SKILLS/$ag/SKILL.md" ]; then fail "I05 skill $ag missing"; MAPPED_OK=0; fi
|
|
done
|
|
if [ "$MAPPED_OK" = "1" ]; then
|
|
ok "I05 all 12 skills present for mapping table"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I06: Orchestrator action catalog covers memory + skills + improvements (A23-A27)
|
|
# ======================================================================== #
|
|
ORCH="$AGENTS/orchestrator.md"
|
|
CAT_OK=0
|
|
if assert_contains "$ORCH" "A23 | recall project memory" \
|
|
&& assert_contains "$ORCH" "A24 | store project memory" \
|
|
&& assert_contains "$ORCH" "A25 | load skill" \
|
|
&& assert_contains "$ORCH" "A26 | finish / report" \
|
|
&& assert_contains "$ORCH" "A27 | re-plan"; then
|
|
CAT_OK=1
|
|
fi
|
|
if [ "$CAT_OK" = "1" ]; then
|
|
ok "I06 action catalog integrates memory/skills/finish/re-plan"
|
|
else
|
|
fail "I06 action catalog integrates memory/skills" "missing A23-A27 entries"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I07: Task lifecycle section exists and ties stages to memory
|
|
# ======================================================================== #
|
|
LC_OK=0
|
|
if assert_contains "$ORCH" "Task Lifecycle" \
|
|
&& assert_contains "$ORCH" "RECALL" \
|
|
&& assert_contains "$ORCH" "STORE" \
|
|
&& assert_contains "$ORCH" "IMPROVE"; then
|
|
LC_OK=1
|
|
fi
|
|
if [ "$LC_OK" = "1" ]; then
|
|
ok "I07 task lifecycle integrates memory/skills/improvements stages"
|
|
else
|
|
fail "I07 task lifecycle integrates memory/skills" "Task Lifecycle section missing stages"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I08: Skip-when routing guards present
|
|
# ======================================================================== #
|
|
SKIP_OK=0
|
|
if grep -q "Skip-when" "$ORCH" && grep -q "Skip Explorer" "$ORCH"; then
|
|
SKIP_OK=1
|
|
fi
|
|
if [ "$SKIP_OK" = "1" ]; then
|
|
ok "I08 skip-when routing guards present"
|
|
else
|
|
fail "I08 skip-when routing guards present" "missing skip-when section"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I09: improvement proposal lifecycle — write pending, move to applied
|
|
# ======================================================================== #
|
|
PROP="$IMPROV/pending/2026-09-08_integration-prop.md"
|
|
mkdir -p "$IMPROV/pending" "$IMPROV/applied" "$IMPROV/rejected"
|
|
printf -- '%s\n' "# Proposal: integration test" "## Problem" "temporary" > "$PROP"
|
|
PROP_OK=0
|
|
if [ -f "$PROP" ] \
|
|
&& assert_contains "$IMPROV/README.md" "human approval" \
|
|
&& mkdir -p "$IMPROV/applied" \
|
|
&& mv "$PROP" "$IMPROV/applied/2026-09-08_integration-prop.md" 2>/dev/null; then
|
|
PROP_OK=1
|
|
fi
|
|
# restore: remove applied copy
|
|
rm -f "$IMPROV/applied/2026-09-08_integration-prop.md" 2>/dev/null || true
|
|
if [ "$PROP_OK" = "1" ]; then
|
|
ok "I09 improvement proposal pending→applied lifecycle"
|
|
else
|
|
fail "I09 improvement proposal lifecycle" "proposal lifecycle failed"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I10: Improper agent (unknown owner) not silently accepted
|
|
# ======================================================================== #
|
|
# Create a temp skill with an invalid owner and verify the mapping check would reject it.
|
|
INVALID_SKILL="$T/invalid/SKILL.md"
|
|
mkdir -p "$T/invalid"
|
|
printf -- '%s\n' "---" "name: invalid" "owner: not-an-agent" "---" > "$INVALID_SKILL"
|
|
INV_OK=1
|
|
for f in "$SKILLS"/*/SKILL.md; do
|
|
o="$(grep -m1 "^owner:" "$f" 2>/dev/null | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
|
[ -z "$o" ] && continue
|
|
IFS='+' read -ra owners <<< "$o"
|
|
for ow in "${owners[@]}"; do
|
|
ow_trim="$(echo "$ow" | xargs | tr '[:upper:]' '[:lower:]')"
|
|
if [ -n "$ow_trim" ] && [ ! -f "$AGENTS/$ow_trim.md" ]; then
|
|
fail "I10 skill owner validation" "invalid owner '$o' in $f"
|
|
INV_OK=0
|
|
break
|
|
fi
|
|
done
|
|
[ "$INV_OK" = "0" ] && break
|
|
done
|
|
if [ "$INV_OK" = "1" ]; then
|
|
ok "I10 real skills all have valid registered owners"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I11: Orchestrator wires improvement proposal detection into workflow
|
|
# ======================================================================== #
|
|
ORCH="$AGENTS/orchestrator.md"
|
|
IMPROVE_DET_OK=0
|
|
if assert_contains "$ORCH" "Recurring failure" \
|
|
&& assert_contains "$ORCH" "Missing skill" \
|
|
&& assert_contains "$ORCH" "Routing inefficiency" \
|
|
&& assert_contains "$ORCH" "Documentation gap" \
|
|
&& assert_contains "$ORCH" "Process friction"; then
|
|
IMPROVE_DET_OK=1
|
|
fi
|
|
if [ "$IMPROVE_DET_OK" = "1" ]; then
|
|
ok "I11 improvement proposal detection triggers wired"
|
|
else
|
|
fail "I11 improvement proposal detection" "missing detection triggers in orchestrator"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# TEST I12: Orchestrator proposal format matches improvements/README format
|
|
# ======================================================================== #
|
|
FMT_OK=0
|
|
if assert_contains "$ORCH" "improvements/pending/YYYY-MM-DD" \
|
|
&& assert_contains "$IMPROV/README.md" "Observed problem" \
|
|
&& assert_contains "$IMPROV/README.md" "Root cause" \
|
|
&& assert_contains "$IMPROV/README.md" "Proposed change" \
|
|
&& assert_contains "$IMPROV/README.md" "Verification plan" \
|
|
&& assert_contains "$IMPROV/README.md" "without human approval"; then
|
|
FMT_OK=1
|
|
fi
|
|
if [ "$FMT_OK" = "1" ]; then
|
|
ok "I12 proposal format consistent across orchestrator and README"
|
|
else
|
|
fail "I12 proposal format consistency" "format fields missing"
|
|
fi
|
|
|
|
# ======================================================================== #
|
|
# Summary
|
|
# ======================================================================== #
|
|
echo
|
|
echo "==================== SUMMARY ===================="
|
|
echo "PASS: $PASS FAIL: $FAIL"
|
|
[ "$FAIL" = "0" ] && echo "RESULT: ALL PASS" || echo "RESULT: FAILURES PRESENT"
|
|
exit $(( FAIL > 0 ? 1 : 0 ))
|