Phases 5-6, 10, 12, 14: task lifecycle, integration tests, proposal wiring
- orchestrator.md: added 'Selection with memory and skills' and 'Skip-when efficiency guards' to Dynamic Agent Selection; added memory-recall/skill-load fields to Decomposition work items; added full 'Task Lifecycle' stage loop (RECALL→UNDERSTAND→ESTIMATE→LOAD→ PLAN→DISPATCH→VERIFY→LEARN→STORE→IMPROVE→REPORT) with memory stage gate; rewrote Improvement Proposals with evidence-driven detection triggers (recurring failure, missing skill, routing inefficiency, docs gap, process friction, new pattern), detection flow, and concrete proposal format. - docs/AGENT_ARCHITECTURE.md: documented detection triggers and the task lifecycle in section 14. - scripts/test-integration.sh: new integration suite (12 tests) — memory-lifecycle store/recall/list/search end-to-end, session create/list/cleanup, skill frontmatter/owner validation (incl. multi-owner resolution), agent↔skill mapping integrity, action catalog memory/skills integration, task lifecycle presence, skip-when guards, improvement proposal pending→applied lifecycle, proposal format consistency, and real-skill owner validation. Also documents/works around grep -q SIGPIPE under pipefail. All 51 structural/integration tests across four suites pass.
This commit is contained in:
Your Name
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# test-integration.sh — Integration tests for memory, skills, routing, and
|
||||
# improvement systems. Verifies that the components work together end-to-end:
|
||||
# - memory-lifecycle.sh actually stores/recalls/lists/searches real entries
|
||||
# - agent↔skill mapping is consistent (every listed agent skill exists)
|
||||
# - every skill file is valid markdown with required frontmatter
|
||||
# - every skill whose owner is an agent is registered in that agent's prompt
|
||||
# - improvement proposal lifecycle (pending → applied) works
|
||||
# - skip-when routing guards are present in the orchestrator
|
||||
#
|
||||
# Run: bash scripts/test-integration.sh
|
||||
|
||||
set -euo pipefail
|
||||
TEAM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
AGENTS="$TEAM_ROOT/agents"
|
||||
SKILLS="$TEAM_ROOT/skills"
|
||||
MEMORY="$TEAM_ROOT/memory"
|
||||
IMPROV="$TEAM_ROOT/improvements"
|
||||
LS="$TEAM_ROOT/scripts/memory-lifecycle.sh"
|
||||
|
||||
PASS=0; FAIL=0
|
||||
T="$(mktemp -d)"
|
||||
trap 'rm -rf "$T"' EXIT
|
||||
|
||||
ok() { PASS=$((PASS+1)); echo "PASS $1"; }
|
||||
fail(){ FAIL=$((FAIL+1)); echo "FAIL $1 — ${2:-}" >&2; }
|
||||
|
||||
assert_file() { [ -f "$1" ] && return 0; return 1; }
|
||||
assert_dir() { [ -d "$1" ] && return 0; return 1; }
|
||||
assert_contains() { grep -qF "$2" "$1" 2>/dev/null && return 0; return 1; }
|
||||
|
||||
echo "init — team: $TEAM_ROOT"
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I01: memory-lifecycle.sh end-to-end store + recall + list + search
|
||||
# ======================================================================== #
|
||||
# Create an actual lesson file in the temp memory workspace, store it, verify it
|
||||
# is found by recall and search, then remove it (leave memory clean).
|
||||
LESSON_FILE="$T/lesson-integration-test.md"
|
||||
cat > "$LESSON_FILE" <<'EOF'
|
||||
---
|
||||
date: 2026-09-08
|
||||
category: lesson
|
||||
title: Integration test lesson
|
||||
tags: [integration, memory]
|
||||
---
|
||||
# Integration test lesson
|
||||
|
||||
A lesson stored and retrieved by the memory lifecycle script during integration
|
||||
testing. This entry is temporary and will be removed after verification.
|
||||
EOF
|
||||
|
||||
SC_OK=0
|
||||
# NOTE: capture output into variables rather than piping `memory-lifecycle.sh`
|
||||
# into `grep -q`. grep -q closes the pipe early (SIGPIPE, rc=141), which fails
|
||||
# the pipeline under `pipefail`.
|
||||
STORE_OK=0
|
||||
if bash "$LS" store lessons "$LESSON_FILE" >/dev/null 2>&1; then
|
||||
STORE_OK=1
|
||||
fi
|
||||
LIST_OUT="$(bash "$LS" list lessons 2>&1 || true)"
|
||||
RECALL_OUT="$(bash "$LS" recall lessons "Integration" 2>&1 || true)"
|
||||
SEARCH_OUT="$(bash "$LS" search "Integration test lesson" 2>&1 || true)"
|
||||
if [ "$STORE_OK" = "1" ] \
|
||||
&& echo "$LIST_OUT" | grep -q "integration-test" \
|
||||
&& echo "$RECALL_OUT" | grep -qi "Integration test lesson" \
|
||||
&& echo "$SEARCH_OUT" | grep -qi "integration-test"; then
|
||||
SC_OK=1
|
||||
fi
|
||||
# clean up whatever got written so memory stays pristine
|
||||
rm -f "$MEMORY/lessons/"*integration-test* 2>/dev/null || true
|
||||
if [ "$SC_OK" = "1" ]; then
|
||||
ok "I01 memory-lifecycle store/recall/list/search end-to-end"
|
||||
else
|
||||
fail "I01 memory-lifecycle store/recall/list/search end-to-end" "store/recall/list/search chain failed"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I02: Session lifecycle — create, list, cleanup
|
||||
# ======================================================================== #
|
||||
Sess="$MEMORY/sessions/2026-09-08_integration-session-test.md"
|
||||
printf -- '%s\n' "Status: active" "# Integration session test" "## State" "in progress" > "$Sess"
|
||||
SESS_OUT="$(bash "$LS" sessions 2>&1 || true)"
|
||||
SESS_OK=0
|
||||
if echo "$SESS_OUT" | grep -q "integration-session-test" \
|
||||
&& bash "$LS" cleanup >/dev/null 2>&1; then
|
||||
SESS_OK=1
|
||||
fi
|
||||
rm -f "$Sess" 2>/dev/null || true
|
||||
if [ "$SESS_OK" = "1" ]; then
|
||||
ok "I02 session create/list/cleanup"
|
||||
else
|
||||
fail "I02 session create/list/cleanup" "sessions chain failed — session listed?"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I03: agent↔skill mapping integrity — agent prompt names a skill or skill exists
|
||||
# ======================================================================== #
|
||||
# The agent-skill mapping in orchestrator.md should reference only skills that
|
||||
# exist as files. This is a hard consistency check.
|
||||
RULES_OK=1
|
||||
# every skill directory in skills/ has a SKILL.md with valid frontmatter
|
||||
for d in "$SKILLS"/*/; do
|
||||
n="$(basename "$d")"
|
||||
f="$d/SKILL.md"
|
||||
if ! assert_file "$f"; then fail "I03 skill $n has no SKILL.md"; RULES_OK=0; break; fi
|
||||
if ! grep -q "^name:" "$f" || ! grep -q "^owner:" "$f"; then
|
||||
fail "I03 skill $n missing frontmatter"; RULES_OK=0; break
|
||||
fi
|
||||
# description should be a single line (required frontmatter)
|
||||
if ! grep -q "^description:" "$f"; then fail "I03 skill $n missing description"; RULES_OK=0; break; fi
|
||||
done
|
||||
if [ "$RULES_OK" = "1" ]; then
|
||||
ok "I03 all skill files have valid frontmatter"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I04: Every skill's owner is a registered agent (case-insensitive,
|
||||
# and multi-owner "A + B" resolved to each named agent)
|
||||
# ======================================================================== #
|
||||
OWNER_OK=1
|
||||
for d in "$SKILLS"/*/; do
|
||||
n="$(basename "$d")"
|
||||
f="$d/SKILL.md"
|
||||
owner="$(grep -m1 "^owner:" "$f" 2>/dev/null | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
||||
[ -z "$owner" ] && continue
|
||||
# Resolve multi-owner "A + B" and case differences to agent file names (lowercase)
|
||||
IFS='+' read -ra owners <<< "$owner"
|
||||
for o in "${owners[@]}"; do
|
||||
o_trim="$(echo "$o" | xargs | tr '[:upper:]' '[:lower:]')"
|
||||
[ -f "$AGENTS/$o_trim.md" ] || { fail "I04 skill $n owner '$owner' not an agent" "missing agents/$o_trim.md"; OWNER_OK=0; }
|
||||
done
|
||||
done
|
||||
if [ "$OWNER_OK" = "1" ]; then
|
||||
ok "I04 every skill owner is a registered agent"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I05: Every agent named in orchestrator's skill mapping owns a real skill
|
||||
# ======================================================================== #
|
||||
# Agents in the Agent-Skill Mapping table should own at least one skill in skills/.
|
||||
MAPPED_OWNERS="$(grep -A2 "name:.*tdd" "$SKILLS/tdd/SKILL.md" >/dev/null && echo ok)"
|
||||
OWNER_LIST=""
|
||||
for d in "$SKILLS"/*/; do
|
||||
o="$(grep -m1 "^owner:" "$d/SKILL.md" | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
||||
[ -z "$o" ] && continue
|
||||
OWNER_LIST="$OWNER_LIST $o"
|
||||
done
|
||||
# Each mapped primary agent in the orchestrator should appear as an owner
|
||||
MAPPED_OK=1
|
||||
for ag in tdd systematic-debugging architecture-design code-review security-review \
|
||||
repository-analysis failure-analysis refactoring test-analysis \
|
||||
incident-investigation browser-automation research; do
|
||||
if ! [ -f "$SKILLS/$ag/SKILL.md" ]; then fail "I05 skill $ag missing"; MAPPED_OK=0; fi
|
||||
done
|
||||
if [ "$MAPPED_OK" = "1" ]; then
|
||||
ok "I05 all 12 skills present for mapping table"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I06: Orchestrator action catalog covers memory + skills + improvements (A23-A27)
|
||||
# ======================================================================== #
|
||||
ORCH="$AGENTS/orchestrator.md"
|
||||
CAT_OK=0
|
||||
if assert_contains "$ORCH" "A23 | recall project memory" \
|
||||
&& assert_contains "$ORCH" "A24 | store project memory" \
|
||||
&& assert_contains "$ORCH" "A25 | load skill" \
|
||||
&& assert_contains "$ORCH" "A26 | finish / report" \
|
||||
&& assert_contains "$ORCH" "A27 | re-plan"; then
|
||||
CAT_OK=1
|
||||
fi
|
||||
if [ "$CAT_OK" = "1" ]; then
|
||||
ok "I06 action catalog integrates memory/skills/finish/re-plan"
|
||||
else
|
||||
fail "I06 action catalog integrates memory/skills" "missing A23-A27 entries"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I07: Task lifecycle section exists and ties stages to memory
|
||||
# ======================================================================== #
|
||||
LC_OK=0
|
||||
if assert_contains "$ORCH" "Task Lifecycle" \
|
||||
&& assert_contains "$ORCH" "RECALL" \
|
||||
&& assert_contains "$ORCH" "STORE" \
|
||||
&& assert_contains "$ORCH" "IMPROVE"; then
|
||||
LC_OK=1
|
||||
fi
|
||||
if [ "$LC_OK" = "1" ]; then
|
||||
ok "I07 task lifecycle integrates memory/skills/improvements stages"
|
||||
else
|
||||
fail "I07 task lifecycle integrates memory/skills" "Task Lifecycle section missing stages"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I08: Skip-when routing guards present
|
||||
# ======================================================================== #
|
||||
SKIP_OK=0
|
||||
if grep -q "Skip-when" "$ORCH" && grep -q "Skip Explorer" "$ORCH"; then
|
||||
SKIP_OK=1
|
||||
fi
|
||||
if [ "$SKIP_OK" = "1" ]; then
|
||||
ok "I08 skip-when routing guards present"
|
||||
else
|
||||
fail "I08 skip-when routing guards present" "missing skip-when section"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I09: improvement proposal lifecycle — write pending, move to applied
|
||||
# ======================================================================== #
|
||||
PROP="$IMPROV/pending/2026-09-08_integration-prop.md"
|
||||
mkdir -p "$IMPROV/pending" "$IMPROV/applied" "$IMPROV/rejected"
|
||||
printf -- '%s\n' "# Proposal: integration test" "## Problem" "temporary" > "$PROP"
|
||||
PROP_OK=0
|
||||
if [ -f "$PROP" ] \
|
||||
&& assert_contains "$IMPROV/README.md" "human approval" \
|
||||
&& mkdir -p "$IMPROV/applied" \
|
||||
&& mv "$PROP" "$IMPROV/applied/2026-09-08_integration-prop.md" 2>/dev/null; then
|
||||
PROP_OK=1
|
||||
fi
|
||||
# restore: remove applied copy
|
||||
rm -f "$IMPROV/applied/2026-09-08_integration-prop.md" 2>/dev/null || true
|
||||
if [ "$PROP_OK" = "1" ]; then
|
||||
ok "I09 improvement proposal pending→applied lifecycle"
|
||||
else
|
||||
fail "I09 improvement proposal lifecycle" "proposal lifecycle failed"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I10: Improper agent (unknown owner) not silently accepted
|
||||
# ======================================================================== #
|
||||
# Create a temp skill with an invalid owner and verify the mapping check would reject it.
|
||||
INVALID_SKILL="$T/invalid/SKILL.md"
|
||||
mkdir -p "$T/invalid"
|
||||
printf -- '%s\n' "---" "name: invalid" "owner: not-an-agent" "---" > "$INVALID_SKILL"
|
||||
INV_OK=1
|
||||
for f in "$SKILLS"/*/SKILL.md; do
|
||||
o="$(grep -m1 "^owner:" "$f" 2>/dev/null | sed 's/^owner:[[:space:]]*//' | sed 's/[",]//g')"
|
||||
[ -z "$o" ] && continue
|
||||
IFS='+' read -ra owners <<< "$o"
|
||||
for ow in "${owners[@]}"; do
|
||||
ow_trim="$(echo "$ow" | xargs | tr '[:upper:]' '[:lower:]')"
|
||||
if [ -n "$ow_trim" ] && [ ! -f "$AGENTS/$ow_trim.md" ]; then
|
||||
fail "I10 skill owner validation" "invalid owner '$o' in $f"
|
||||
INV_OK=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ "$INV_OK" = "0" ] && break
|
||||
done
|
||||
if [ "$INV_OK" = "1" ]; then
|
||||
ok "I10 real skills all have valid registered owners"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I11: Orchestrator wires improvement proposal detection into workflow
|
||||
# ======================================================================== #
|
||||
ORCH="$AGENTS/orchestrator.md"
|
||||
IMPROVE_DET_OK=0
|
||||
if assert_contains "$ORCH" "Recurring failure" \
|
||||
&& assert_contains "$ORCH" "Missing skill" \
|
||||
&& assert_contains "$ORCH" "Routing inefficiency" \
|
||||
&& assert_contains "$ORCH" "Documentation gap" \
|
||||
&& assert_contains "$ORCH" "Process friction"; then
|
||||
IMPROVE_DET_OK=1
|
||||
fi
|
||||
if [ "$IMPROVE_DET_OK" = "1" ]; then
|
||||
ok "I11 improvement proposal detection triggers wired"
|
||||
else
|
||||
fail "I11 improvement proposal detection" "missing detection triggers in orchestrator"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# TEST I12: Orchestrator proposal format matches improvements/README format
|
||||
# ======================================================================== #
|
||||
FMT_OK=0
|
||||
if assert_contains "$ORCH" "improvements/pending/YYYY-MM-DD" \
|
||||
&& assert_contains "$IMPROV/README.md" "Observed problem" \
|
||||
&& assert_contains "$IMPROV/README.md" "Root cause" \
|
||||
&& assert_contains "$IMPROV/README.md" "Proposed change" \
|
||||
&& assert_contains "$IMPROV/README.md" "Verification plan" \
|
||||
&& assert_contains "$IMPROV/README.md" "without human approval"; then
|
||||
FMT_OK=1
|
||||
fi
|
||||
if [ "$FMT_OK" = "1" ]; then
|
||||
ok "I12 proposal format consistent across orchestrator and README"
|
||||
else
|
||||
fail "I12 proposal format consistency" "format fields missing"
|
||||
fi
|
||||
|
||||
# ======================================================================== #
|
||||
# Summary
|
||||
# ======================================================================== #
|
||||
echo
|
||||
echo "==================== SUMMARY ===================="
|
||||
echo "PASS: $PASS FAIL: $FAIL"
|
||||
[ "$FAIL" = "0" ] && echo "RESULT: ALL PASS" || echo "RESULT: FAILURES PRESENT"
|
||||
exit $(( FAIL > 0 ? 1 : 0 ))
|
||||
Reference in New Issue
Block a user