364 lines
17 KiB
Bash
Executable File
364 lines
17 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# verify-api.sh — post-deploy smoke test for the AgenticCode REST API.
|
|
#
|
|
# Usage:
|
|
# ./x-scripts/verify-api.sh [-p <project>] # default project: ac
|
|
# AC_SERVER_URL=http://host:8787 ./x-scripts/verify-api.sh -p pur
|
|
#
|
|
# Exit 0 = every check passed, 1 = at least one failed, 2 = usage/precondition error.
|
|
#
|
|
# What this IS: a check that the *deployed* server, against the *real* graph, still answers
|
|
# plausibly — the failure class that only ever showed up in manual live probing ('//file' in
|
|
# REST paths, duplicated rows, an inherited @Path collapsing to 'POST /', ?paths=pom.xml
|
|
# being accepted). It runs in well under a minute and touches no build tooling.
|
|
#
|
|
# What this is NOT: a replacement for the integration tests. Those pin semantics; this pins
|
|
# "the thing we just deployed is not obviously broken". Never treat a green run here as a
|
|
# quality gate.
|
|
#
|
|
# Assertions are INVARIANTS (> 0, no duplicates, header present, required field set), never
|
|
# fixed row counts — counts move with every refresh and differ per project.
|
|
#
|
|
# Mutation: the run is read-only except for one call, `refresh?paths=pom.xml`, which by
|
|
# design resolves no source file and therefore starts no ingest. It does invalidate the
|
|
# project metadata cache. Nothing else writes.
|
|
|
|
set -uo pipefail
|
|
|
|
SERVER="${AC_SERVER_URL:-http://localhost:8787}"
|
|
API="$SERVER/api"
|
|
PROJECT="ac"
|
|
|
|
usage() { echo "Usage: $0 [-p <project>]" >&2; exit 2; }
|
|
while getopts ":p:h" opt; do
|
|
case "$opt" in
|
|
p) PROJECT="$OPTARG" ;;
|
|
h) usage ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
|
|
command -v curl >/dev/null 2>&1 || { echo "verify-api.sh: curl is required." >&2; exit 2; }
|
|
command -v python3 >/dev/null 2>&1 || { echo "verify-api.sh: python3 is required." >&2; exit 2; }
|
|
|
|
PASSED=0
|
|
FAILED=0
|
|
START=$(date +%s)
|
|
|
|
GREEN=$'\033[0;32m'; RED=$'\033[0;31m'; BOLD=$'\033[1m'; DIM=$'\033[2m'; OFF=$'\033[0m'
|
|
|
|
pass() { PASSED=$((PASSED + 1)); printf ' %sPASS%s %-52s %s%s%s\n' "$GREEN" "$OFF" "$1" "$DIM" "${2:-}" "$OFF"; }
|
|
fail() { FAILED=$((FAILED + 1)); printf ' %sFAIL%s %-52s %s\n' "$RED" "$OFF" "$1" "${2:-}"; }
|
|
group() { printf '\n%s%s%s\n' "$BOLD" "$1" "$OFF"; }
|
|
|
|
# check <name> <condition-exit-code> <detail>
|
|
check() {
|
|
if [[ "$2" -eq 0 ]]; then pass "$1" "${3:-}"; else fail "$1" "${3:-}"; fi
|
|
}
|
|
|
|
BODY=$(mktemp); HEAD=$(mktemp)
|
|
trap 'rm -f "$BODY" "$HEAD"' EXIT
|
|
|
|
# get <path...> -> sets STATUS, body in $BODY, headers in $HEAD
|
|
get() {
|
|
STATUS=$(curl -s -m 30 -o "$BODY" -D "$HEAD" -w '%{http_code}' "$@" 2>/dev/null)
|
|
[[ -n "$STATUS" ]] || STATUS=000
|
|
}
|
|
|
|
hdr() { grep -i "^$1:" "$HEAD" | head -1 | cut -d' ' -f2- | tr -d '\r'; }
|
|
|
|
# py <expr-script> — runs python3 against the body; exit 0 means the assertion held.
|
|
py() { python3 -c "$1" "$BODY" "${@:2}" 2>/dev/null; }
|
|
|
|
printf '%sverify-api.sh%s server=%s project=%s\n' "$BOLD" "$OFF" "$SERVER" "$PROJECT"
|
|
|
|
# --- 1. reachability & version ------------------------------------------------------------
|
|
group "1. Reachability & version"
|
|
|
|
get "$API/version"
|
|
check "GET /api/version returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
if [[ $STATUS == 200 ]]; then
|
|
VERSION=$(py 'import json,sys; d=json.load(open(sys.argv[1])); v=str(d.get("version","")); print(v); sys.exit(0 if v else 1)')
|
|
check "version is non-empty" $? "version=$VERSION"
|
|
else
|
|
echo " server unreachable at $SERVER — is the stack deployed (./manage-ac.sh deploy)?" >&2
|
|
fi
|
|
|
|
get "$API/projects"
|
|
check "GET /api/projects returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if any(p.get("name")==sys.argv[2] for p in d) else 1)' "$PROJECT"
|
|
check "project '$PROJECT' is listed" $?
|
|
|
|
if [[ $FAILED -gt 0 ]]; then
|
|
echo
|
|
echo "Aborting: the server is not usable, later checks would only add noise." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- 2. scope & freshness headers (item 130) ----------------------------------------------
|
|
group "2. Scope & freshness headers (item 130)"
|
|
|
|
get "$API/projects/$PROJECT/modules?limit=1"
|
|
EXCL=$(hdr X-AC-Exclude-Dirs); AT=$(hdr X-AC-Ingested-At); INC=$(hdr X-AC-Ingest-Incomplete)
|
|
check "X-AC-Exclude-Dirs present" "$([[ -n $EXCL ]] && echo 0 || echo 1)" "$EXCL"
|
|
check "X-AC-Ingested-At present" "$([[ -n $AT ]] && echo 0 || echo 1)" "$AT"
|
|
check "X-AC-Ingest-Incomplete=false" "$([[ $INC == false ]] && echo 0 || echo 1)" \
|
|
"$([[ $INC == false ]] || echo "got '$INC' — a refresh was aborted or is running")"
|
|
|
|
# --- 3. paging contract (item 131) --------------------------------------------------------
|
|
group "3. Paging contract (item 131)"
|
|
|
|
# Endpoints that carry X-AC-Total-Count, each with a query that yields rows in any project.
|
|
paging_check() {
|
|
local label="$1" path="$2"
|
|
|
|
get "$API/projects/$PROJECT/$path"
|
|
local total; total=$(hdr X-AC-Total-Count)
|
|
if ! [[ $total =~ ^[0-9]+$ ]]; then
|
|
fail "$label: X-AC-Total-Count is numeric" "got '$total' (status=$STATUS)"
|
|
return
|
|
fi
|
|
pass "$label: X-AC-Total-Count is numeric" "total=$total"
|
|
|
|
if [[ $total -eq 0 ]]; then
|
|
printf ' %sSKIP%s %-52s %s%s%s\n' "$DIM" "$OFF" "$label: paging (no rows to page)" "$DIM" "total=0" "$OFF"
|
|
return
|
|
fi
|
|
|
|
# countOnly must report the same total without returning the page.
|
|
get "$API/projects/$PROJECT/$(printf %s "$path" | sed 's/?/?countOnly=true\&/')"
|
|
local co; co=$(hdr X-AC-Total-Count)
|
|
check "$label: countOnly agrees with full total" \
|
|
"$([[ $co == "$total" ]] && echo 0 || echo 1)" "countOnly=$co full=$total"
|
|
|
|
# limit=1 must flag truncation whenever more than one row exists.
|
|
get "$API/projects/$PROJECT/$(printf %s "$path" | sed 's/?/?limit=1\&/')"
|
|
local tr; tr=$(hdr X-AC-Truncated)
|
|
local want=false; [[ $total -gt 1 ]] && want=true
|
|
check "$label: limit=1 sets X-AC-Truncated=$want" \
|
|
"$([[ $tr == "$want" ]] && echo 0 || echo 1)" "truncated=$tr total=$total"
|
|
|
|
# Two pages of 1 must be disjoint — the offset actually moves.
|
|
local p0 p1
|
|
get "$API/projects/$PROJECT/$(printf %s "$path" | sed 's/?/?limit=1\&offset=0\&/')"
|
|
p0=$(py 'import json,sys; d=json.load(open(sys.argv[1])); print(json.dumps(d[0],sort_keys=True) if d else "")')
|
|
get "$API/projects/$PROJECT/$(printf %s "$path" | sed 's/?/?limit=1\&offset=1\&/')"
|
|
p1=$(py 'import json,sys; d=json.load(open(sys.argv[1])); print(json.dumps(d[0],sort_keys=True) if d else "")')
|
|
if [[ $total -gt 1 ]]; then
|
|
check "$label: offset=0 and offset=1 are disjoint" \
|
|
"$([[ -n $p0 && -n $p1 && $p0 != "$p1" ]] && echo 0 || echo 1)"
|
|
fi
|
|
}
|
|
|
|
paging_check "search/identifier" "search/identifier?name=e&contains=true"
|
|
|
|
# A deep page must not blow up: a row with a NULL startLine used to escape as an unstructured
|
|
# 500 ("Cannot coerce NULL to Java int") that only appears past the first few hundred rows.
|
|
get "$API/projects/$PROJECT/search/identifier?name=e&contains=true&limit=500"
|
|
check "search/identifier: a 500-row page does not fault" \
|
|
"$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
paging_check "search/annotation" "search/annotation?name=ApplicationScoped"
|
|
paging_check "search/value" "search/value?value=project"
|
|
paging_check "search/references" "search/references?name=Logger"
|
|
paging_check "rest-endpoints" "rest-endpoints?"
|
|
paging_check "counterparts" "counterparts?"
|
|
# Item 194: the store endpoints answer for every project (an empty list on a non-TypeScript one).
|
|
get "$API/projects/$PROJECT/store"
|
|
check "store: answers 200 with a list" "$([[ $STATUS == 200 && $(head -c1 "$BODY") == '[' ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
# Item 195: bindings is a paged list on every project (empty on a non-TypeScript one).
|
|
paging_check "bindings" "bindings?"
|
|
# Item 196: theme and styles answer on every project (empty on a non-TypeScript one).
|
|
get "$API/projects/$PROJECT/theme"
|
|
check "theme: answers 200 with a list" "$([[ $STATUS == 200 && $(head -c1 "$BODY") == '[' ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
paging_check "styles" "styles?"
|
|
|
|
# --- 4. data plausibility -----------------------------------------------------------------
|
|
group "4. Data plausibility"
|
|
|
|
# rest-endpoints: the concrete bugs that only real data exposed.
|
|
get "$API/projects/$PROJECT/rest-endpoints?limit=500"
|
|
if [[ $STATUS == 200 ]]; then
|
|
# A pure Natural project declares no HTTP endpoints — absence is correct there, not a defect.
|
|
if ! py 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1])) else 1)'; then
|
|
printf ' %sSKIP%s %-52s %s%s%s\n' "$DIM" "$OFF" "rest-endpoints (project declares none)" \
|
|
"$DIM" "0 rows — expected for a non-JAX-RS project" "$OFF"
|
|
REST_ROWS=0
|
|
else
|
|
pass "rest-endpoints returns rows"
|
|
REST_ROWS=1
|
|
fi
|
|
if [[ $REST_ROWS == 1 ]]; then
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); bad=[e["path"] for e in d if "//" in e["path"]]; print(*bad[:3]); sys.exit(1 if bad else 0)'
|
|
check "no path contains '//'" $?
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); bad=[e["path"] for e in d if not e["path"].startswith("/")]; print(*bad[:3]); sys.exit(1 if bad else 0)'
|
|
check "every path starts with '/'" $?
|
|
# NOTE: the query already applies DISTINCT, so this cannot surface item 75's duplicate
|
|
# CONTAINS edges — it only guards against that DISTINCT being dropped again.
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
k=[(e["httpMethod"],e["path"],e["module"],e["handler"]) for e in d]
|
|
dup=len(k)-len(set(k)); print("duplicates:",dup); sys.exit(1 if dup else 0)'
|
|
check "no duplicate endpoint rows" $?
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if all("outbound" in e for e in d) else 1)'
|
|
check "every row carries the outbound flag" $?
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if any(e["httpMethod"]=="GET" and e["path"]=="/api/version" for e in d) else 1)'
|
|
SELF=$?
|
|
if [[ $PROJECT == ac ]]; then
|
|
check "the server's own GET /api/version is found" $SELF
|
|
fi
|
|
fi
|
|
else
|
|
fail "rest-endpoints returns 200" "status=$STATUS"
|
|
fi
|
|
|
|
# Pick a module that actually exists in this project, then exercise the module endpoints.
|
|
get "$API/projects/$PROJECT/modules?limit=200"
|
|
CANDIDATES=$(py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
c=[m for m in d if m.get("ingestDepth")=="FULL"] or d
|
|
print("\n".join(m["name"] for m in c[:20]))')
|
|
# Prefer a module that actually calls something — a leaf would make the callees check vacuous.
|
|
MODULE=""; CALLEE_MODULE=""
|
|
while IFS= read -r cand; do
|
|
[[ -n $cand ]] || continue
|
|
[[ -n $MODULE ]] || MODULE="$cand"
|
|
E=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1],safe=""))' "$cand")
|
|
get "$API/projects/$PROJECT/modules/$E/callees?limit=20"
|
|
if [[ $STATUS == 200 ]] && py 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1])).get("items") else 1)'; then
|
|
CALLEE_MODULE="$cand"; break
|
|
fi
|
|
done <<< "$CANDIDATES"
|
|
check "a FULL-ingested module is available" "$([[ -n $MODULE ]] && echo 0 || echo 1)" "module=$MODULE"
|
|
|
|
nonempty_rows() { # <label> <path> — 200 and a non-empty array
|
|
get "$API/projects/$PROJECT/$2"
|
|
if [[ $STATUS != 200 ]]; then fail "$1" "status=$STATUS"; return; fi
|
|
py 'import json,sys; d=json.load(open(sys.argv[1])); print(len(d),"rows"); sys.exit(0 if d else 1)'
|
|
check "$1" $? ""
|
|
}
|
|
|
|
fields_set() { # <label> <path> <field...> — 200 and every row has the fields
|
|
get "$API/projects/$PROJECT/$2"
|
|
if [[ $STATUS != 200 ]]; then fail "$1" "status=$STATUS"; return; fi
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
rows=d if isinstance(d,list) else [d]
|
|
miss={f for r in rows for f in sys.argv[2:] if r.get(f) is None}
|
|
print("missing:",",".join(sorted(miss)) or "-")
|
|
sys.exit(1 if miss else 0)' "${@:3}"
|
|
check "$1" $? ""
|
|
}
|
|
|
|
if [[ -n $MODULE ]]; then
|
|
ENC=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1],safe=""))' "$MODULE")
|
|
# callees answers with the shared file-index envelope {sourceFiles, items}: every item must
|
|
# name a callee and every sourceFileIndex must resolve into the sourceFiles table.
|
|
if [[ -z $CALLEE_MODULE ]]; then
|
|
printf ' %sSKIP%s %-52s %s%s%s\n' "$DIM" "$OFF" "callees: sourceFileIndex resolves" \
|
|
"$DIM" "no module among the first 20 has callees" "$OFF"
|
|
else
|
|
CENC=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1],safe=""))' "$CALLEE_MODULE")
|
|
get "$API/projects/$PROJECT/modules/$CENC/callees?limit=20"
|
|
if [[ $STATUS == 200 ]]; then
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
files=d.get("sourceFiles",[]); items=d.get("items",[])
|
|
def idx(i):
|
|
v = i.get("sourceFileIndex")
|
|
return v if isinstance(v, int) else -1
|
|
bad=[i.get("name") for i in items if not i.get("name") or not (0 <= idx(i) < len(files))]
|
|
print(len(items),"callees,",len(files),"files; bad:",bad[:3] or "-")
|
|
sys.exit(1 if (not items or bad) else 0)'
|
|
check "callees: every item resolves its sourceFileIndex" $? "via $CALLEE_MODULE"
|
|
else
|
|
fail "callees returns 200" "status=$STATUS"
|
|
fi
|
|
fi
|
|
fields_set "functions rows carry name/startLine" "modules/$ENC/functions?limit=20" name startLine
|
|
get "$API/projects/$PROJECT/modules/$ENC/context"
|
|
check "context returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
get "$API/projects/$PROJECT/modules/$ENC/digest"
|
|
check "digest returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
get "$API/projects/$PROJECT/modules/$ENC/graph"
|
|
check "graph returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
get "$API/projects/$PROJECT/modules/$ENC/source"
|
|
check "source returns 200 (not STALE_SOURCE)" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" \
|
|
"$([[ $STATUS == 200 ]] || echo "status=$STATUS — the graph is behind disk, refresh needed")"
|
|
fi
|
|
|
|
nonempty_rows "modules returns rows" "modules?limit=5"
|
|
nonempty_rows "loc returns rows" "loc?limit=5"
|
|
# search/source answers {regex, truncated, matches} — not a bare array.
|
|
get "$API/projects/$PROJECT/search/source?regex=project&limit=5"
|
|
if [[ $STATUS == 200 ]]; then
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
m=d.get("matches",[])
|
|
bad=[x for x in m if not x.get("sourceFile") or not x.get("lineNo")]
|
|
print(len(m),"matches; truncated:",d.get("truncated"))
|
|
sys.exit(1 if (not m or bad) else 0)'
|
|
check "search/source matches carry sourceFile/lineNo" $? ""
|
|
else
|
|
fail "search/source returns 200" "status=$STATUS"
|
|
fi
|
|
|
|
# A node id from a live result must resolve through /nodes/{id} and /nodes/{id}/source.
|
|
# Pick a RESOLVED hit — an unresolved placeholder has no source file by design and would
|
|
# legitimately answer 400 NO_SOURCE_FILE.
|
|
SEED="${MODULE:-e}"
|
|
get "$API/projects/$PROJECT/search/identifier?name=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1],safe=""))' "$SEED")&contains=true&limit=200"
|
|
NODE=$(py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
r=[n for n in d if n.get("sourceFile") and not n.get("unresolved")]
|
|
print(r[0]["id"] if r else "")')
|
|
if [[ -n $NODE ]]; then
|
|
get "$API/projects/$PROJECT/nodes/$NODE"
|
|
check "a live node id resolves via /nodes/{id}" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
get "$API/projects/$PROJECT/nodes/$NODE/source"
|
|
check "/nodes/{id}/source returns 200" "$([[ $STATUS == 200 ]] && echo 0 || echo 1)" "status=$STATUS"
|
|
else
|
|
fail "search/identifier yields a node id"
|
|
fi
|
|
|
|
# --- 5. negative cases --------------------------------------------------------------------
|
|
group "5. Negative cases"
|
|
|
|
structured_404() { # <label> <path> <expected-code>
|
|
get "$API/projects/$2"
|
|
if [[ $STATUS != 404 ]]; then fail "$1" "status=$STATUS (expected 404)"; return; fi
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
ok = d.get("code")==sys.argv[2] and d.get("error") and isinstance(d.get("details"),dict)
|
|
print("code="+str(d.get("code")))
|
|
sys.exit(0 if ok else 1)' "$3"
|
|
check "$1" $? ""
|
|
}
|
|
|
|
structured_404 "unknown project -> 404 PROJECT_NOT_FOUND" "nope-does-not-exist/modules" PROJECT_NOT_FOUND
|
|
structured_404 "unknown module -> 404 MODULE_NOT_FOUND" "$PROJECT/modules/ZZZ-DOES-NOT-EXIST/callers" MODULE_NOT_FOUND
|
|
|
|
# Item 129 regression: a non-source path must come back unresolved, not be silently ingested.
|
|
STATUS=$(curl -s -m 30 -o "$BODY" -D "$HEAD" -w '%{http_code}' -X POST \
|
|
"$API/projects/$PROJECT/refresh?paths=pom.xml" 2>/dev/null)
|
|
if [[ $STATUS == 200 ]]; then
|
|
py 'import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
u=d.get("unresolved") or []
|
|
print("unresolved:",u, "persisted:", d.get("filesPersisted"))
|
|
sys.exit(0 if "pom.xml" in u and not d.get("filesPersisted") else 1)'
|
|
check "refresh?paths=pom.xml is unresolved, nothing ingested" $? ""
|
|
else
|
|
fail "refresh?paths=pom.xml returns 200" "status=$STATUS"
|
|
fi
|
|
|
|
# --- summary ------------------------------------------------------------------------------
|
|
ELAPSED=$(( $(date +%s) - START ))
|
|
printf '\n%s' "$BOLD"
|
|
if [[ $FAILED -eq 0 ]]; then
|
|
printf '%sAll %d checks passed%s (%ss, server v%s, project %s)\n' "$GREEN" "$PASSED" "$OFF" "$ELAPSED" "${VERSION:-?}" "$PROJECT"
|
|
exit 0
|
|
fi
|
|
printf '%s%d passed, %d FAILED%s (%ss, server v%s, project %s)\n' "$RED" "$PASSED" "$FAILED" "$OFF" "$ELAPSED" "${VERSION:-?}" "$PROJECT"
|
|
exit 1
|