blackroad-os/blackroad
9
1
Fork 0
Code Issues 1 Pull Requests Actions 3 Packages Projects Releases 1 Wiki Activity
Files
ca0d622681f368e173a2e8982f021d978050bb80
blackroad/scripts/blackroad-e2e-test.sh
Alexa Amundson 78fbe80f2a Initial monorepo — everything BlackRoad in one place 
bin/       230 CLI tools (ask-*, br-*, agent-*, roadid, carpool)
scripts/   99 automation scripts
fleet/     Node configs and deployment
workers/   Cloudflare Worker sources (roadpay, road-search, squad webhooks)
roadc/     RoadC programming language
roadnet/   Mesh network (5 APs, WireGuard)
operator/  Memory system scripts
config/    System configs
dotfiles/  Shell configs
docs/      Documentation

BlackRoad OS — Pave Tomorrow.

RoadChain-SHA2048: d1a24f55318d338b
RoadChain-Identity: alexa@sovereign
RoadChain-Full: d1a24f55318d338b24b60bad7be39286379c76ae5470817482100cb0ddbbcb97e147d07ac7243da0a9f0363e4e5c833d612b9c0df3a3cd20802465420278ef74875a5b77f55af6fe42a931b8b635b3d0d0b6bde9abf33dc42eea52bc03c951406d8cbe49f1a3d29b26a94dade05e9477f34a7d4d4c6ec4005c3c2ac54e73a68440c512c8e83fd9b1fe234750b898ef8f4032c23db173961fe225e67a0432b5293a9714f76c5c57ed5fdf35b9fb40fd73c03ebf88b7253c6a0575f5afb6a6b49b3bda310602fb1ef676859962dad2aebbb2875814b30eee0a8ba195e482d4cbc91d8819e7f38f6db53e8063401649c77bb994371473cabfb917fb53e8cbe73d60
2026-03-14 17:08:41 -05:00

413 lines
19 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# BlackRoad OS End-to-End Test Runner
# One command to rule them all — validates the entire stack
# Usage: ./blackroad-e2e-test.sh
set -o pipefail
# ── BlackRoad Brand Colors ──────────────────────────────────────────
PINK='\033[38;5;205m'
AMBER='\033[38;5;214m'
BLUE='\033[38;5;69m'
VIOLET='\033[38;5;135m'
GREEN='\033[38;5;82m'
RED='\033[38;5;196m'
DIM='\033[2m'
BOLD='\033[1m'
RESET='\033[0m'
# ── Test Counters ───────────────────────────────────────────────────
PASS_COUNT=0
FAIL_COUNT=0
TOTAL_COUNT=0
FAILURES=()
START_TIME=$(date +%s)
TEST_TIMEOUT=35
# ── Timeout Command (macOS compat) ─────────────────────────────────
if command -v gtimeout &>/dev/null; then
TIMEOUT_CMD="gtimeout"
elif command -v timeout &>/dev/null; then
TIMEOUT_CMD="timeout"
else
# Pure-bash fallback using background process
TIMEOUT_CMD=""
fi
# ── Pi Definitions ──────────────────────────────────────────────────
declare -A PI_IPS=(
[Alice]="192.168.4.49"
[Cecilia]="192.168.4.96"
[Octavia]="192.168.4.101"
[Aria]="192.168.4.98"
[Lucidia]="192.168.4.38"
)
declare -A PI_SSH_USERS=(
[Alice]="pi"
[Cecilia]="blackroad"
[Octavia]="pi"
[Aria]="blackroad"
[Lucidia]="octavia"
)
# ── Cloudflare Domains (blackroad.io) ────────────────────────────────
CF_DOMAINS=(
"blackroad.io"
"auth.blackroad.io"
"chat.blackroad.io"
"portal.blackroad.io"
"index.blackroad.io"
"images.blackroad.io"
"resume.blackroad.io"
"api.blackroad.io"
"status.blackroad.io"
"app.blackroad.io"
"docs.blackroad.io"
"git.blackroad.io"
"ollama.blackroad.io"
)
# ── Service Port Map (Pi → port list) ──────────────────────────────
# Alice: Pi-hole(80,53), PostgreSQL(5432)
# Cecilia: Ollama(11434), MinIO(9000,9001), PostgreSQL(5432), CECE API(8787)
# Octavia: OctoPrint(5000), InfluxDB(8086), GitHub Runner
# Aria: Ollama(11434), InfluxDB(8086), Headscale(8088), Portainer(9443)
declare -A PI_SERVICES
PI_SERVICES[Alice]="Pi-hole:80 DNS:53 PostgreSQL:5432"
PI_SERVICES[Cecilia]="Ollama:11434 MinIO-API:9000 MinIO-Console:9001 PostgreSQL:5432"
PI_SERVICES[Octavia]="Gitea:3100 NATS:4222 Ollama:11434"
PI_SERVICES[Aria]="Ollama:11434 InfluxDB:8086 Headscale:8088 Portainer:9443"
PI_SERVICES[Lucidia]="SSH:22 DNS:53"
# ── Helpers ─────────────────────────────────────────────────────────
header() {
echo ""
echo -e "${PINK}${BOLD}╔══════════════════════════════════════════════════════════════╗${RESET}"
echo -e "${PINK}${BOLD}║${RESET} ${AMBER}${BOLD}B L A C K R O A D O S${RESET} ${VIOLET}End-to-End Test Runner${RESET} ${PINK}${BOLD}║${RESET}"
echo -e "${PINK}${BOLD}╚══════════════════════════════════════════════════════════════╝${RESET}"
echo -e "${DIM} Host: $(hostname) | $(date '+%Y-%m-%d %H:%M:%S %Z') | PID $$${RESET}"
echo ""
}
section() {
echo ""
echo -e "${BLUE}${BOLD}── $1 ──────────────────────────────────────────${RESET}"
}
_run_with_timeout() {
local secs="$1"
shift
if [[ -n "$TIMEOUT_CMD" ]]; then
"$TIMEOUT_CMD" "$secs" bash -c "$*" 2>&1
else
# Pure-bash timeout fallback for macOS without coreutils
local output_file
output_file=$(mktemp)
bash -c "$*" >"$output_file" 2>&1 &
local pid=$!
local i=0
while kill -0 "$pid" 2>/dev/null; do
if [[ $i -ge $secs ]]; then
kill -9 "$pid" 2>/dev/null
wait "$pid" 2>/dev/null
cat "$output_file"
rm -f "$output_file"
return 124
fi
sleep 1
i=$((i + 1))
done
wait "$pid"
local rc=$?
cat "$output_file"
rm -f "$output_file"
return $rc
fi
}
run_test() {
local name="$1"
shift
TOTAL_COUNT=$((TOTAL_COUNT + 1))
local output
if output=$(_run_with_timeout "$TEST_TIMEOUT" "$*"); then
PASS_COUNT=$((PASS_COUNT + 1))
echo -e " ${GREEN}[PASS]${RESET} ${name}"
if [[ -n "$output" && "$VERBOSE" == "1" ]]; then
echo -e " ${DIM}${output}${RESET}"
fi
return 0
else
local rc=$?
FAIL_COUNT=$((FAIL_COUNT + 1))
FAILURES+=("$name")
if [[ $rc -eq 124 ]]; then
echo -e " ${RED}[FAIL]${RESET} ${name} ${DIM}(timeout after ${TEST_TIMEOUT}s)${RESET}"
else
echo -e " ${RED}[FAIL]${RESET} ${name}"
fi
if [[ -n "$output" ]]; then
echo -e " ${DIM}${output:0:200}${RESET}"
fi
return 1
fi
}
# ── HEADER ──────────────────────────────────────────────────────────
header
# ════════════════════════════════════════════════════════════════════
# 1. PI MESH — ping, SSH, uptime
# ════════════════════════════════════════════════════════════════════
section "1. Pi Mesh Connectivity"
for pi in Alice Cecilia Octavia Aria Lucidia; do
ip="${PI_IPS[$pi]}"
user="${PI_SSH_USERS[$pi]}"
run_test "$pi ping ($ip)" \
"ping -c1 -W3 '$ip' >/dev/null 2>&1"
run_test "$pi SSH" \
"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes '$user@$ip' 'echo ok' >/dev/null 2>&1"
run_test "$pi uptime" \
"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes '$user@$ip' 'uptime' 2>/dev/null | head -1"
done
# ════════════════════════════════════════════════════════════════════
# 2. SERVICES — check ports on each Pi via SSH
# ════════════════════════════════════════════════════════════════════
section "2. Service Port Checks (via SSH)"
for pi in Alice Cecilia Octavia Aria Lucidia; do
ip="${PI_IPS[$pi]}"
user="${PI_SSH_USERS[$pi]}"
services="${PI_SERVICES[$pi]}"
for svc in $services; do
svc_name="${svc%%:*}"
port="${svc##*:}"
run_test "$pi → $svc_name (:$port)" \
"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes '$user@$ip' 'ss -tln | grep -q :$port' 2>/dev/null"
done
done
# ════════════════════════════════════════════════════════════════════
# 3. CLOUDFLARE — curl top 10 domains
# ════════════════════════════════════════════════════════════════════
section "3. Cloudflare Domains"
for domain in "${CF_DOMAINS[@]}"; do
run_test "https://$domain" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 'https://$domain'); [[ \"\$code\" =~ ^(200|301|302|303|307|308|401|403|404)$ ]]"
done
# ════════════════════════════════════════════════════════════════════
# 4. API GATEWAY — health, agents, tools
# ════════════════════════════════════════════════════════════════════
section "4. API Gateway"
API_BASE="https://api.blackroad.io"
run_test "GET $API_BASE/healthz" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 '$API_BASE/healthz'); [[ \"\$code\" =~ ^(200|204)$ ]]"
run_test "GET $API_BASE/v1/models" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 '$API_BASE/v1/models'); [[ \"\$code\" =~ ^(200|401|403)$ ]]"
run_test "GET $API_BASE/v1/providers" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 '$API_BASE/v1/providers'); [[ \"\$code\" =~ ^(200|401|403)$ ]]"
# ════════════════════════════════════════════════════════════════════
# 5. MEMORY SYSTEM — log, verify, search (local CLI)
# ════════════════════════════════════════════════════════════════════
section "5. Memory System"
run_test "Memory: log entry" \
"bash ~/memory-system.sh log 'e2e-test' 'Automated e2e test run' 2>&1"
run_test "Memory: verify" \
"bash ~/memory-system.sh verify 2>&1"
run_test "Memory: search" \
"bash ~/memory-indexer.sh search 'e2e-test' 2>&1"
# ════════════════════════════════════════════════════════════════════
# 6. TASK MARKETPLACE — post, claim, complete (local CLI)
# ════════════════════════════════════════════════════════════════════
section "6. Task Marketplace"
TASK_ID="e2e-$(date +%s)"
run_test "Task: post" \
"bash ~/memory-task-marketplace.sh post '$TASK_ID' 'E2E Test' 'Automated test' high e2e 2>&1"
run_test "Task: claim" \
"bash ~/memory-task-marketplace.sh claim '$TASK_ID' 2>&1"
run_test "Task: complete" \
"bash ~/memory-task-marketplace.sh complete '$TASK_ID' 2>&1"
# ════════════════════════════════════════════════════════════════════
# 7. NATS — pub/sub round trip
# ════════════════════════════════════════════════════════════════════
section "7. NATS Messaging"
NATS_HOST="${PI_IPS[Octavia]}"
NATS_USER="${PI_SSH_USERS[Octavia]}"
run_test "NATS: port reachable (via SSH)" \
"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes $NATS_USER@$NATS_HOST 'ss -tln | grep -q :4222' 2>/dev/null"
if command -v nats &>/dev/null; then
NATS_SUBJ="e2e.test.$(date +%s)"
run_test "NATS: pub/sub round trip" \
"nats pub '$NATS_SUBJ' 'hello from e2e' --server='nats://$NATS_HOST:4222' --timeout=5s 2>/dev/null"
else
run_test "NATS: pub/sub (nats CLI not installed)" \
"false"
fi
# ════════════════════════════════════════════════════════════════════
# 8. OLLAMA on Cecilia — models + inference (via SSH)
# ════════════════════════════════════════════════════════════════════
section "8. Ollama (Cecilia — via SSH)"
CECILIA_IP="${PI_IPS[Cecilia]}"
run_test "Ollama: API reachable (via SSH)" \
"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes blackroad@$CECILIA_IP 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:11434/ 2>/dev/null' | grep -q 200"
run_test "Ollama: list models (via SSH)" \
"count=\$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes blackroad@$CECILIA_IP 'curl -s http://localhost:11434/api/tags' 2>/dev/null | python3 -c \"import sys,json; d=json.load(sys.stdin); print(len(d.get('models',[])))\" 2>/dev/null); [[ \$count -gt 0 ]]"
OLD_TIMEOUT=$TEST_TIMEOUT
TEST_TIMEOUT=60
run_test "Ollama: quick inference (tinyllama)" \
"resp=\$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes blackroad@$CECILIA_IP 'curl -s --max-time 50 -X POST http://localhost:11434/api/generate -d \"{\\\"model\\\":\\\"tinyllama:latest\\\",\\\"prompt\\\":\\\"hi\\\",\\\"stream\\\":false}\" 2>/dev/null' | python3 -c \"import sys,json; d=json.load(sys.stdin); print(d.get('response',''))\" 2>/dev/null); [[ -n \"\$resp\" ]]"
TEST_TIMEOUT=$OLD_TIMEOUT
# ════════════════════════════════════════════════════════════════════
# 9. GIT SYNC — Gitea API (via SSH to Octavia)
# ════════════════════════════════════════════════════════════════════
section "9. Git Sync (Gitea — via SSH)"
OCTAVIA_IP="${PI_IPS[Octavia]}"
OCTAVIA_USER="${PI_SSH_USERS[Octavia]}"
run_test "Gitea: API reachable (via SSH)" \
"code=\$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes $OCTAVIA_USER@$OCTAVIA_IP 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:3100/api/v1/repos/search?limit=1' 2>/dev/null); [[ \"\$code\" =~ ^(200|401|403)$ ]]"
run_test "Gitea: repo count > 0 (via SSH)" \
"code=\$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes $OCTAVIA_USER@$OCTAVIA_IP 'curl -s http://localhost:3100/api/v1/repos/search?limit=1 -o /dev/null -w \"%{http_code}\"' 2>/dev/null); [[ \"\$code\" == \"200\" ]]"
# ════════════════════════════════════════════════════════════════════
# 10. LOCAL BUILDS — key projects type-check
# ════════════════════════════════════════════════════════════════════
section "10. Local Builds"
LOCAL_PROJECTS=(
"$HOME/blackroad-web"
"$HOME/blackroad-operator"
"$HOME/blackroad-os-kpis"
)
for proj in "${LOCAL_PROJECTS[@]}"; do
proj_name=$(basename "$proj")
if [[ -d "$proj" ]]; then
run_test "$proj_name: exists" "true"
# Only check node_modules if package.json has dependencies
if grep -q '"dependencies"' "$proj/package.json" 2>/dev/null || grep -q '"devDependencies"' "$proj/package.json" 2>/dev/null; then
run_test "$proj_name: node_modules" \
"[[ -d '$proj/node_modules' ]]"
fi
if [[ -f "$proj/tsconfig.json" ]]; then
run_test "$proj_name: tsc --noEmit" \
"cd '$proj' && npx tsc --noEmit 2>&1 | tail -5"
fi
else
run_test "$proj_name: exists" "false"
fi
done
# ════════════════════════════════════════════════════════════════════
# 11. STRIPE — auth + products
# ════════════════════════════════════════════════════════════════════
section "11. Stripe"
if command -v stripe &>/dev/null; then
run_test "Stripe CLI: auth check" \
"stripe config --list 2>/dev/null | grep -q 'test_mode'"
run_test "Stripe: list products" \
"stripe products list --limit=1 2>/dev/null | grep -q 'id'"
else
if [[ -n "$STRIPE_SECRET_KEY" ]]; then
run_test "Stripe API: auth" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 \
-u \"\$STRIPE_SECRET_KEY:\" 'https://api.stripe.com/v1/balance'); \
[[ \"\$code\" == '200' ]]"
run_test "Stripe API: list products" \
"code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 \
-u \"\$STRIPE_SECRET_KEY:\" 'https://api.stripe.com/v1/products?limit=1'); \
[[ \"\$code\" == '200' ]]"
else
run_test "Stripe: CLI or STRIPE_SECRET_KEY available" "false"
fi
fi
# ════════════════════════════════════════════════════════════════════
# 12. RCLONE — remotes + backup dir
# ════════════════════════════════════════════════════════════════════
section "12. rclone"
if command -v rclone &>/dev/null; then
run_test "rclone: installed" "true"
run_test "rclone: has remotes" \
"count=\$(rclone listremotes 2>/dev/null | wc -l); [[ \$count -gt 0 ]]"
run_test "rclone: verify backup dir" \
"rclone listremotes 2>/dev/null | head -1 | while read remote; do \
rclone lsd \"\${remote}\" --max-depth 1 2>/dev/null | head -1; \
done | grep -q '.'"
else
run_test "rclone: installed" "false"
fi
# ════════════════════════════════════════════════════════════════════
# SUMMARY
# ════════════════════════════════════════════════════════════════════
END_TIME=$(date +%s)
ELAPSED=$((END_TIME - START_TIME))
MINUTES=$((ELAPSED / 60))
SECS=$((ELAPSED % 60))
echo ""
echo -e "${PINK}${BOLD}══════════════════════════════════════════════════════════════${RESET}"
echo -e "${AMBER}${BOLD} RESULTS${RESET}"
echo -e "${PINK}${BOLD}══════════════════════════════════════════════════════════════${RESET}"
echo ""
if [[ $FAIL_COUNT -eq 0 ]]; then
echo -e " ${GREEN}${BOLD}ALL TESTS PASSED${RESET} ${GREEN}${PASS_COUNT}/${TOTAL_COUNT}${RESET}"
else
echo -e " ${AMBER}${BOLD}${PASS_COUNT}/${TOTAL_COUNT} tests passed${RESET} ${RED}(${FAIL_COUNT} failed)${RESET}"
echo ""
echo -e " ${RED}${BOLD}Failures:${RESET}"
for f in "${FAILURES[@]}"; do
echo -e " ${RED}x${RESET} $f"
done
fi
echo ""
echo -e " ${DIM}Completed in ${MINUTES}m ${SECS}s${RESET}"
echo ""
exit $FAIL_COUNT
Reference in New Issue View Git Blame Copy Permalink