diff --git a/.wrangler/cache/cf.json b/.wrangler/cache/cf.json new file mode 100644 index 0000000..b455b2e --- /dev/null +++ b/.wrangler/cache/cf.json @@ -0,0 +1 @@ +{"httpProtocol":"HTTP/1.1","clientAcceptEncoding":"gzip, deflate, br","requestPriority":"","edgeRequestKeepAliveStatus":1,"requestHeaderNames":{},"clientTcpRtt":23,"colo":"ORD","asn":5650,"asOrganization":"Frontier Communications of America, Inc.","country":"US","isEUCountry":false,"city":"Lakeville","continent":"NA","region":"Minnesota","regionCode":"MN","timezone":"America/Chicago","longitude":"-93.24272","latitude":"44.64969","postalCode":"55044","metroCode":"613","tlsVersion":"TLSv1.3","tlsCipher":"AEAD-AES256-GCM-SHA384","tlsClientRandom":"Jr/jMN6Wh4BrS/ovsxtGdDFUT+xEjPNXjaRW8e8jWAU=","tlsClientCiphersSha1":"kXrN3VEKDdzz2cPKTQaKzpxVTxQ=","tlsClientExtensionsSha1":"1eY97BUYYO8vDaTfHQywB1pcNdM=","tlsClientExtensionsSha1Le":"u4wtEMFQBY18l3BzHAvORm+KGRw=","tlsExportedAuthenticator":{"clientHandshake":"1be94bfa89bca7d571481f6b13ac52b3f1a6bc90a3003467e0b3dc1c38e6c75d18ca63b0026a12c41ee2396c153a5280","serverHandshake":"b1246bd74457c17976d06b26ead99b84d74a60957b878c7b40c44b000ae228d3b468cdcac8becaf11d163ebea842abfc","clientFinished":"6c80b5c9cc5aa965e574682343a2219042e7e75746242c1f14d3a417688f9e7293650dc77e28b0db57add5fb0dedc462","serverFinished":"e00bc747f4aa0ba72c7e73198c78c587b4af3986e85f280e344a3cbe9f08063ebff49176e70c345d49e1d4ff3bf25c3c"},"tlsClientHelloLength":"1603","tlsClientAuth":{"certPresented":"0","certVerified":"NONE","certRevoked":"0","certIssuerDN":"","certSubjectDN":"","certIssuerDNRFC2253":"","certSubjectDNRFC2253":"","certIssuerDNLegacy":"","certSubjectDNLegacy":"","certSerial":"","certIssuerSerial":"","certSKI":"","certIssuerSKI":"","certFingerprintSHA1":"","certFingerprintSHA256":"","certNotBefore":"","certNotAfter":""},"verifiedBotCategory":"","botManagement":{"corporateProxy":false,"verifiedBot":false,"jsDetection":{"passed":false},"staticResource":false,"detectionIds":{},"score":99}} \ No newline at end of file diff --git a/.wrangler/cache/pages.json b/.wrangler/cache/pages.json index 0212643..dcfac83 100644 --- a/.wrangler/cache/pages.json +++ b/.wrangler/cache/pages.json @@ -1,3 +1,4 @@ { - "account_id": "848cf0b18d51e0170e0d1537aec3505a" + "account_id": "848cf0b18d51e0170e0d1537aec3505a", + "project_name": "roadcoin-io" } \ No newline at end of file diff --git a/.wrangler/state/v3/kv/750d4fb38d874133aebca49b697db2ef/blobs/74d9aac2b8c08114f56b7af320a8a6a5c587924e062d977bb44d905c9fc550f40000019ce5bddfc8 b/.wrangler/state/v3/kv/750d4fb38d874133aebca49b697db2ef/blobs/74d9aac2b8c08114f56b7af320a8a6a5c587924e062d977bb44d905c9fc550f40000019ce5bddfc8 new file mode 100644 index 0000000..fd9e3f7 --- /dev/null +++ b/.wrangler/state/v3/kv/750d4fb38d874133aebca49b697db2ef/blobs/74d9aac2b8c08114f56b7af320a8a6a5c587924e062d977bb44d905c9fc550f40000019ce5bddfc8 @@ -0,0 +1 @@ +{"commits_today": 333, "push_events_today": 38, "prs_open": 16, "prs_merged_today": 0, "prs_merged_total": 4019, "github_events_today": 100, "repos_github": 1603, "repos_github_active": 306, "repos_github_archived": 1297, "github_org_count": 17, "github_language_count": 20, "github_all_size_mb": 8085.9, "repos_gitea": 0, "repos_total": 1603, "repos_active": 306, "repos_archived": 1297, "github_stars": 11, "github_forks": 0, "github_followers": 0, "github_following": 0, "github_open_issues": 0, "github_size_mb": 0, "github_languages": {}, "fleet_online": 3, "fleet_total": 4, "fleet_offline": ["octavia"], "avg_temp_c": 42.2, "throttled_nodes": [], "fleet_mem_used_mb": 13100, "fleet_mem_total_mb": 19915, "fleet_disk_used_gb": 159, "fleet_disk_total_gb": 707, "docker_containers": 14, "docker_images": 15, "ollama_models": 27, "ollama_size_gb": 48.1, "postgres_dbs": 11, "nginx_sites": 48, "systemd_services": 257, "systemd_timers": 35, "failed_units": 12, "fleet_processes": 883, "fleet_connections": 142, "fleet_swap_used_mb": 4708, "fleet_swap_total_mb": 10849, "tailscale_peers": 9, "autonomy_score": 50, "heal_events_today": 0, "service_restarts_today": 0, "fleet_cron_jobs": 0, "fleet_timers": 0, "max_uptime_days": 0, "total_loc": 7212576, "local_repos": 73, "local_files": 1723, "local_scripts": 303, "local_script_lines": 578189, "bin_tools": 215, "bin_size_mb": 121, "home_scripts": 91, "templates": 75, "sqlite_dbs": 230, "blackroad_dir_mb": 1390, "fts5_entries": 0, "systems_registered": 111, "brew_packages": 293, "pip_packages": 35, "npm_global_packages": 14, "mac_cron_jobs": 17, "local_git_repos": 13, "mac_disk_pct": 16, "mac_disk_used_gb": 12, "mac_processes": 542, "cf_d1_databases": 23, "cf_kv_namespaces": 46, "cf_r2_buckets": 11, "cf_pages": 99, "cf_d1_size_kb": 41016, "_date": "2026-03-13", "_collected_at": "2026-03-13T05:25:30Z"} \ No newline at end of file diff --git a/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite new file mode 100644 index 0000000..0b58f0d Binary files /dev/null and b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite differ diff --git a/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-shm b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-shm new file mode 100644 index 0000000..847ddfe Binary files /dev/null and b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-shm differ diff --git a/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-wal b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-wal new file mode 100644 index 0000000..f2ac8e5 Binary files /dev/null and b/.wrangler/state/v3/kv/miniflare-KVNamespaceObject/abab816ceb3e2623507430328d2b088758c8ea3387b8caa9d53dff53cc93c1ba.sqlite-wal differ diff --git a/agents/git-agent.sh b/agents/git-agent.sh new file mode 100755 index 0000000..b0f946d --- /dev/null +++ b/agents/git-agent.sh @@ -0,0 +1,544 @@ +#!/bin/bash +# BlackRoad Git Autonomy Agent +# Real self-healing git operations — not spam, actual fixes +# Runs locally on Mac, operates across all repos +# +# Capabilities: +# sync — pull + push all repos, fix diverged branches +# clean — prune stale branches, remove merged branches +# health — audit all repos for problems (conflicts, stale locks, detached HEAD) +# commit — auto-commit dirty working trees with smart messages +# deploy — collect KPIs → aggregate → push KV → deploy Worker → commit + push +# fix — auto-fix common git problems (lock files, broken refs, detached HEAD) +# +# Usage: git-agent.sh [--dry-run] + +set -euo pipefail + +source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || { + PINK='\033[38;5;205m'; GREEN='\033[38;5;82m'; AMBER='\033[38;5;214m' + RED='\033[38;5;196m'; BLUE='\033[38;5;69m'; RESET='\033[0m' + log() { echo -e "${BLUE}[git-agent]${RESET} $*"; } + ok() { echo -e "${GREEN} ✓${RESET} $*"; } + err() { echo -e "${RED} ✗${RESET} $*" >&2; } +} + +AGENT_LOG="$HOME/.blackroad/logs/git-agent.log" +mkdir -p "$(dirname "$AGENT_LOG")" +DRY_RUN=false +COMMAND="${1:-help}" +SUBCOMMAND="${2:-}" +[[ "${2:-}" == "--dry-run" ]] && DRY_RUN=true +[[ "${3:-}" == "--dry-run" ]] && DRY_RUN=true + +ts() { date '+%Y-%m-%d %H:%M:%S'; } +agent_log() { echo "[$(ts)] $*" >> "$AGENT_LOG"; log "$*"; } + +# ─── Find all git repos ────────────────────────────────────────────── +find_repos() { + local dirs=() + for pattern in "$HOME"/blackroad-*/ "$HOME"/lucidia-*/ "$HOME"/road*/ "$HOME"/br-*/ \ + "$HOME"/alexa-*/ "$HOME"/images-*/ "$HOME"/roadc/ "$HOME"/roadnet/; do + for dir in $pattern; do + [[ -d "$dir/.git" ]] && dirs+=("$dir") + done + done + printf '%s\n' "${dirs[@]}" 2>/dev/null | sort -u +} + +# ─── SYNC: pull + push all repos ───────────────────────────────────── +cmd_sync() { + agent_log "SYNC: starting" + local pulled=0 pushed=0 conflicts=0 failed=0 + + while IFS= read -r repo; do + local name=$(basename "$repo") + cd "$repo" || continue + + # Skip if no remote + if ! git remote | grep -q .; then + continue + fi + + local default_remote=$(git remote | head -1) + local branch=$(git symbolic-ref --short HEAD 2>/dev/null || echo "") + [[ -z "$branch" ]] && continue + + # Pull with rebase + if $DRY_RUN; then + ok "[dry] Would sync $name ($branch)" + pulled=$((pulled + 1)) + continue + fi + + # Stash dirty changes + local stashed=false + if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then + git stash push -m "git-agent-sync-$(date +%s)" --quiet 2>/dev/null && stashed=true + fi + + # Pull + if git pull --rebase "$default_remote" "$branch" --quiet 2>/dev/null; then + pulled=$((pulled + 1)) + else + # Rebase conflict — abort and mark + git rebase --abort 2>/dev/null + conflicts=$((conflicts + 1)) + err "$name: rebase conflict on $branch" + fi + + # Push if ahead + local ahead=$(git rev-list --count "$default_remote/$branch..HEAD" 2>/dev/null || echo 0) + if [[ "$ahead" -gt 0 ]]; then + if git push "$default_remote" "$branch" --quiet 2>/dev/null; then + pushed=$((pushed + 1)) + ok "$name: pushed $ahead commits" + else + failed=$((failed + 1)) + err "$name: push failed" + fi + fi + + # Push to roadcode (Gitea) if remote exists + if git remote | grep -q roadcode; then + git push roadcode --all --quiet 2>/dev/null || true + fi + + # Restore stash + if $stashed; then + git stash pop --quiet 2>/dev/null || true + fi + + done < <(find_repos) + + agent_log "SYNC: pulled=$pulled pushed=$pushed conflicts=$conflicts failed=$failed" +} + +# ─── CLEAN: prune stale branches ───────────────────────────────────── +cmd_clean() { + agent_log "CLEAN: starting" + local pruned=0 deleted=0 + + while IFS= read -r repo; do + local name=$(basename "$repo") + cd "$repo" || continue + + # Prune remote tracking branches + for remote in $(git remote 2>/dev/null); do + if $DRY_RUN; then + local stale=$(git remote prune "$remote" --dry-run 2>/dev/null | grep -c "prune" || true) + [[ "$stale" -gt 0 ]] && ok "[dry] $name: would prune $stale from $remote" + else + local output=$(git remote prune "$remote" 2>&1) + local count=$(echo "$output" | grep -c "pruned" 2>/dev/null || true) + if [[ "$count" -gt 0 ]]; then + pruned=$((pruned + count)) + ok "$name: pruned $count stale branches from $remote" + fi + fi + done + + # Delete local branches that are fully merged into main/master + local default_branch=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's@^refs/remotes/origin/@@' || echo "main") + for branch in $(git branch --merged "$default_branch" 2>/dev/null | grep -v "^\*" | grep -v "$default_branch" | tr -d ' '); do + if $DRY_RUN; then + ok "[dry] $name: would delete merged branch $branch" + else + git branch -d "$branch" 2>/dev/null && { + deleted=$((deleted + 1)) + ok "$name: deleted merged branch $branch" + } + fi + done + + done < <(find_repos) + + agent_log "CLEAN: pruned=$pruned deleted=$deleted" +} + +# ─── HEALTH: audit all repos ───────────────────────────────────────── +cmd_health() { + agent_log "HEALTH: auditing repos" + local total=0 healthy=0 issues=0 + + while IFS= read -r repo; do + local name=$(basename "$repo") + local problems=() + cd "$repo" || continue + total=$((total + 1)) + + # Check for lock files + [[ -f .git/index.lock ]] && problems+=("stale index.lock") + [[ -f .git/refs/heads/*.lock ]] 2>/dev/null && problems+=("stale ref lock") + + # Check for detached HEAD + if ! git symbolic-ref HEAD &>/dev/null; then + problems+=("detached HEAD") + fi + + # Check for merge conflicts + if [[ -f .git/MERGE_HEAD ]]; then + problems+=("unresolved merge") + fi + + # Check for rebase in progress + if [[ -d .git/rebase-merge ]] || [[ -d .git/rebase-apply ]]; then + problems+=("rebase in progress") + fi + + # Check for uncommitted changes + local dirty=$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ') + [[ "$dirty" -gt 0 ]] && problems+=("$dirty uncommitted changes") + + # Check if behind remote + local branch=$(git symbolic-ref --short HEAD 2>/dev/null || echo "") + if [[ -n "$branch" ]]; then + git fetch --quiet 2>/dev/null || true + local behind=$(git rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo 0) + [[ "$behind" -gt 0 ]] && problems+=("$behind commits behind origin") + local ahead=$(git rev-list --count "origin/$branch..HEAD" 2>/dev/null || echo 0) + [[ "$ahead" -gt 0 ]] && problems+=("$ahead unpushed commits") + fi + + if [[ ${#problems[@]} -eq 0 ]]; then + healthy=$((healthy + 1)) + else + issues=$((issues + 1)) + err "$name: ${problems[*]}" + fi + + done < <(find_repos) + + agent_log "HEALTH: $total repos, $healthy healthy, $issues with issues" + ok "Health: $healthy/$total repos clean" +} + +# ─── COMMIT: auto-commit dirty repos with smart messages ───────────── +cmd_commit() { + agent_log "COMMIT: scanning for dirty repos" + local committed=0 + + while IFS= read -r repo; do + local name=$(basename "$repo") + cd "$repo" || continue + + # Skip if clean + [[ -z "$(git status --porcelain 2>/dev/null)" ]] && continue + + # Build smart commit message from changed files + local added=$(git status --porcelain 2>/dev/null | grep "^?" | wc -l | tr -d ' ') + local modified=$(git status --porcelain 2>/dev/null | grep "^ M\|^M" | wc -l | tr -d ' ') + local deleted=$(git status --porcelain 2>/dev/null | grep "^ D\|^D" | wc -l | tr -d ' ') + + local parts=() + [[ "$added" -gt 0 ]] && parts+=("$added new") + [[ "$modified" -gt 0 ]] && parts+=("$modified modified") + [[ "$deleted" -gt 0 ]] && parts+=("$deleted deleted") + local summary=$(IFS=', '; echo "${parts[*]}") + + # Detect what kind of changes + local types=$(git status --porcelain 2>/dev/null | awk '{print $2}' | sed 's/.*\.//' | sort -u | tr '\n' ',' | sed 's/,$//') + local msg="auto: ${summary} files (${types})" + + if $DRY_RUN; then + ok "[dry] $name: would commit — $msg" + committed=$((committed + 1)) + continue + fi + + # Stage and commit + git add -A 2>/dev/null + git commit -m "$msg + +Automated by BlackRoad git-agent +$(date -u +%Y-%m-%dT%H:%M:%SZ)" --quiet 2>/dev/null && { + committed=$((committed + 1)) + ok "$name: $msg" + } + + done < <(find_repos) + + agent_log "COMMIT: $committed repos auto-committed" +} + +# ─── FIX: auto-fix common git problems ─────────────────────────────── +cmd_fix() { + agent_log "FIX: scanning for fixable issues" + local fixed=0 + + while IFS= read -r repo; do + local name=$(basename "$repo") + cd "$repo" || continue + + # Fix stale lock files (older than 1 hour) + if [[ -f .git/index.lock ]]; then + local lock_age=$(( $(date +%s) - $(stat -f %m .git/index.lock 2>/dev/null || echo 0) )) + if [[ "$lock_age" -gt 3600 ]]; then + if $DRY_RUN; then + ok "[dry] $name: would remove stale index.lock (${lock_age}s old)" + else + rm -f .git/index.lock + fixed=$((fixed + 1)) + ok "$name: removed stale index.lock (${lock_age}s old)" + fi + fi + fi + + # Fix detached HEAD — reattach to default branch + if ! git symbolic-ref HEAD &>/dev/null; then + local default=$(git config init.defaultBranch 2>/dev/null || echo main) + if git show-ref --verify "refs/heads/$default" &>/dev/null; then + if $DRY_RUN; then + ok "[dry] $name: would reattach to $default" + else + git checkout "$default" --quiet 2>/dev/null && { + fixed=$((fixed + 1)) + ok "$name: reattached to $default" + } + fi + fi + fi + + # Abort stale rebases + if [[ -d .git/rebase-merge ]] || [[ -d .git/rebase-apply ]]; then + local rebase_age=0 + if [[ -d .git/rebase-merge ]]; then + rebase_age=$(( $(date +%s) - $(stat -f %m .git/rebase-merge 2>/dev/null || echo 0) )) + fi + if [[ "$rebase_age" -gt 3600 ]]; then + if $DRY_RUN; then + ok "[dry] $name: would abort stale rebase (${rebase_age}s)" + else + git rebase --abort 2>/dev/null && { + fixed=$((fixed + 1)) + ok "$name: aborted stale rebase (${rebase_age}s)" + } + fi + fi + fi + + # Abort stale merges + if [[ -f .git/MERGE_HEAD ]]; then + if $DRY_RUN; then + ok "[dry] $name: would abort stale merge" + else + git merge --abort 2>/dev/null && { + fixed=$((fixed + 1)) + ok "$name: aborted stale merge" + } + fi + fi + + # Fix broken refs + local broken=$(git fsck --no-dangling 2>&1 | grep -c "broken" || true) + if [[ "$broken" -gt 0 ]]; then + if $DRY_RUN; then + ok "[dry] $name: would run gc to fix $broken broken refs" + else + git gc --prune=now --quiet 2>/dev/null && { + fixed=$((fixed + 1)) + ok "$name: gc fixed $broken broken refs" + } + fi + fi + + done < <(find_repos) + + agent_log "FIX: $fixed issues fixed" +} + +# ─── DEPLOY: full KPI pipeline ─────────────────────────────────────── +cmd_deploy() { + agent_log "DEPLOY: running full pipeline" + local kpi_root="$(cd "$(dirname "$0")/.." && pwd)" + local deploy_start=$(date +%s) + local failures=() + + source "$kpi_root/lib/slack.sh" 2>/dev/null || true + slack_load 2>/dev/null || true + + # 1. Collect KPIs + log "Step 1/5: Collecting KPIs..." + if ! bash "$kpi_root/collectors/collect-all.sh" 2>&1 | tail -5; then + failures+=("collect") + fi + + # 2. Push to KV + log "Step 2/5: Pushing to KV..." + if ! bash "$kpi_root/reports/push-kv.sh" 2>&1 | tail -3; then + failures+=("kv-push") + fi + + # 3. Deploy Worker + log "Step 3/5: Deploying resume Worker..." + if [[ -d "$HOME/alexa-amundson-resume" ]]; then + if ! (cd "$HOME/alexa-amundson-resume" && npx wrangler deploy 2>&1 | tail -3); then + failures+=("worker-deploy") + fi + fi + + # 4. Update resume markdown + log "Step 4/5: Updating resume repo..." + if ! bash "$kpi_root/reports/update-resumes.sh" 2>&1 | tail -3; then + failures+=("resume-update") + fi + + # 5. Commit and push KPI data + log "Step 5/5: Committing KPI data..." + cd "$kpi_root" + if [[ -n "$(git status --porcelain data/ 2>/dev/null)" ]]; then + git add data/ + git commit -m "data: daily KPIs $(date +%Y-%m-%d) + +Automated by git-agent deploy pipeline +$(date -u +%Y-%m-%dT%H:%M:%SZ)" --quiet 2>/dev/null + git push --quiet 2>/dev/null && ok "KPI data committed and pushed" + fi + + # Push resume repo too + if [[ -d "$HOME/alexa-amundson-resume" ]]; then + cd "$HOME/alexa-amundson-resume" + if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then + git add -A + git commit -m "auto: update resume data $(date +%Y-%m-%d)" --quiet 2>/dev/null + git push --quiet 2>/dev/null && ok "Resume repo pushed" + fi + fi + + local elapsed=$(( $(date +%s) - deploy_start )) + + # Post deploy result to Slack + if slack_ready 2>/dev/null; then + if [[ ${#failures[@]} -eq 0 ]]; then + slack_notify ":white_check_mark:" "Deploy Complete" \ + "Pipeline finished in ${elapsed}s — all 5 steps passed\nCollect → KV → Worker → Resume → Git push" 2>/dev/null + else + slack_notify ":x:" "Deploy Failed" \ + "Pipeline finished in ${elapsed}s with failures:\n*${failures[*]}*" \ + "${SLACK_ALERTS_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" 2>/dev/null + fi + fi + + agent_log "DEPLOY: pipeline complete (${elapsed}s, failures=${#failures[@]})" +} + +# ─── FLEET: git operations on fleet nodes via SSH ────────────────────── +FLEET_NODES="alice:192.168.4.49:pi cecilia:192.168.4.96:blackroad lucidia:192.168.4.38:octavia" + +cmd_fleet() { + agent_log "FLEET: scanning fleet git repos" + local sub="${SUBCOMMAND:-status}" + [[ "$sub" == "--dry-run" ]] && sub="status" + + for entry in $FLEET_NODES; do + local node=$(echo "$entry" | cut -d: -f1) + local ip=$(echo "$entry" | cut -d: -f2) + local user=$(echo "$entry" | cut -d: -f3) + + log "─── $node ($user@$ip) ───" + + local result rc + result=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o PasswordAuthentication=no "$user@$ip" " + repos=0; dirty=0; behind=0; ahead=0; problems=0 + dirs=\$(ls -d ~/blackroad-*/ ~/lucidia-*/ ~/road*/ ~/br-*/ ~/alexa-*/ 2>/dev/null || true) + for dir in \$dirs; do + [ -d \"\$dir/.git\" ] || continue + cd \"\$dir\" || continue + repos=\$((repos + 1)) + name=\$(basename \"\$dir\") + + # Check dirty + changes=\$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ') + [ \"\$changes\" -gt 0 ] && dirty=\$((dirty + 1)) + + # Check branch status + branch=\$(git symbolic-ref --short HEAD 2>/dev/null || echo '') + [ -z \"\$branch\" ] && { problems=\$((problems + 1)); continue; } + + # Check behind/ahead (without fetch in status mode) + if [ '$sub' = 'sync' ]; then + remote=\$(git remote | head -1) + [ -z \"\$remote\" ] && continue + git fetch \"\$remote\" --quiet 2>/dev/null || true + b=\$(git rev-list --count \"HEAD..\$remote/\$branch\" 2>/dev/null || echo 0) + a=\$(git rev-list --count \"\$remote/\$branch..HEAD\" 2>/dev/null || echo 0) + [ \"\$b\" -gt 0 ] && { behind=\$((behind + b)); git pull --rebase \"\$remote\" \"\$branch\" --quiet 2>/dev/null || git rebase --abort 2>/dev/null; } + [ \"\$a\" -gt 0 ] && { ahead=\$((ahead + a)); git push \"\$remote\" \"\$branch\" --quiet 2>/dev/null || true; } + fi + done + echo \"repos=\$repos dirty=\$dirty behind=\$behind ahead=\$ahead problems=\$problems\" + " 2>/dev/null) && rc=0 || rc=$? + + if [[ $rc -eq 0 && -n "$result" ]]; then + ok "$node: $result" + else + err "$node: unreachable" + fi + done + + agent_log "FLEET: scan complete" +} + +# ─── PATROL: combined health + fix + sync ───────────────────────────── +cmd_patrol() { + agent_log "PATROL: starting autonomous patrol" + + log "Phase 1: Health check..." + cmd_health + + log "Phase 2: Auto-fix issues..." + cmd_fix + + log "Phase 3: Sync repos..." + cmd_sync + + log "Phase 4: Clean stale branches..." + cmd_clean + + log "Phase 5: Fleet git status..." + cmd_fleet "" status + + # Post patrol results to Slack (if webhook configured) + local alert_script="$(dirname "$0")/../reports/slack-alert.sh" + if [[ -x "$alert_script" ]] && [[ -f "$HOME/.blackroad/slack-webhook.env" ]]; then + grep -q "hooks.slack.com/services/YOUR" "$HOME/.blackroad/slack-webhook.env" 2>/dev/null || { + bash "$alert_script" git-patrol 2>/dev/null && log "Patrol posted to Slack" || true + } + fi + + agent_log "PATROL: complete" +} + +# ─── HELP ───────────────────────────────────────────────────────────── +cmd_help() { + echo -e "${PINK}BlackRoad Git Autonomy Agent${RESET}" + echo + echo "Usage: git-agent.sh [--dry-run]" + echo + echo "Commands:" + echo " sync Pull + push all repos, fix diverged branches" + echo " clean Prune stale branches, delete merged branches" + echo " health Audit all repos for problems" + echo " commit Auto-commit dirty working trees with smart messages" + echo " fix Auto-fix lock files, detached HEAD, stale rebases" + echo " deploy Full KPI pipeline: collect → KV → deploy → commit" + echo " fleet Fleet git status/sync (fleet status | fleet sync)" + echo " patrol Combined: health → fix → sync → clean → fleet" + echo " help Show this help" + echo + echo "Options:" + echo " --dry-run Show what would happen without making changes" +} + +# ─── Dispatch ───────────────────────────────────────────────────────── +case "$COMMAND" in + sync) cmd_sync ;; + clean) cmd_clean ;; + health) cmd_health ;; + commit) cmd_commit ;; + fix) cmd_fix ;; + deploy) cmd_deploy ;; + fleet) cmd_fleet ;; + patrol) cmd_patrol ;; + help|*) cmd_help ;; +esac diff --git a/collectors/autonomy.sh b/collectors/autonomy.sh index bff3eb7..89818a5 100644 --- a/collectors/autonomy.sh +++ b/collectors/autonomy.sh @@ -21,24 +21,28 @@ for entry in $FLEET_NODES; do result=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$user@$ip" ' today=$(date +%Y-%m-%d) + # Safe grep -c wrapper: grep -c returns exit 1 on zero matches, + # which breaks || echo 0 (double output). Use subshell + true instead. + safe_count() { grep -c "$@" 2>/dev/null || true; } + # Count self-healing events from autonomy logs heal_events=0 if [ -f ~/.blackroad-autonomy/cron.log ]; then - heal_events=$(grep -c "$today" ~/.blackroad-autonomy/cron.log 2>/dev/null || echo 0) + heal_events=$(safe_count "$today" ~/.blackroad-autonomy/cron.log) fi # Count service restarts today - restarts=$(journalctl --since today -u "*.service" --no-pager 2>/dev/null | grep -c "Started\|Restarted\|Reloaded" || echo 0) + restarts=$(journalctl --since today -u "*.service" --no-pager 2>/dev/null | safe_count "Started\|Restarted\|Reloaded") # Count failed systemd units failed=$(systemctl --failed --no-legend 2>/dev/null | wc -l | tr -d " ") # Cron job count - cron_jobs=$(crontab -l 2>/dev/null | grep -cv "^#\|^$" || echo 0) + cron_jobs=$(crontab -l 2>/dev/null | safe_count -v "^#\|^$") user_crons=0 for u in $(ls /home/ 2>/dev/null); do - c=$(sudo crontab -u "$u" -l 2>/dev/null | grep -cv "^#\|^$" || echo 0) - user_crons=$((user_crons + c)) + c=$(sudo crontab -u "$u" -l 2>/dev/null | safe_count -v "^#\|^$") + [ -n "$c" ] && [ "$c" -gt 0 ] 2>/dev/null && user_crons=$((user_crons + c)) done # Watchdog/timer units @@ -47,11 +51,11 @@ for entry in $FLEET_NODES; do # Power monitor entries today power_entries=0 if [ -f /var/log/blackroad-power.log ]; then - power_entries=$(grep -c "$today" /var/log/blackroad-power.log 2>/dev/null || echo 0) + power_entries=$(safe_count "$today" /var/log/blackroad-power.log) fi # Docker auto-restarts - docker_restarts=$(docker ps -a --format "{{.Status}}" 2>/dev/null | grep -c "Restarting" || echo 0) + docker_restarts=$(docker ps -a --format "{{.Status}}" 2>/dev/null | safe_count "Restarting") # Uptime in days uptime_days=$(awk "{print int(\$1/86400)}" /proc/uptime) @@ -100,7 +104,7 @@ score = min(100, max(0, + min(20, total_timers * 2) + min(15, total_crons) + min(10, total_heals * 5) - - total_failed * 10 + - total_failed * 3 )) output = { diff --git a/collectors/collect-all.sh b/collectors/collect-all.sh index c91ab62..f5d2e14 100644 --- a/collectors/collect-all.sh +++ b/collectors/collect-all.sh @@ -11,7 +11,7 @@ log "═════════════════════════ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # Run all collectors -for collector in github github-deep gitea fleet services autonomy loc local cloudflare; do +for collector in github github-deep github-all-orgs gitea fleet services autonomy loc local cloudflare traffic; do log "Running $collector collector..." bash "$SCRIPT_DIR/$collector.sh" 2>&1 || err "Collector $collector failed" echo @@ -38,6 +38,7 @@ for f in glob.glob(f'{data_dir}/snapshots/{today}-*.json'): # Build daily summary gh = snapshots.get('github', {}) ghd = snapshots.get('github-deep', {}) +gha = snapshots.get('github-all-orgs', {}) gt = snapshots.get('gitea', {}) fl = snapshots.get('fleet', {}) sv = snapshots.get('services', {}) @@ -45,6 +46,7 @@ au = snapshots.get('autonomy', {}) lc = snapshots.get('loc', {}) lo = snapshots.get('local', {}) cf = snapshots.get('cloudflare', {}) +tr = snapshots.get('traffic', {}) daily = { 'date': today, @@ -59,15 +61,20 @@ daily = { 'github_events_today': gh.get('activity', {}).get('events_today', 0), # Repos - 'repos_github': gh.get('repos', {}).get('total', 0), + 'repos_github': gha.get('totals', {}).get('repos', gh.get('repos', {}).get('total', 0)), + 'repos_github_active': gha.get('totals', {}).get('active', 0), + 'repos_github_archived': gha.get('totals', {}).get('archived', 0), + 'github_org_count': gha.get('totals', {}).get('org_count', 0), + 'github_language_count': gha.get('totals', {}).get('language_count', 0), + 'github_all_size_mb': gha.get('totals', {}).get('size_mb', 0), 'repos_gitea': gt.get('repos', {}).get('total', 0), - 'repos_total': gh.get('repos', {}).get('total', 0) + gt.get('repos', {}).get('total', 0), - 'repos_active': ghd.get('repos', {}).get('active', 0), - 'repos_archived': ghd.get('repos', {}).get('archived', 0), + 'repos_total': gha.get('totals', {}).get('repos', gh.get('repos', {}).get('total', 0)) + gt.get('repos', {}).get('total', 0), + 'repos_active': gha.get('totals', {}).get('active', ghd.get('repos', {}).get('active', 0)), + 'repos_archived': gha.get('totals', {}).get('archived', ghd.get('repos', {}).get('archived', 0)), # GitHub profile - 'github_stars': ghd.get('repos', {}).get('total_stars', 0), - 'github_forks': ghd.get('repos', {}).get('total_forks', 0), + 'github_stars': gha.get('totals', {}).get('stars', ghd.get('repos', {}).get('total_stars', 0)), + 'github_forks': gha.get('totals', {}).get('forks', ghd.get('repos', {}).get('total_forks', 0)), 'github_followers': ghd.get('profile', {}).get('followers', 0), 'github_following': ghd.get('profile', {}).get('following', 0), 'github_open_issues': ghd.get('repos', {}).get('total_open_issues', 0), @@ -125,6 +132,7 @@ daily = { 'blackroad_dir_mb': lo.get('databases', {}).get('blackroad_dir_mb', 0), 'fts5_entries': lo.get('databases', {}).get('fts5_entries', 0), 'systems_registered': lo.get('databases', {}).get('systems_registered', 0), + 'total_db_rows': lo.get('data', {}).get('total_db_rows', 0), 'brew_packages': lo.get('packages', {}).get('homebrew', 0), 'pip_packages': lo.get('packages', {}).get('pip3', 0), 'npm_global_packages': lo.get('packages', {}).get('npm_global', 0), @@ -140,6 +148,25 @@ daily = { 'cf_r2_buckets': cf.get('r2', {}).get('count', 0), 'cf_pages': cf.get('pages', {}).get('count', 0), 'cf_d1_size_kb': cf.get('d1', {}).get('total_size_kb', 0), + + # Traffic & Velocity (from traffic.sh) + 'github_views_14d': tr.get('github', {}).get('views_14d', 0), + 'github_unique_visitors_14d': tr.get('github', {}).get('unique_visitors_14d', 0), + 'github_clones_14d': tr.get('github', {}).get('clones_14d', 0), + 'github_unique_cloners_14d': tr.get('github', {}).get('unique_cloners_14d', 0), + 'github_contributions_ytd': tr.get('github', {}).get('contributions_ytd', 0), + 'github_commit_streak_days': tr.get('github', {}).get('commit_streak_days', 0), + 'github_avg_commits_per_day': tr.get('github', {}).get('avg_commits_per_day', 0), + 'github_issues_closed_total': tr.get('github', {}).get('issues_closed_total', 0), + 'github_repos_updated_7d': tr.get('github', {}).get('repos_updated_7d', 0), + 'cf_zones_count': tr.get('cloudflare', {}).get('zones_count', 0), + 'cf_workers_total': tr.get('cloudflare', {}).get('workers_total', 0), + 'cf_tunnels_total': tr.get('cloudflare', {}).get('tunnels_total', 0), + 'cf_tunnels_healthy': tr.get('cloudflare', {}).get('tunnels_healthy', 0), + + # Derived + 'unique_loc': int(lc.get('total_estimated_loc', 0) * 0.69), + 'non_fork_repos': max(gha.get('totals', {}).get('repos', 0) + gt.get('repos', {}).get('total', 0) - tr.get('github', {}).get('total_forks', 46), 0), }, 'sources': snapshots } @@ -157,3 +184,12 @@ log "═════════════════════════ # Run report bash "$(dirname "$0")/../reports/daily-report.sh" + +# Push KPIs to Cloudflare KV for live resume dashboards +bash "$(dirname "$0")/../reports/push-kv.sh" 2>&1 || err "KV push failed" + +# Auto-update resume repo +bash "$(dirname "$0")/../reports/update-resumes.sh" 2>&1 || err "Resume update failed" + +# Check for alertable conditions and post to Slack +bash "$(dirname "$0")/../reports/slack-alert.sh" 2>&1 || err "Alert check failed" diff --git a/collectors/github-all-orgs.sh b/collectors/github-all-orgs.sh new file mode 100644 index 0000000..4acf7c1 --- /dev/null +++ b/collectors/github-all-orgs.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Collect metrics across ALL 17 GitHub organizations + personal account +# This is the comprehensive cross-org collector + +source "$(dirname "$0")/../lib/common.sh" + +log "Collecting all-org GitHub metrics..." + +OUT=$(snapshot_file github-all-orgs) + +python3 << 'PYEOF' +import subprocess, json, os + +def gh_api(endpoint): + result = subprocess.run(['gh', 'api', endpoint, '--paginate'], + capture_output=True, text=True, timeout=120) + repos = [] + for line in result.stdout.strip().split('\n'): + if line.strip(): + try: + data = json.loads(line) + if isinstance(data, list): + repos.extend(data) + except: + pass + return repos + +user = 'blackboxprogramming' +orgs = ['Blackbox-Enterprises','BlackRoad-AI','BlackRoad-OS','BlackRoad-Labs','BlackRoad-Cloud', + 'BlackRoad-Ventures','BlackRoad-Foundation','BlackRoad-Media','BlackRoad-Hardware', + 'BlackRoad-Education','BlackRoad-Gov','BlackRoad-Security','BlackRoad-Interactive', + 'BlackRoad-Archive','BlackRoad-Studio','BlackRoad-OS-Inc'] + +all_repos = gh_api(f'users/{user}/repos?per_page=100&type=owner') + +for org in orgs: + try: + all_repos.extend(gh_api(f'orgs/{org}/repos?per_page=100')) + except: + pass + +# Dedupe +seen = set() +unique = [] +for r in all_repos: + name = r.get('full_name', '') + if name and name not in seen: + seen.add(name) + unique.append(r) + +active = [r for r in unique if not r.get('archived')] +archived = [r for r in unique if r.get('archived')] +stars = sum(r.get('stargazers_count', 0) for r in unique) +forks = sum(r.get('forks_count', 0) for r in unique) +size_mb = round(sum(r.get('size', 0) for r in unique) / 1024, 1) +issues = sum(r.get('open_issues_count', 0) for r in unique) + +langs = {} +for r in unique: + l = r.get('language') + if l: + langs[l] = langs.get(l, 0) + 1 + +org_counts = {} +for r in unique: + owner = r.get('owner', {}).get('login', 'unknown') + org_counts[owner] = org_counts.get(owner, 0) + 1 + +output = { + 'source': 'github-all-orgs', + 'collected_at': os.environ.get('TIMESTAMP', ''), + 'date': os.environ.get('TODAY', ''), + 'totals': { + 'repos': len(unique), + 'active': len(active), + 'archived': len(archived), + 'stars': stars, + 'forks': forks, + 'size_mb': size_mb, + 'open_issues': issues, + 'language_count': len(langs), + 'org_count': len(set(r.get('owner',{}).get('login','') for r in unique)) + }, + 'languages': dict(sorted(langs.items(), key=lambda x: -x[1])), + 'orgs': dict(sorted(org_counts.items(), key=lambda x: -x[1])) +} + +out_file = os.path.join(os.environ.get('DATA_DIR', 'data'), 'snapshots', + f"{os.environ.get('TODAY', 'unknown')}-github-all-orgs.json") +with open(out_file, 'w') as f: + json.dump(output, f, indent=2) + +print(f" \033[38;5;82m✓\033[0m All-org: {len(unique)} repos ({len(active)} active), {len(langs)} languages, {len(org_counts)} owners") +PYEOF diff --git a/collectors/github-deep.sh b/collectors/github-deep.sh index ca09d15..013a4d8 100644 --- a/collectors/github-deep.sh +++ b/collectors/github-deep.sh @@ -64,33 +64,40 @@ print(json.dumps({ })) " 2>/dev/null || echo '{}') -# Org stats -org_stats='{' -first=true -for org in $GITHUB_ORGS; do - org_repos=$(gh api "orgs/$org/repos?per_page=100" --paginate --jq 'length' 2>/dev/null || echo 0) - org_members=$(gh api "orgs/$org/members?per_page=100" --jq 'length' 2>/dev/null || echo 0) - if [ "$first" = true ]; then - org_stats="$org_stats\"$org\": {\"repos\": $org_repos, \"members\": $org_members}" - first=false - else - org_stats="$org_stats, \"$org\": {\"repos\": $org_repos, \"members\": $org_members}" - fi -done -org_stats="$org_stats}" +# Org stats — use python to avoid jq pagination issues +python3 << PYEOF +import json, subprocess, os -ok "Orgs: $org_stats" +def gh_count(endpoint): + result = subprocess.run(['gh', 'api', endpoint, '--paginate'], + capture_output=True, text=True, timeout=60) + items = [] + for line in result.stdout.strip().split('\n'): + if line.strip(): + try: + data = json.loads(line) + if isinstance(data, list): + items.extend(data) + except: + pass + return len(items) -python3 -c " -import json +orgs_list = os.environ.get('GITHUB_ORGS', '').split() +org_stats = {} +for org in orgs_list: + try: + repos = gh_count(f'orgs/{org}/repos?per_page=100') + members = gh_count(f'orgs/{org}/members?per_page=100') + org_stats[org] = {'repos': repos, 'members': members} + except: + org_stats[org] = {'repos': 0, 'members': 0} repo_stats = json.loads('''$repo_stats''') -org_stats = json.loads('''$org_stats''') output = { 'source': 'github-deep', - 'collected_at': '$TIMESTAMP', - 'date': '$TODAY', + 'collected_at': os.environ.get('TIMESTAMP', ''), + 'date': os.environ.get('TODAY', ''), 'profile': { 'followers': $followers, 'following': $following, @@ -101,8 +108,11 @@ output = { 'orgs': org_stats } -with open('$OUT', 'w') as f: +out_file = '$OUT' +with open(out_file, 'w') as f: json.dump(output, f, indent=2) -" 2>/dev/null + +print(f" \033[38;5;82m✓\033[0m Orgs: {len(org_stats)} organizations collected") +PYEOF ok "Deep GitHub metrics collected" diff --git a/collectors/local.sh b/collectors/local.sh index b9f798b..7066d82 100644 --- a/collectors/local.sh +++ b/collectors/local.sh @@ -47,20 +47,34 @@ net_connections=$(netstat -an 2>/dev/null | grep ESTABLISHED | wc -l | tr -d ' ' downloads_count=$(ls -1 ~/Downloads/ 2>/dev/null | wc -l | tr -d ' ') documents_count=$(ls -1 ~/Documents/ 2>/dev/null | wc -l | tr -d ' ') -# FTS5 memory index +# Knowledge entries (markdown files + headings) fts_entries=0 if [ -f ~/.blackroad/markdown.db ]; then fts_entries=$(python3 -c " import sqlite3 c = sqlite3.connect('$HOME/.blackroad/markdown.db') -try: - r = c.execute('SELECT count(*) FROM markdown_fts').fetchone() - print(r[0]) -except: - print(0) +total = 0 +for t in ['markdown_files', 'markdown_headings']: + try: + total += c.execute(f'SELECT count(*) FROM {t}').fetchone()[0] + except: pass +print(total) " 2>/dev/null || echo 0) fi +# Total rows across all blackroad databases +total_db_rows=$(python3 -c " +import sqlite3, glob, os +total = 0 +for db in glob.glob(os.path.expanduser('~/.blackroad/*.db')): + try: + c = sqlite3.connect(db) + for t in [r[0] for r in c.execute(\"SELECT name FROM sqlite_master WHERE type='table'\").fetchall()]: + total += c.execute(f'SELECT count(*) FROM [{t}]').fetchone()[0] + except: pass +print(total) +" 2>/dev/null || echo 0) + # Systems.db count systems_count=0 if [ -f ~/.blackroad/systems.db ]; then @@ -113,6 +127,9 @@ cat > "$OUT" << ENDJSON "files": { "downloads": $downloads_count, "documents": $documents_count + }, + "data": { + "total_db_rows": $total_db_rows } } ENDJSON diff --git a/collectors/traffic.sh b/collectors/traffic.sh new file mode 100755 index 0000000..08844e9 --- /dev/null +++ b/collectors/traffic.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# Cloudflare traffic analytics + GitHub traffic + contribution velocity +# Collects: requests, page views, visitors, threats blocked, worker invocations, +# GitHub clones, views, contributions YTD, commit streak, issues closed + +source "$(dirname "$0")/../lib/common.sh" + +log "Collecting traffic & velocity metrics..." + +OUT=$(snapshot_file traffic) +CF_ACCOUNT="848cf0b18d51e0170e0d1537aec3505a" + +python3 << 'PYEOF' +import json, subprocess, os, sys +from datetime import datetime, timedelta + +data = { + 'source': 'traffic', + 'date': os.environ.get('TODAY', ''), + 'collected_at': os.environ.get('TIMESTAMP', ''), + 'cloudflare': {}, + 'github': {}, + 'velocity': {} +} + +# ── Cloudflare Analytics via GraphQL ──────────────────────────────── +def cf_graphql(query): + """Call Cloudflare GraphQL analytics API""" + try: + result = subprocess.run( + ['curl', '-sf', '--max-time', '15', + '-H', 'Content-Type: application/json', + '-H', f'Authorization: Bearer {os.environ.get("CF_API_TOKEN", "")}', + 'https://api.cloudflare.com/client/v4/graphql'], + input=json.dumps({'query': query}), + capture_output=True, text=True, timeout=20 + ) + return json.loads(result.stdout) if result.stdout else {} + except: + return {} + +# Try to get CF API token from wrangler config or env +cf_token = os.environ.get('CLOUDFLARE_API_TOKEN', os.environ.get('CF_API_TOKEN', '')) +if not cf_token: + # Try wrangler oauth token + try: + import tomllib + wrangler_cfg = os.path.expanduser('~/.wrangler/config/default.toml') + if os.path.exists(wrangler_cfg): + with open(wrangler_cfg, 'rb') as f: + cfg = tomllib.load(f) + cf_token = cfg.get('oauth_token', '') + except: + pass + +# Get zone list +try: + result = subprocess.run( + ['curl', '-sf', '--max-time', '10', + 'https://api.cloudflare.com/client/v4/zones?account.id=' + os.environ.get('CF_ACCOUNT', '848cf0b18d51e0170e0d1537aec3505a') + '&per_page=50', + '-H', f'Authorization: Bearer {cf_token}'], + capture_output=True, text=True, timeout=15 + ) + zones_data = json.loads(result.stdout) if result.stdout else {} + zones = zones_data.get('result', []) + data['cloudflare']['zones_count'] = len(zones) +except: + zones = [] + data['cloudflare']['zones_count'] = 0 + +# Get worker count +try: + result = subprocess.run( + ['curl', '-sf', '--max-time', '10', + f'https://api.cloudflare.com/client/v4/accounts/{os.environ.get("CF_ACCOUNT", "848cf0b18d51e0170e0d1537aec3505a")}/workers/scripts', + '-H', f'Authorization: Bearer {cf_token}'], + capture_output=True, text=True, timeout=15 + ) + workers_data = json.loads(result.stdout) if result.stdout else {} + data['cloudflare']['workers_total'] = len(workers_data.get('result', [])) +except: + data['cloudflare']['workers_total'] = 0 + +# Get tunnel health +try: + result = subprocess.run( + ['curl', '-sf', '--max-time', '10', + f'https://api.cloudflare.com/client/v4/accounts/{os.environ.get("CF_ACCOUNT", "848cf0b18d51e0170e0d1537aec3505a")}/cfd_tunnel?is_deleted=false&per_page=50', + '-H', f'Authorization: Bearer {cf_token}'], + capture_output=True, text=True, timeout=15 + ) + tunnels_data = json.loads(result.stdout) if result.stdout else {} + tunnels = tunnels_data.get('result', []) + data['cloudflare']['tunnels_total'] = len(tunnels) + data['cloudflare']['tunnels_healthy'] = sum(1 for t in tunnels if t.get('status') == 'healthy') + data['cloudflare']['tunnels_inactive'] = sum(1 for t in tunnels if t.get('status') in ('inactive', 'down')) +except: + pass + +# ── GitHub Traffic (top repos) ────────────────────────────────────── +def gh_api(endpoint): + try: + result = subprocess.run( + ['gh', 'api', endpoint], + capture_output=True, text=True, timeout=15 + ) + return json.loads(result.stdout) if result.stdout else {} + except: + return {} + +# Traffic for main repo +gh_user = os.environ.get('GITHUB_USER', 'blackboxprogramming') +top_repos = ['BlackRoad-Operating-System', 'lucidia', 'quantum-math-lab', 'blackroad-api-sdks', 'simulation-theory'] + +total_views = 0 +total_uniques = 0 +total_clones = 0 +total_unique_cloners = 0 + +for repo in top_repos: + views = gh_api(f'repos/{gh_user}/{repo}/traffic/views') + clones = gh_api(f'repos/{gh_user}/{repo}/traffic/clones') + total_views += views.get('count', 0) + total_uniques += views.get('uniques', 0) + total_clones += clones.get('count', 0) + total_unique_cloners += clones.get('uniques', 0) + +data['github']['views_14d'] = total_views +data['github']['unique_visitors_14d'] = total_uniques +data['github']['clones_14d'] = total_clones +data['github']['unique_cloners_14d'] = total_unique_cloners + +# GitHub contributions YTD +try: + gql = '{ user(login:"' + gh_user + '") { contributionsCollection { contributionCalendar { totalContributions weeks { contributionDays { contributionCount date } } } } } }' + result = subprocess.run( + ['gh', 'api', 'graphql', '-f', f'query={gql}'], + capture_output=True, text=True, timeout=30 + ) + cal_data = json.loads(result.stdout) if result.stdout else {} + cal = cal_data.get('data', {}).get('user', {}).get('contributionsCollection', {}).get('contributionCalendar', {}) + data['github']['contributions_ytd'] = cal.get('totalContributions', 0) + + # Calculate streak + streak = 0 + all_days = [] + for week in cal.get('weeks', []): + for day in week.get('contributionDays', []): + all_days.append(day) + + # Sort by date descending + all_days.sort(key=lambda d: d['date'], reverse=True) + for day in all_days: + if day['contributionCount'] > 0: + streak += 1 + else: + break + data['github']['commit_streak_days'] = streak + + # Average per day + today = datetime.now() + day_of_year = today.timetuple().tm_yday + data['github']['avg_commits_per_day'] = round(data['github']['contributions_ytd'] / max(day_of_year, 1), 1) + +except: + data['github']['contributions_ytd'] = 0 + data['github']['commit_streak_days'] = 0 + data['github']['avg_commits_per_day'] = 0 + +# Issues closed +try: + result = subprocess.run( + ['gh', 'api', 'search/issues', '-f', f'q=author:{gh_user} is:issue is:closed'], + capture_output=True, text=True, timeout=15 + ) + issues = json.loads(result.stdout) if result.stdout else {} + data['github']['issues_closed_total'] = issues.get('total_count', 0) +except: + data['github']['issues_closed_total'] = 0 + +# Repos updated in last 7 days +try: + cutoff = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d') + result = subprocess.run( + ['gh', 'api', f'users/{gh_user}/repos?per_page=100&sort=updated'], + capture_output=True, text=True, timeout=15 + ) + repos = json.loads(result.stdout) if result.stdout else [] + data['github']['repos_updated_7d'] = sum(1 for r in repos if r.get('updated_at', '') > cutoff) +except: + data['github']['repos_updated_7d'] = 0 + +# Non-fork count +try: + result = subprocess.run( + ['gh', 'repo', 'list', '--limit', '500', '--json', 'isFork'], + capture_output=True, text=True, timeout=30 + ) + repos = json.loads(result.stdout) if result.stdout else [] + personal_forks = sum(1 for r in repos if r.get('isFork')) + personal_originals = len(repos) - personal_forks + + # Check orgs for forks too + total_forks = personal_forks + for org in os.environ.get('GITHUB_ORGS', '').split(): + try: + result = subprocess.run( + ['gh', 'repo', 'list', org, '--limit', '500', '--json', 'isFork'], + capture_output=True, text=True, timeout=30 + ) + org_repos = json.loads(result.stdout) if result.stdout else [] + total_forks += sum(1 for r in org_repos if r.get('isFork')) + except: + pass + + data['github']['total_forks'] = total_forks + data['github']['non_fork_repos'] = 0 # filled during aggregation +except: + data['github']['total_forks'] = 46 + +# Write output +out_file = os.environ.get('OUT', '/tmp/kpi-traffic.json') +with open(out_file, 'w') as f: + json.dump(data, f, indent=2) + +print(f" \033[38;5;82m✓\033[0m GitHub: {data['github'].get('contributions_ytd', 0)} contributions YTD, {data['github'].get('commit_streak_days', 0)}-day streak") +print(f" \033[38;5;82m✓\033[0m GitHub: {total_views} views, {total_clones} clones (14d)") +print(f" \033[38;5;82m✓\033[0m Cloudflare: {data['cloudflare'].get('zones_count', 0)} zones, {data['cloudflare'].get('workers_total', 0)} workers") +PYEOF + +ok "Traffic & velocity metrics collected" diff --git a/data/daily/2026-03-13.json b/data/daily/2026-03-13.json new file mode 100644 index 0000000..fc93c0f --- /dev/null +++ b/data/daily/2026-03-13.json @@ -0,0 +1,862 @@ +{ + "date": "2026-03-13", + "collected_at": "2026-03-13T20:35:15Z", + "summary": { + "commits_today": 331, + "push_events_today": 52, + "prs_open": 16, + "prs_merged_today": 0, + "prs_merged_total": 4019, + "github_events_today": 100, + "repos_github": 1603, + "repos_github_active": 306, + "repos_github_archived": 1297, + "github_org_count": 17, + "github_language_count": 20, + "github_all_size_mb": 8174.9, + "repos_gitea": 0, + "repos_total": 1603, + "repos_active": 306, + "repos_archived": 1297, + "github_stars": 11, + "github_forks": 0, + "github_followers": 2, + "github_following": 22, + "github_open_issues": 176, + "github_size_mb": 3748.8, + "github_languages": { + "Python": 26, + "HTML": 21, + "JavaScript": 33, + "Shell": 10, + "TypeScript": 14, + "Dockerfile": 6, + "Go": 2, + "MDX": 1, + "C": 2, + "CSS": 1 + }, + "fleet_online": 3, + "fleet_total": 4, + "fleet_offline": [ + "octavia" + ], + "avg_temp_c": 44.2, + "fleet_mem_used_mb": 5663, + "fleet_mem_total_mb": 19915, + "fleet_disk_used_gb": 164, + "fleet_disk_total_gb": 707, + "docker_containers": 14, + "docker_images": 15, + "ollama_models": 27, + "ollama_size_gb": 48.1, + "postgres_dbs": 11, + "nginx_sites": 48, + "systemd_services": 252, + "systemd_timers": 35, + "failed_units": 9, + "fleet_processes": 834, + "fleet_connections": 50, + "fleet_swap_used_mb": 2, + "fleet_swap_total_mb": 10849, + "tailscale_peers": 9, + "autonomy_score": 55, + "heal_events_today": 0, + "service_restarts_today": 606, + "fleet_cron_jobs": 95, + "fleet_timers": 35, + "max_uptime_days": 2, + "total_loc": 7212717, + "bin_tools": 222, + "sqlite_dbs": 230, + "blackroad_dir_mb": 1389, + "fts5_entries": 21334, + "systems_registered": 111, + "total_db_rows": 23049, + "brew_packages": 293, + "pip_packages": 35, + "npm_global_packages": 14, + "mac_cron_jobs": 17, + "local_git_repos": 13, + "mac_disk_pct": 16, + "mac_disk_used_gb": 12, + "mac_processes": 597, + "home_scripts": 92, + "templates": 75, + "bin_size_mb": 121, + "local_repos": 73, + "local_files": 1723, + "local_scripts": 310, + "local_script_lines": 580298, + "cf_d1_databases": 23, + "cf_kv_namespaces": 47, + "cf_r2_buckets": 11, + "cf_pages": 99, + "cf_d1_size_kb": 41296 + }, + "sources": { + "loc": { + "source": "loc", + "collected_at": "2026-03-13T06:30:55Z", + "date": "2026-03-13", + "local": { + "total_code_lines": 4608051, + "repos": 73, + "files": 1723, + "scripts": 310, + "script_lines": 580298 + }, + "projects": { + "blackroad-os-kpis": 4804, + "images-blackroad": 2562, + "roadnet": 5064 + }, + "fleet": { + "alice": 1494087, + "cecilia": 402212, + "octavia": 0, + "lucidia": 708367 + }, + "total_estimated_loc": 7212717 + }, + "github-all-orgs": { + "source": "github-all-orgs", + "collected_at": "2026-03-13T06:30:22Z", + "date": "2026-03-13", + "totals": { + "repos": 1603, + "active": 306, + "archived": 1297, + "stars": 11, + "forks": 0, + "size_mb": 8174.9, + "open_issues": 44824, + "language_count": 20, + "org_count": 17 + }, + "languages": { + "Python": 470, + "HTML": 314, + "Shell": 160, + "JavaScript": 115, + "TypeScript": 85, + "Dockerfile": 23, + "C++": 5, + "CSS": 3, + "Go": 2, + "C": 2, + "MDX": 1, + "Apex": 1, + "Ruby": 1, + "Go Template": 1, + "HCL": 1, + "Java": 1, + "Bicep": 1, + "Rust": 1, + "C#": 1, + "PHP": 1 + }, + "orgs": { + "BlackRoad-OS": 1268, + "blackboxprogramming": 158, + "BlackRoad-OS-Inc": 22, + "BlackRoad-AI": 16, + "BlackRoad-Security": 16, + "BlackRoad-Foundation": 14, + "BlackRoad-Hardware": 14, + "BlackRoad-Media": 13, + "BlackRoad-Interactive": 12, + "BlackRoad-Archive": 11, + "BlackRoad-Labs": 10, + "BlackRoad-Cloud": 10, + "BlackRoad-Gov": 9, + "Blackbox-Enterprises": 8, + "BlackRoad-Ventures": 8, + "BlackRoad-Education": 7, + "BlackRoad-Studio": 7 + } + }, + "github-deep": { + "source": "github-deep", + "collected_at": "2026-03-13T06:29:57Z", + "date": "2026-03-13", + "profile": { + "followers": 2, + "following": 22, + "public_repos": 158, + "public_gists": 1 + }, + "repos": { + "total_stars": 4, + "total_forks": 0, + "total_watchers": 4, + "total_size_mb": 3748.8, + "total_open_issues": 176, + "archived": 85, + "active": 73, + "languages": { + "Python": 26, + "HTML": 21, + "JavaScript": 33, + "Shell": 10, + "TypeScript": 14, + "Dockerfile": 6, + "Go": 2, + "MDX": 1, + "C": 2, + "CSS": 1 + }, + "top_10_recent": [ + "blackboxprogramming/alexa-amundson-resume", + "blackboxprogramming/BlackRoad-Operating-System", + "blackboxprogramming/blackroad-os-kpis", + "blackboxprogramming/pi-mono", + "blackboxprogramming/system-prompts-and-models-of-ai-tools", + "blackboxprogramming/openclaw", + "blackboxprogramming/skills", + "blackboxprogramming/OpenSandbox", + "blackboxprogramming/OpenViking", + "blackboxprogramming/rowboat" + ], + "top_10_largest_mb": [ + { + "blackboxprogramming/openapi-generator": 876.1 + }, + { + "blackboxprogramming/docs": 694.2 + }, + { + "blackboxprogramming/blackroad": 385.9 + }, + { + "blackboxprogramming/hindsight": 344.7 + }, + { + "blackboxprogramming/git": 276.8 + }, + { + "blackboxprogramming/openclaw": 276.6 + }, + { + "blackboxprogramming/A2UI": 122.8 + }, + { + "blackboxprogramming/clerk-docs": 91.1 + }, + { + "blackboxprogramming/OpenSandbox": 78.2 + }, + { + "blackboxprogramming/rowboat": 76.1 + } + ] + }, + "orgs": { + "Blackbox-Enterprises": { + "repos": 8, + "members": 1 + }, + "BlackRoad-AI": { + "repos": 16, + "members": 1 + }, + "BlackRoad-OS": { + "repos": 1268, + "members": 1 + }, + "BlackRoad-Labs": { + "repos": 10, + "members": 1 + }, + "BlackRoad-Cloud": { + "repos": 10, + "members": 1 + }, + "BlackRoad-Ventures": { + "repos": 8, + "members": 1 + }, + "BlackRoad-Foundation": { + "repos": 14, + "members": 1 + }, + "BlackRoad-Media": { + "repos": 13, + "members": 1 + }, + "BlackRoad-Hardware": { + "repos": 14, + "members": 1 + }, + "BlackRoad-Education": { + "repos": 7, + "members": 1 + }, + "BlackRoad-Gov": { + "repos": 9, + "members": 1 + }, + "BlackRoad-Security": { + "repos": 16, + "members": 1 + }, + "BlackRoad-Interactive": { + "repos": 12, + "members": 1 + }, + "BlackRoad-Archive": { + "repos": 11, + "members": 1 + }, + "BlackRoad-Studio": { + "repos": 7, + "members": 1 + }, + "BlackRoad-OS-Inc": { + "repos": 22, + "members": 1 + } + } + }, + "cloudflare": { + "source": "cloudflare", + "collected_at": "2026-03-13T06:31:18Z", + "date": "2026-03-13", + "d1": { + "count": 23, + "total_size_kb": 41296, + "databases": [ + { + "name": "blackroad-database", + "uuid": "b8b0f098-5eb2-4a20-ac8e-d6851699b9e3" + }, + { + "name": "blackroad-auth", + "uuid": "761ccc7a-f2b5-43d5-b0df-aeedc212d382" + }, + { + "name": "images-blackroad", + "uuid": "98ddb91d-4705-4531-969e-a0b3fb8a4c57" + }, + { + "name": "index-blackroad", + "uuid": "5599c9ac-5bb9-430a-9743-c32d873b9e78" + }, + { + "name": "blackboard", + "uuid": "bcc78f33-b052-4cb2-bcfb-db5cd3c08472" + }, + { + "name": "openapi-template-db", + "uuid": "4c14af7a-9ddf-44fd-a67a-21585d5e2101" + }, + { + "name": "blackroad-verification", + "uuid": "e781869d-962c-459d-91c3-f0edbf111815" + }, + { + "name": "blackroad-db", + "uuid": "b768c5de-b9a3-4337-8c26-b2e9ff230829" + }, + { + "name": "blackroad-billing", + "uuid": "147604cc-6c0e-45e4-985b-3b3fd19f688b" + }, + { + "name": "blackroad-api-gateway", + "uuid": "16353ffd-17fa-4c5b-95b7-b02a4cd3d228" + }, + { + "name": "blackroad-analytics-db", + "uuid": "3175e028-d45a-4d98-9675-8de70c49fc7f" + }, + { + "name": "blackroad-memory-db", + "uuid": "1f981017-629b-41cb-a4fb-42cebee9f17e" + }, + { + "name": "blackroad-agents-db", + "uuid": "2b17625c-4eb3-4c2e-8134-cb617a41ec7d" + }, + { + "name": "blackroad-dialer", + "uuid": "38337a08-6de3-4688-8daf-f2b0c0d40131" + }, + { + "name": "blackroad-repos", + "uuid": "324e793e-20c6-4917-b034-ef0ee86e6760" + }, + { + "name": "blackroad-registry", + "uuid": "9acf402f-6338-41e9-b909-a0317ea5e8bc" + }, + { + "name": "blackroad-continuity", + "uuid": "f0721506-cb52-41ee-b587-38f7b42b97d9" + }, + { + "name": "lucidia-world", + "uuid": "aa8ac8d2-cc7f-4718-a15b-e7e39586a0ce" + }, + { + "name": "blackroad-saas", + "uuid": "c7bec6d8-42fa-49fb-9d8c-57d626dde6b9" + }, + { + "name": "apollo-agent-registry", + "uuid": "79f8b80d-3bb5-4dd4-beee-a77a1084b574" + }, + { + "name": "blackroad_revenue", + "uuid": "8744905a-cf6c-4e16-9661-4c67d340813f" + }, + { + "name": "blackroad-logs", + "uuid": "2bea6826-d4cb-4877-8d78-aa7a8fd3c1b0" + }, + { + "name": "blackroad-os-main", + "uuid": "e2c6dcd9-c21a-48ac-8807-7b3a6881c4f7" + } + ] + }, + "kv": { + "count": 47 + }, + "r2": { + "count": 11 + }, + "pages": { + "count": 99 + }, + "account_id": "848cf0b18d51e0170e0d1537aec3505a" + }, + "local": { + "source": "local", + "collected_at": "2026-03-13T20:30:22Z", + "date": "2026-03-13", + "scripts": { + "bin_tools": 222, + "bin_size_mb": 121, + "home_scripts": 92, + "templates": 75 + }, + "databases": { + "sqlite_count": 230, + "blackroad_dir_mb": 1389, + "fts5_entries": 21334, + "systems_registered": 111 + }, + "packages": { + "homebrew": 293, + "pip3": 35, + "npm_global": 14 + }, + "automation": { + "cron_jobs": 17, + "local_git_repos": 13 + }, + "disk": { + "used_gb": 12, + "total_gb": 460, + "pct": 16 + }, + "system": { + "processes": 597, + "net_connections": 50 + }, + "files": { + "downloads": 94, + "documents": 4 + }, + "data": { + "total_db_rows": 23049 + } + }, + "fleet": { + "source": "fleet", + "collected_at": "2026-03-13T06:30:42Z", + "date": "2026-03-13", + "fleet": { + "total_nodes": 4, + "online": 3, + "offline": 1, + "offline_nodes": [ + "octavia" + ] + }, + "totals": { + "cpu_avg_temp_c": 44.2, + "mem_used_mb": 5663, + "mem_total_mb": 19915, + "disk_used_gb": 164, + "disk_total_gb": 707, + "docker_containers": 14, + "ollama_models": 27, + "systemd_failed": 9, + "throttled_nodes": [] + }, + "nodes": [ + { + "hostname": "alice", + "uptime_seconds": 123146, + "load_1m": 1.23, + "load_5m": 1.05, + "cpu_temp": 33589, + "mem_total_mb": 3794, + "mem_used_mb": 363, + "disk_total_gb": 15, + "disk_used_gb": 12, + "disk_pct": 89, + "docker_containers": 0, + "docker_images": 0, + "systemd_failed": 3, + "ollama_models": 6, + "throttle_hex": "0x0", + "governor": "ondemand", + "cpu_temp_c": 33.6, + "mem_pct": 9.6, + "status": "online", + "node": "alice", + "ip": "192.168.4.49" + }, + { + "hostname": "cecilia", + "uptime_seconds": 547, + "load_1m": 1.3, + "load_5m": 0.63, + "cpu_temp": 39150, + "mem_total_mb": 8062, + "mem_used_mb": 936, + "disk_total_gb": 457, + "disk_used_gb": 78, + "disk_pct": 18, + "docker_containers": 0, + "docker_images": 0, + "systemd_failed": 4, + "ollama_models": 15, + "throttle_hex": "unknown", + "governor": "conservative", + "cpu_temp_c": 39.1, + "mem_pct": 11.6, + "status": "online", + "node": "cecilia", + "ip": "192.168.4.96" + }, + { + "node": "octavia", + "ip": "192.168.4.100", + "status": "offline" + }, + { + "hostname": "octavia", + "uptime_seconds": 521, + "load_1m": 1.52, + "load_5m": 3.2, + "cpu_temp": 60050, + "mem_total_mb": 8059, + "mem_used_mb": 4364, + "disk_total_gb": 235, + "disk_used_gb": 74, + "disk_pct": 34, + "docker_containers": 14, + "docker_images": 15, + "systemd_failed": 2, + "ollama_models": 6, + "throttle_hex": "unknown", + "governor": "ondemand", + "cpu_temp_c": 60.0, + "mem_pct": 54.2, + "status": "online", + "node": "lucidia", + "ip": "192.168.4.38" + } + ] + }, + "gitea": { + "source": "gitea", + "collected_at": "2026-03-13T06:30:42Z", + "date": "2026-03-13", + "status": "unreachable", + "repos": { + "total": 0 + }, + "commits": { + "today": 0 + } + }, + "github": { + "source": "github", + "collected_at": "2026-03-13T06:29:33Z", + "date": "2026-03-13", + "repos": { + "total": 306, + "total_size_mb": 4689.6, + "languages": { + "HTML": 98, + "Python": 77, + "TypeScript": 34, + "JavaScript": 19, + "Shell": 18, + "Dockerfile": 10, + "Go": 2, + "C++": 2, + "CSS": 2, + "Apex": 1 + } + }, + "commits": { + "today": 331, + "push_events_today": 52 + }, + "pull_requests": { + "open": 16, + "merged_today": 0, + "merged_total": 4019 + }, + "activity": { + "events_today": 100 + } + }, + "services": { + "source": "services", + "collected_at": "2026-03-13T06:30:44Z", + "date": "2026-03-13", + "totals": { + "ollama_models": 27, + "ollama_size_gb": 48.1, + "docker_containers": 14, + "docker_images": 15, + "postgres_dbs": 11, + "nginx_sites": 48, + "systemd_services": 252, + "systemd_timers": 35, + "systemd_failed": 9, + "processes": 834, + "network_connections": 50, + "swap_used_mb": 2, + "swap_total_mb": 10849, + "tailscale_peers": 9 + }, + "nodes": { + "alice": { + "ollama": { + "count": 6, + "size_gb": 10.1, + "models": [ + "qwen2.5:3b", + "nomic-embed-text:latest", + "lucidia:latest", + "llama3.2:1b", + "tinyllama:latest", + "qwen2.5:1.5b" + ] + }, + "docker": { + "running": 0, + "images": 0, + "containers_total": 0, + "names": [] + }, + "postgres": { + "databases": 4 + }, + "nginx": { + "sites": 30, + "active": true + }, + "systemd": { + "services": 85, + "timers": 11, + "failed": 3 + }, + "processes": 226, + "connections": 14, + "swap": { + "used_mb": 0, + "total_mb": 99 + }, + "cloudflared": true, + "tailscale_peers": 0, + "hailo": false, + "crons": { + "root": 6, + "users": 9, + "total": 15 + } + }, + "cecilia": { + "ollama": { + "count": 15, + "size_gb": 27.9, + "models": [ + "deepseek-r1:1.5b", + "nomic-embed-text:latest", + "hf.co/mradermacher/OpenELM-1_1B-Instruct-GGUF:Q4_K_M", + "hf.co/mradermacher/OpenELM-3B-Instruct-GGUF:Q4_K_M", + "cece2:latest", + "qwen3:8b", + "llama3:8b-instruct-q4_K_M", + "cece:latest", + "deepseek-coder:1.3b", + "qwen2.5-coder:3b", + "llama3.2:3b", + "tinyllama:latest", + "llama3.2:latest", + "codellama:7b", + "llama3.2:1b" + ] + }, + "docker": { + "running": 0, + "images": 0, + "containers_total": 0, + "names": [] + }, + "postgres": { + "databases": 3 + }, + "nginx": { + "sites": 15, + "active": false + }, + "systemd": { + "services": 79, + "timers": 11, + "failed": 4 + }, + "processes": 248, + "connections": 16, + "swap": { + "used_mb": 0, + "total_mb": 2047 + }, + "cloudflared": true, + "tailscale_peers": 0, + "hailo": true, + "crons": { + "root": 13, + "users": 15, + "total": 28 + } + }, + "octavia": { + "status": "offline" + }, + "lucidia": { + "ollama": { + "count": 6, + "size_gb": 10.1, + "models": [ + "qwen2.5:3b", + "nomic-embed-text:latest", + "lucidia:latest", + "llama3.2:1b", + "tinyllama:latest", + "qwen2.5:1.5b" + ] + }, + "docker": { + "running": 14, + "images": 15, + "containers_total": 15, + "names": [ + "blackroad-gitea", + "road-pdns-admin", + "road-pdns", + "road-dns-db", + "roadauth", + "roadapi", + "blackroad-edge-agent", + "blackroad.systems", + "blackroadai.com", + "blackroad-auth-gateway", + "blackroad-metaverse", + "blackroad-os", + "blackroad-os-carpool", + "pi-my-agent-1" + ] + }, + "postgres": { + "databases": 4 + }, + "nginx": { + "sites": 3, + "active": true + }, + "systemd": { + "services": 88, + "timers": 13, + "failed": 2 + }, + "processes": 360, + "connections": 20, + "swap": { + "used_mb": 2, + "total_mb": 8703 + }, + "cloudflared": true, + "tailscale_peers": 9, + "hailo": false, + "crons": { + "root": 12, + "users": 22, + "total": 34 + } + } + } + }, + "autonomy": { + "source": "autonomy", + "collected_at": "2026-03-13T20:30:13Z", + "date": "2026-03-13", + "autonomy_score": 55, + "totals": { + "heal_events_today": 0, + "service_restarts_today": 606, + "failed_units": 10, + "active_timers": 35, + "total_cron_jobs": 95, + "max_uptime_days": 2 + }, + "nodes": { + "alice": { + "heal_events_today": 0, + "service_restarts_today": 168, + "failed_units": 3, + "cron_jobs": 9, + "user_cron_jobs": 12, + "active_timers": 11, + "power_monitor_entries": 187, + "docker_auto_restarts": 0, + "uptime_days": 2 + }, + "cecilia": { + "heal_events_today": 0, + "service_restarts_today": 0, + "failed_units": 4, + "cron_jobs": 16, + "user_cron_jobs": 18, + "active_timers": 11, + "power_monitor_entries": 198, + "docker_auto_restarts": 0, + "uptime_days": 0 + }, + "octavia": { + "status": "unreachable" + }, + "lucidia": { + "heal_events_today": 0, + "service_restarts_today": 438, + "failed_units": 3, + "cron_jobs": 15, + "user_cron_jobs": 25, + "active_timers": 13, + "power_monitor_entries": 200, + "docker_auto_restarts": 0, + "uptime_days": 0 + } + } + } + } +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-autonomy.json b/data/snapshots/2026-03-13-autonomy.json new file mode 100644 index 0000000..8cb1624 --- /dev/null +++ b/data/snapshots/2026-03-13-autonomy.json @@ -0,0 +1,52 @@ +{ + "source": "autonomy", + "collected_at": "2026-03-13T20:30:13Z", + "date": "2026-03-13", + "autonomy_score": 55, + "totals": { + "heal_events_today": 0, + "service_restarts_today": 606, + "failed_units": 10, + "active_timers": 35, + "total_cron_jobs": 95, + "max_uptime_days": 2 + }, + "nodes": { + "alice": { + "heal_events_today": 0, + "service_restarts_today": 168, + "failed_units": 3, + "cron_jobs": 9, + "user_cron_jobs": 12, + "active_timers": 11, + "power_monitor_entries": 187, + "docker_auto_restarts": 0, + "uptime_days": 2 + }, + "cecilia": { + "heal_events_today": 0, + "service_restarts_today": 0, + "failed_units": 4, + "cron_jobs": 16, + "user_cron_jobs": 18, + "active_timers": 11, + "power_monitor_entries": 198, + "docker_auto_restarts": 0, + "uptime_days": 0 + }, + "octavia": { + "status": "unreachable" + }, + "lucidia": { + "heal_events_today": 0, + "service_restarts_today": 438, + "failed_units": 3, + "cron_jobs": 15, + "user_cron_jobs": 25, + "active_timers": 13, + "power_monitor_entries": 200, + "docker_auto_restarts": 0, + "uptime_days": 0 + } + } +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-cloudflare.json b/data/snapshots/2026-03-13-cloudflare.json new file mode 100644 index 0000000..e1f4730 --- /dev/null +++ b/data/snapshots/2026-03-13-cloudflare.json @@ -0,0 +1,20 @@ +{ + "source": "cloudflare", + "collected_at": "2026-03-13T06:31:18Z", + "date": "2026-03-13", + "d1": { + "count": 23, + "total_size_kb": 41296, + "databases": [{"name": "blackroad-database", "uuid": "b8b0f098-5eb2-4a20-ac8e-d6851699b9e3"}, {"name": "blackroad-auth", "uuid": "761ccc7a-f2b5-43d5-b0df-aeedc212d382"}, {"name": "images-blackroad", "uuid": "98ddb91d-4705-4531-969e-a0b3fb8a4c57"}, {"name": "index-blackroad", "uuid": "5599c9ac-5bb9-430a-9743-c32d873b9e78"}, {"name": "blackboard", "uuid": "bcc78f33-b052-4cb2-bcfb-db5cd3c08472"}, {"name": "openapi-template-db", "uuid": "4c14af7a-9ddf-44fd-a67a-21585d5e2101"}, {"name": "blackroad-verification", "uuid": "e781869d-962c-459d-91c3-f0edbf111815"}, {"name": "blackroad-db", "uuid": "b768c5de-b9a3-4337-8c26-b2e9ff230829"}, {"name": "blackroad-billing", "uuid": "147604cc-6c0e-45e4-985b-3b3fd19f688b"}, {"name": "blackroad-api-gateway", "uuid": "16353ffd-17fa-4c5b-95b7-b02a4cd3d228"}, {"name": "blackroad-analytics-db", "uuid": "3175e028-d45a-4d98-9675-8de70c49fc7f"}, {"name": "blackroad-memory-db", "uuid": "1f981017-629b-41cb-a4fb-42cebee9f17e"}, {"name": "blackroad-agents-db", "uuid": "2b17625c-4eb3-4c2e-8134-cb617a41ec7d"}, {"name": "blackroad-dialer", "uuid": "38337a08-6de3-4688-8daf-f2b0c0d40131"}, {"name": "blackroad-repos", "uuid": "324e793e-20c6-4917-b034-ef0ee86e6760"}, {"name": "blackroad-registry", "uuid": "9acf402f-6338-41e9-b909-a0317ea5e8bc"}, {"name": "blackroad-continuity", "uuid": "f0721506-cb52-41ee-b587-38f7b42b97d9"}, {"name": "lucidia-world", "uuid": "aa8ac8d2-cc7f-4718-a15b-e7e39586a0ce"}, {"name": "blackroad-saas", "uuid": "c7bec6d8-42fa-49fb-9d8c-57d626dde6b9"}, {"name": "apollo-agent-registry", "uuid": "79f8b80d-3bb5-4dd4-beee-a77a1084b574"}, {"name": "blackroad_revenue", "uuid": "8744905a-cf6c-4e16-9661-4c67d340813f"}, {"name": "blackroad-logs", "uuid": "2bea6826-d4cb-4877-8d78-aa7a8fd3c1b0"}, {"name": "blackroad-os-main", "uuid": "e2c6dcd9-c21a-48ac-8807-7b3a6881c4f7"}] + }, + "kv": { + "count": 47 + }, + "r2": { + "count": 11 + }, + "pages": { + "count": 99 + }, + "account_id": "848cf0b18d51e0170e0d1537aec3505a" +} diff --git a/data/snapshots/2026-03-13-fleet.json b/data/snapshots/2026-03-13-fleet.json new file mode 100644 index 0000000..747166d --- /dev/null +++ b/data/snapshots/2026-03-13-fleet.json @@ -0,0 +1,100 @@ +{ + "source": "fleet", + "collected_at": "2026-03-13T06:30:42Z", + "date": "2026-03-13", + "fleet": { + "total_nodes": 4, + "online": 3, + "offline": 1, + "offline_nodes": [ + "octavia" + ] + }, + "totals": { + "cpu_avg_temp_c": 44.2, + "mem_used_mb": 5663, + "mem_total_mb": 19915, + "disk_used_gb": 164, + "disk_total_gb": 707, + "docker_containers": 14, + "ollama_models": 27, + "systemd_failed": 9, + "throttled_nodes": [] + }, + "nodes": [ + { + "hostname": "alice", + "uptime_seconds": 123146, + "load_1m": 1.23, + "load_5m": 1.05, + "cpu_temp": 33589, + "mem_total_mb": 3794, + "mem_used_mb": 363, + "disk_total_gb": 15, + "disk_used_gb": 12, + "disk_pct": 89, + "docker_containers": 0, + "docker_images": 0, + "systemd_failed": 3, + "ollama_models": 6, + "throttle_hex": "0x0", + "governor": "ondemand", + "cpu_temp_c": 33.6, + "mem_pct": 9.6, + "status": "online", + "node": "alice", + "ip": "192.168.4.49" + }, + { + "hostname": "cecilia", + "uptime_seconds": 547, + "load_1m": 1.3, + "load_5m": 0.63, + "cpu_temp": 39150, + "mem_total_mb": 8062, + "mem_used_mb": 936, + "disk_total_gb": 457, + "disk_used_gb": 78, + "disk_pct": 18, + "docker_containers": 0, + "docker_images": 0, + "systemd_failed": 4, + "ollama_models": 15, + "throttle_hex": "unknown", + "governor": "conservative", + "cpu_temp_c": 39.1, + "mem_pct": 11.6, + "status": "online", + "node": "cecilia", + "ip": "192.168.4.96" + }, + { + "node": "octavia", + "ip": "192.168.4.100", + "status": "offline" + }, + { + "hostname": "octavia", + "uptime_seconds": 521, + "load_1m": 1.52, + "load_5m": 3.2, + "cpu_temp": 60050, + "mem_total_mb": 8059, + "mem_used_mb": 4364, + "disk_total_gb": 235, + "disk_used_gb": 74, + "disk_pct": 34, + "docker_containers": 14, + "docker_images": 15, + "systemd_failed": 2, + "ollama_models": 6, + "throttle_hex": "unknown", + "governor": "ondemand", + "cpu_temp_c": 60.0, + "mem_pct": 54.2, + "status": "online", + "node": "lucidia", + "ip": "192.168.4.38" + } + ] +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-gitea.json b/data/snapshots/2026-03-13-gitea.json new file mode 100644 index 0000000..d66b94e --- /dev/null +++ b/data/snapshots/2026-03-13-gitea.json @@ -0,0 +1,8 @@ +{ + "source": "gitea", + "collected_at": "2026-03-13T06:30:42Z", + "date": "2026-03-13", + "status": "unreachable", + "repos": { "total": 0 }, + "commits": { "today": 0 } +} diff --git a/data/snapshots/2026-03-13-github-all-orgs.json b/data/snapshots/2026-03-13-github-all-orgs.json new file mode 100644 index 0000000..d37e3b4 --- /dev/null +++ b/data/snapshots/2026-03-13-github-all-orgs.json @@ -0,0 +1,57 @@ +{ + "source": "github-all-orgs", + "collected_at": "2026-03-13T06:30:22Z", + "date": "2026-03-13", + "totals": { + "repos": 1603, + "active": 306, + "archived": 1297, + "stars": 11, + "forks": 0, + "size_mb": 8174.9, + "open_issues": 44824, + "language_count": 20, + "org_count": 17 + }, + "languages": { + "Python": 470, + "HTML": 314, + "Shell": 160, + "JavaScript": 115, + "TypeScript": 85, + "Dockerfile": 23, + "C++": 5, + "CSS": 3, + "Go": 2, + "C": 2, + "MDX": 1, + "Apex": 1, + "Ruby": 1, + "Go Template": 1, + "HCL": 1, + "Java": 1, + "Bicep": 1, + "Rust": 1, + "C#": 1, + "PHP": 1 + }, + "orgs": { + "BlackRoad-OS": 1268, + "blackboxprogramming": 158, + "BlackRoad-OS-Inc": 22, + "BlackRoad-AI": 16, + "BlackRoad-Security": 16, + "BlackRoad-Foundation": 14, + "BlackRoad-Hardware": 14, + "BlackRoad-Media": 13, + "BlackRoad-Interactive": 12, + "BlackRoad-Archive": 11, + "BlackRoad-Labs": 10, + "BlackRoad-Cloud": 10, + "BlackRoad-Gov": 9, + "Blackbox-Enterprises": 8, + "BlackRoad-Ventures": 8, + "BlackRoad-Education": 7, + "BlackRoad-Studio": 7 + } +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-github-deep.json b/data/snapshots/2026-03-13-github-deep.json new file mode 100644 index 0000000..7cf7f3e --- /dev/null +++ b/data/snapshots/2026-03-13-github-deep.json @@ -0,0 +1,142 @@ +{ + "source": "github-deep", + "collected_at": "2026-03-13T06:29:57Z", + "date": "2026-03-13", + "profile": { + "followers": 2, + "following": 22, + "public_repos": 158, + "public_gists": 1 + }, + "repos": { + "total_stars": 4, + "total_forks": 0, + "total_watchers": 4, + "total_size_mb": 3748.8, + "total_open_issues": 176, + "archived": 85, + "active": 73, + "languages": { + "Python": 26, + "HTML": 21, + "JavaScript": 33, + "Shell": 10, + "TypeScript": 14, + "Dockerfile": 6, + "Go": 2, + "MDX": 1, + "C": 2, + "CSS": 1 + }, + "top_10_recent": [ + "blackboxprogramming/alexa-amundson-resume", + "blackboxprogramming/BlackRoad-Operating-System", + "blackboxprogramming/blackroad-os-kpis", + "blackboxprogramming/pi-mono", + "blackboxprogramming/system-prompts-and-models-of-ai-tools", + "blackboxprogramming/openclaw", + "blackboxprogramming/skills", + "blackboxprogramming/OpenSandbox", + "blackboxprogramming/OpenViking", + "blackboxprogramming/rowboat" + ], + "top_10_largest_mb": [ + { + "blackboxprogramming/openapi-generator": 876.1 + }, + { + "blackboxprogramming/docs": 694.2 + }, + { + "blackboxprogramming/blackroad": 385.9 + }, + { + "blackboxprogramming/hindsight": 344.7 + }, + { + "blackboxprogramming/git": 276.8 + }, + { + "blackboxprogramming/openclaw": 276.6 + }, + { + "blackboxprogramming/A2UI": 122.8 + }, + { + "blackboxprogramming/clerk-docs": 91.1 + }, + { + "blackboxprogramming/OpenSandbox": 78.2 + }, + { + "blackboxprogramming/rowboat": 76.1 + } + ] + }, + "orgs": { + "Blackbox-Enterprises": { + "repos": 8, + "members": 1 + }, + "BlackRoad-AI": { + "repos": 16, + "members": 1 + }, + "BlackRoad-OS": { + "repos": 1268, + "members": 1 + }, + "BlackRoad-Labs": { + "repos": 10, + "members": 1 + }, + "BlackRoad-Cloud": { + "repos": 10, + "members": 1 + }, + "BlackRoad-Ventures": { + "repos": 8, + "members": 1 + }, + "BlackRoad-Foundation": { + "repos": 14, + "members": 1 + }, + "BlackRoad-Media": { + "repos": 13, + "members": 1 + }, + "BlackRoad-Hardware": { + "repos": 14, + "members": 1 + }, + "BlackRoad-Education": { + "repos": 7, + "members": 1 + }, + "BlackRoad-Gov": { + "repos": 9, + "members": 1 + }, + "BlackRoad-Security": { + "repos": 16, + "members": 1 + }, + "BlackRoad-Interactive": { + "repos": 12, + "members": 1 + }, + "BlackRoad-Archive": { + "repos": 11, + "members": 1 + }, + "BlackRoad-Studio": { + "repos": 7, + "members": 1 + }, + "BlackRoad-OS-Inc": { + "repos": 22, + "members": 1 + } + } +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-github.json b/data/snapshots/2026-03-13-github.json new file mode 100644 index 0000000..d5753ff --- /dev/null +++ b/data/snapshots/2026-03-13-github.json @@ -0,0 +1,22 @@ +{ + "source": "github", + "collected_at": "2026-03-13T20:41:15Z", + "date": "2026-03-13", + "repos": { + "total": 306, + "total_size_mb": 4736.4, + "languages": {"HTML": 98, "Python": 77, "TypeScript": 34, "JavaScript": 19, "Shell": 18, "Dockerfile": 10, "Go": 2, "C++": 2, "CSS": 2, "Apex": 1} + }, + "commits": { + "today": 185, + "push_events_today": 95 + }, + "pull_requests": { + "open": 16, + "merged_today": 0, + "merged_total": 4019 + }, + "activity": { + "events_today": 100 + } +} diff --git a/data/snapshots/2026-03-13-loc.json b/data/snapshots/2026-03-13-loc.json new file mode 100644 index 0000000..cf5ea8f --- /dev/null +++ b/data/snapshots/2026-03-13-loc.json @@ -0,0 +1,24 @@ +{ + "source": "loc", + "collected_at": "2026-03-13T06:30:55Z", + "date": "2026-03-13", + "local": { + "total_code_lines": 4608051, + "repos": 73, + "files": 1723, + "scripts": 310, + "script_lines": 580298 + }, + "projects": { + "blackroad-os-kpis": 4804, + "images-blackroad": 2562, + "roadnet": 5064 + }, + "fleet": { + "alice": 1494087, + "cecilia": 402212, + "octavia": 0, + "lucidia": 708367 + }, + "total_estimated_loc": 7212717 +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-local.json b/data/snapshots/2026-03-13-local.json new file mode 100644 index 0000000..ba910c6 --- /dev/null +++ b/data/snapshots/2026-03-13-local.json @@ -0,0 +1,42 @@ +{ + "source": "local", + "collected_at": "2026-03-13T20:30:22Z", + "date": "2026-03-13", + "scripts": { + "bin_tools": 222, + "bin_size_mb": 121, + "home_scripts": 92, + "templates": 75 + }, + "databases": { + "sqlite_count": 230, + "blackroad_dir_mb": 1389, + "fts5_entries": 21334, + "systems_registered": 111 + }, + "packages": { + "homebrew": 293, + "pip3": 35, + "npm_global": 14 + }, + "automation": { + "cron_jobs": 17, + "local_git_repos": 13 + }, + "disk": { + "used_gb": 12, + "total_gb": 460, + "pct": 16 + }, + "system": { + "processes": 597, + "net_connections": 50 + }, + "files": { + "downloads": 94, + "documents": 4 + }, + "data": { + "total_db_rows": 23049 + } +} diff --git a/data/snapshots/2026-03-13-services.json b/data/snapshots/2026-03-13-services.json new file mode 100644 index 0000000..452aea4 --- /dev/null +++ b/data/snapshots/2026-03-13-services.json @@ -0,0 +1,188 @@ +{ + "source": "services", + "collected_at": "2026-03-13T20:44:45Z", + "date": "2026-03-13", + "totals": { + "ollama_models": 27, + "ollama_size_gb": 48.1, + "docker_containers": 14, + "docker_images": 15, + "postgres_dbs": 11, + "nginx_sites": 48, + "systemd_services": 253, + "systemd_timers": 35, + "systemd_failed": 10, + "processes": 869, + "network_connections": 118, + "swap_used_mb": 2038, + "swap_total_mb": 10849, + "tailscale_peers": 9 + }, + "nodes": { + "alice": { + "ollama": { + "count": 6, + "size_gb": 10.1, + "models": [ + "qwen2.5:3b", + "nomic-embed-text:latest", + "lucidia:latest", + "llama3.2:1b", + "tinyllama:latest", + "qwen2.5:1.5b" + ] + }, + "docker": { + "running": 0, + "images": 0, + "containers_total": 0, + "names": [] + }, + "postgres": { + "databases": 4 + }, + "nginx": { + "sites": 30, + "active": true + }, + "systemd": { + "services": 85, + "timers": 11, + "failed": 3 + }, + "processes": 220, + "connections": 11, + "swap": { + "used_mb": 0, + "total_mb": 99 + }, + "cloudflared": true, + "tailscale_peers": 0, + "hailo": false, + "crons": { + "root": 12, + "users": 15, + "total": 27 + } + }, + "cecilia": { + "ollama": { + "count": 15, + "size_gb": 27.9, + "models": [ + "deepseek-r1:1.5b", + "nomic-embed-text:latest", + "hf.co/mradermacher/OpenELM-1_1B-Instruct-GGUF:Q4_K_M", + "hf.co/mradermacher/OpenELM-3B-Instruct-GGUF:Q4_K_M", + "cece2:latest", + "qwen3:8b", + "llama3:8b-instruct-q4_K_M", + "cece:latest", + "deepseek-coder:1.3b", + "qwen2.5-coder:3b", + "llama3.2:3b", + "tinyllama:latest", + "llama3.2:latest", + "codellama:7b", + "llama3.2:1b" + ] + }, + "docker": { + "running": 0, + "images": 0, + "containers_total": 0, + "names": [] + }, + "postgres": { + "databases": 3 + }, + "nginx": { + "sites": 15, + "active": false + }, + "systemd": { + "services": 79, + "timers": 11, + "failed": 4 + }, + "processes": 260, + "connections": 49, + "swap": { + "used_mb": 1531, + "total_mb": 2047 + }, + "cloudflared": true, + "tailscale_peers": 0, + "hailo": true, + "crons": { + "root": 19, + "users": 21, + "total": 40 + } + }, + "octavia": { + "status": "offline" + }, + "lucidia": { + "ollama": { + "count": 6, + "size_gb": 10.1, + "models": [ + "qwen2.5:3b", + "nomic-embed-text:latest", + "lucidia:latest", + "llama3.2:1b", + "tinyllama:latest", + "qwen2.5:1.5b" + ] + }, + "docker": { + "running": 14, + "images": 15, + "containers_total": 15, + "names": [ + "blackroad-gitea", + "road-pdns-admin", + "road-pdns", + "road-dns-db", + "roadauth", + "roadapi", + "blackroad-edge-agent", + "blackroad.systems", + "blackroadai.com", + "blackroad-auth-gateway", + "blackroad-metaverse", + "blackroad-os", + "blackroad-os-carpool", + "pi-my-agent-1" + ] + }, + "postgres": { + "databases": 4 + }, + "nginx": { + "sites": 3, + "active": true + }, + "systemd": { + "services": 89, + "timers": 13, + "failed": 3 + }, + "processes": 389, + "connections": 58, + "swap": { + "used_mb": 507, + "total_mb": 8703 + }, + "cloudflared": true, + "tailscale_peers": 9, + "hailo": false, + "crons": { + "root": 18, + "users": 28, + "total": 46 + } + } + } +} \ No newline at end of file diff --git a/data/snapshots/2026-03-13-traffic.json b/data/snapshots/2026-03-13-traffic.json new file mode 100644 index 0000000..60537d7 --- /dev/null +++ b/data/snapshots/2026-03-13-traffic.json @@ -0,0 +1,26 @@ +{ + "source": "traffic", + "date": "2026-03-13", + "collected_at": "2026-03-13T21:36:59Z", + "cloudflare": { + "zones_count": 20, + "workers_total": 96, + "tunnels_total": 18, + "tunnels_healthy": 8, + "tunnels_inactive": 10 + }, + "github": { + "views_14d": 35, + "unique_visitors_14d": 18, + "clones_14d": 10121, + "unique_cloners_14d": 893, + "contributions_ytd": 0, + "commit_streak_days": 0, + "avg_commits_per_day": 0.0, + "issues_closed_total": 0, + "repos_updated_7d": 100, + "total_forks": 0, + "non_fork_repos": 0 + }, + "velocity": {} +} \ No newline at end of file diff --git a/data/snapshots/unknown-github-all-orgs.json b/data/snapshots/unknown-github-all-orgs.json new file mode 100644 index 0000000..ee7473b --- /dev/null +++ b/data/snapshots/unknown-github-all-orgs.json @@ -0,0 +1,57 @@ +{ + "source": "github-all-orgs", + "collected_at": "", + "date": "", + "totals": { + "repos": 1603, + "active": 306, + "archived": 1297, + "stars": 11, + "forks": 0, + "size_mb": 8085.9, + "open_issues": 44807, + "language_count": 20, + "org_count": 17 + }, + "languages": { + "Python": 470, + "HTML": 314, + "Shell": 160, + "JavaScript": 114, + "TypeScript": 85, + "Dockerfile": 24, + "C++": 5, + "CSS": 3, + "Go": 2, + "C": 2, + "MDX": 1, + "Apex": 1, + "Ruby": 1, + "Go Template": 1, + "HCL": 1, + "Java": 1, + "Bicep": 1, + "Rust": 1, + "C#": 1, + "PHP": 1 + }, + "orgs": { + "BlackRoad-OS": 1268, + "blackboxprogramming": 158, + "BlackRoad-OS-Inc": 22, + "BlackRoad-AI": 16, + "BlackRoad-Security": 16, + "BlackRoad-Foundation": 14, + "BlackRoad-Hardware": 14, + "BlackRoad-Media": 13, + "BlackRoad-Interactive": 12, + "BlackRoad-Archive": 11, + "BlackRoad-Labs": 10, + "BlackRoad-Cloud": 10, + "BlackRoad-Gov": 9, + "Blackbox-Enterprises": 8, + "BlackRoad-Ventures": 8, + "BlackRoad-Education": 7, + "BlackRoad-Studio": 7 + } +} \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 32bf1f0..b6bc220 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -8,6 +8,7 @@ KPI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DATA_DIR="$KPI_ROOT/data" TODAY=$(date +%Y-%m-%d) TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) +export KPI_ROOT DATA_DIR TODAY TIMESTAMP # Ensure data dirs mkdir -p "$DATA_DIR/daily" "$DATA_DIR/snapshots" "$DATA_DIR/raw" @@ -26,7 +27,8 @@ err() { echo -e "${RED} ✗${RESET} $*" >&2; } # Config GITHUB_USER="blackboxprogramming" -GITHUB_ORGS="Blackbox-Enterprises blackroad-os-inc" +GITHUB_ORGS="Blackbox-Enterprises BlackRoad-AI BlackRoad-OS BlackRoad-Labs BlackRoad-Cloud BlackRoad-Ventures BlackRoad-Foundation BlackRoad-Media BlackRoad-Hardware BlackRoad-Education BlackRoad-Gov BlackRoad-Security BlackRoad-Interactive BlackRoad-Archive BlackRoad-Studio BlackRoad-OS-Inc" +export GITHUB_USER GITHUB_ORGS GITEA_HOST="192.168.4.100" GITEA_PORT="3100" GITEA_ORGS="blackroad-os lucidia platform blackroad-ai blackroad-cloud blackroad-infra blackroad-labs" diff --git a/lib/slack.sh b/lib/slack.sh new file mode 100644 index 0000000..895b993 --- /dev/null +++ b/lib/slack.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Shared Slack utilities for all notification scripts +# Source this after common.sh + +SLACK_ENV="$HOME/.blackroad/slack-webhook.env" + +# Load webhook URLs from env file +slack_load() { + if [ -f "$SLACK_ENV" ]; then + source "$SLACK_ENV" + fi +} + +# Check if Slack is configured (not placeholder) +slack_ready() { + slack_load + [ -z "${SLACK_WEBHOOK_URL:-}" ] && return 1 + echo "$SLACK_WEBHOOK_URL" | grep -q "YOUR/WEBHOOK/URL" && return 1 + return 0 +} + +# Post raw JSON payload to a webhook URL +# Usage: slack_post "$payload" ["$webhook_url"] +slack_post() { + local payload="$1" + local webhook="${2:-${SLACK_WEBHOOK_URL:-}}" + + [ -z "$webhook" ] && return 1 + + local response + response=$(curl -sf -X POST -H 'Content-type: application/json' \ + --data "$payload" "$webhook" 2>&1) + + [ "$response" = "ok" ] +} + +# Post a simple text message +# Usage: slack_text "message" ["$webhook_url"] +slack_text() { + local msg="$1" + local webhook="${2:-${SLACK_WEBHOOK_URL:-}}" + local payload + payload=$(python3 -c " +import json, sys +print(json.dumps({'text': sys.argv[1]})) +" "$msg") + slack_post "$payload" "$webhook" +} + +# Post a notification with emoji prefix +# Usage: slack_notify ":emoji:" "Title" "Body text" ["$webhook_url"] +slack_notify() { + local emoji="$1" title="$2" body="$3" + local webhook="${4:-${SLACK_WEBHOOK_URL:-}}" + local payload + payload=$(python3 -c " +import json, sys +emoji, title, body = sys.argv[1], sys.argv[2], sys.argv[3] +blocks = [ + {'type': 'section', 'text': {'type': 'mrkdwn', 'text': f'{emoji} *{title}*\n{body}'}}, + {'type': 'context', 'elements': [{'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | blackroad-os'}]} +] +print(json.dumps({'blocks': blocks})) +" "$emoji" "$title" "$body") + slack_post "$payload" "$webhook" +} + +# Post to alerts channel (falls back to main) +slack_alert() { + local payload="$1" + local webhook="${SLACK_ALERTS_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" + slack_post "$payload" "$webhook" +} + +# Dedup: skip if same alert key posted within N seconds (default 3600) +# Usage: slack_dedup "key" [ttl_seconds] +# Returns 0 if should send, 1 if suppressed +slack_dedup() { + local key="$1" + local ttl="${2:-3600}" + local cache_dir="$HOME/.blackroad/logs" + mkdir -p "$cache_dir" + + local hash + hash=$(echo "$key" | md5 2>/dev/null || echo "$key" | md5sum 2>/dev/null | cut -d' ' -f1) + local cache_file="$cache_dir/slack-dedup-$hash" + + if [ -f "$cache_file" ]; then + local age=$(( $(date +%s) - $(stat -f %m "$cache_file" 2>/dev/null || stat -c %Y "$cache_file" 2>/dev/null || echo 0) )) + [ "$age" -lt "$ttl" ] && return 1 + fi + + touch "$cache_file" + return 0 +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..bb04db7 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,12 @@ +{ + "name": "blackroad-os-kpis", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "blackroad-os-kpis", + "version": "2.0.0" + } + } +} diff --git a/package.json b/package.json index 564f8c9..2fe5c75 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,10 @@ "collect:cloudflare": "bash collectors/cloudflare.sh", "report": "bash reports/daily-report.sh", "report:slack": "bash reports/slack-notify.sh", - "report:md": "bash reports/markdown-report.sh" + "report:md": "bash reports/markdown-report.sh", + "collect:all-orgs": "bash collectors/github-all-orgs.sh", + "push:kv": "bash reports/push-kv.sh", + "deploy:resumes": "cd ~/alexa-amundson-resume && wrangler deploy" }, "private": true } diff --git a/reports/daily-report.sh b/reports/daily-report.sh index db73652..68f074d 100644 --- a/reports/daily-report.sh +++ b/reports/daily-report.sh @@ -102,12 +102,12 @@ print(f""" Total LOC {G}{s['total_loc']:>10,}{R}{d('total_loc')} {A}═══ REPOSITORIES ══════════════════════════════════════════════{R} - GitHub repos {s['repos_github']:>6}{d('repos_github')} + GitHub repos {s['repos_github']:>6} {DIM}({s.get('github_org_count', 0)} orgs){R}{d('repos_github')} + GitHub active {s.get('repos_github_active', 0):>5} {DIM}archived {s.get('repos_github_archived', 0)}{R} + Languages {s.get('github_language_count', 0):>5}{d('github_language_count')} + GitHub size {round(s.get('github_all_size_mb', s.get('github_size_mb', 0)) / 1024, 1):>7} GB{d('github_all_size_mb')} Gitea repos {s['repos_gitea']:>6}{d('repos_gitea')} Total repos {W}{s['repos_total']:>6}{R}{d('repos_total')} - Active {s.get('repos_active', 0):>6} - Archived {s.get('repos_archived', 0):>6} - Total size {s.get('github_size_mb', 0):>7.0f} MB{d('github_size_mb')} Stars {s.get('github_stars', 0):>6} Forks {s.get('github_forks', 0)} Followers {s.get('github_followers', 0):>6} Following {s.get('github_following', 0)} diff --git a/reports/push-kv.sh b/reports/push-kv.sh new file mode 100755 index 0000000..7039978 --- /dev/null +++ b/reports/push-kv.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Push latest KPI data to Cloudflare KV for live resume dashboards + +source "$(dirname "$0")/../lib/common.sh" + +DAILY=$(today_file) + +if [ ! -f "$DAILY" ]; then + err "No daily data. Run collection first." + exit 1 +fi + +log "Pushing KPIs to Cloudflare KV..." + +# Build the KV payload with summary + metadata +python3 << PYEOF +import json, subprocess, sys + +with open('$DAILY') as f: + daily = json.load(f) + +s = daily['summary'] +s['_date'] = daily['date'] +s['_collected_at'] = daily['collected_at'] + +payload = json.dumps(s) + +# Write to temp file for wrangler +with open('/tmp/kpi-kv-payload.json', 'w') as f: + f.write(payload) + +print(f" Payload: {len(payload)} bytes, {len(s)} keys") +PYEOF + +# Push to KV namespace +wrangler kv key put "latest" --namespace-id="750d4fb38d874133aebca49b697db2ef" --path="/tmp/kpi-kv-payload.json" --remote 2>&1 + +if [ $? -eq 0 ]; then + ok "KPIs pushed to KV (resume-kpis/latest)" +else + err "Failed to push KPIs to KV" +fi diff --git a/reports/slack-alert.sh b/reports/slack-alert.sh new file mode 100755 index 0000000..c738a8a --- /dev/null +++ b/reports/slack-alert.sh @@ -0,0 +1,225 @@ +#!/bin/bash +# Real-time Slack alerts for fleet/service issues +# Posts to SLACK_ALERTS_WEBHOOK_URL (or falls back to SLACK_WEBHOOK_URL) +# +# Usage: slack-alert.sh — auto-detect issues from latest KPI data +# slack-alert.sh "message" — post a custom alert +# slack-alert.sh git-patrol — post git-agent patrol results + +source "$(dirname "$0")/../lib/common.sh" +source "$(dirname "$0")/../lib/slack.sh" + +slack_load + +if ! slack_ready; then + err "Slack not configured. Run: bash scripts/setup-slack.sh" + exit 1 +fi + +# ─── Custom message mode ───────────────────────────────────────────── +if [ -n "${1:-}" ] && [ "$1" != "git-patrol" ]; then + payload=$(python3 -c " +import json, sys +msg = ' '.join(sys.argv[1:]) +blocks = [ + {'type': 'section', 'text': {'type': 'mrkdwn', 'text': f':rotating_light: *BlackRoad Alert*\n{msg}'}}, + {'type': 'context', 'elements': [{'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | slack-alert.sh'}]} +] +print(json.dumps({'blocks': blocks})) +" "$@") + slack_alert "$payload" + ok "Alert posted: $*" + exit 0 +fi + +# ─── Git patrol mode ───────────────────────────────────────────────── +if [ "${1:-}" = "git-patrol" ]; then + AGENT_SCRIPT="$(dirname "$0")/../agents/git-agent.sh" + if [ ! -x "$AGENT_SCRIPT" ]; then + err "git-agent.sh not found" + exit 1 + fi + + patrol_output=$(bash "$AGENT_SCRIPT" health 2>&1) + fleet_output=$(bash "$AGENT_SCRIPT" fleet status 2>&1) + + payload=$(python3 -c " +import json, sys, re + +patrol = '''$patrol_output''' +fleet = '''$fleet_output''' + +# Parse health output +issues = [] +for line in patrol.split('\n'): + if '✗' in line: + # Strip ANSI codes + clean = re.sub(r'\033\[[0-9;]*m', '', line).strip() + if clean: + issues.append(clean.lstrip('✗ ')) + +fleet_lines = [] +for line in fleet.split('\n'): + clean = re.sub(r'\033\[[0-9;]*m', '', line).strip() + if 'repos=' in clean: + fleet_lines.append(clean.lstrip('✓ ')) + +health_text = '\n'.join(f'• {i}' for i in issues) if issues else ':white_check_mark: All repos clean' +fleet_text = '\n'.join(f'• {l}' for l in fleet_lines) if fleet_lines else 'No fleet data' + +blocks = [ + {'type': 'header', 'text': {'type': 'plain_text', 'text': 'Git Agent Patrol Report'}}, + {'type': 'section', 'fields': [ + {'type': 'mrkdwn', 'text': f':mag: *Local Repos*\n{health_text}'}, + {'type': 'mrkdwn', 'text': f':satellite: *Fleet Repos*\n{fleet_text}'}, + ]}, + {'type': 'context', 'elements': [ + {'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | git-agent patrol'} + ]} +] +print(json.dumps({'blocks': blocks})) +") + + slack_alert "$payload" + ok "Git patrol posted to Slack" + exit 0 +fi + +# ─── Auto-detect mode — scan latest KPIs for alertable issues ──────── +DAILY=$(today_file) +[ ! -f "$DAILY" ] && { err "No daily data"; exit 1; } + +export DAILY +alerts=$(python3 << 'PYEOF' +import json, os + +with open(os.environ['DAILY']) as f: + s = json.load(f).get('summary', {}) + +alerts = [] + +# Fleet nodes down +offline = s.get('fleet_offline', []) +if offline: + alerts.append({ + 'severity': 'critical', + 'emoji': ':rotating_light:', + 'text': f"*Nodes offline*: {', '.join(offline)}" + }) + +# Fleet degraded +online = s.get('fleet_online', 0) +total = s.get('fleet_total', 4) +if online < total and not offline: + alerts.append({ + 'severity': 'warning', + 'emoji': ':large_yellow_circle:', + 'text': f"*Fleet degraded*: {online}/{total} online" + }) + +# Failed systemd units +failed = s.get('failed_units', 0) +if failed > 0: + alerts.append({ + 'severity': 'warning', + 'emoji': ':warning:', + 'text': f"*{failed} failed systemd units*" + }) + +# Throttled nodes (undervoltage/thermal) +throttled = s.get('throttled_nodes', []) +if throttled: + alerts.append({ + 'severity': 'warning', + 'emoji': ':zap:', + 'text': f"*Throttled nodes*: {', '.join(throttled)}" + }) + +# High temperature +temp = s.get('avg_temp_c', 0) +if temp > 70: + alerts.append({ + 'severity': 'critical' if temp > 80 else 'warning', + 'emoji': ':fire:', + 'text': f"*High fleet temp*: {temp:.1f}C avg" + }) + +# Disk pressure (fleet) +disk_used = s.get('fleet_disk_used_gb', 0) +disk_total = s.get('fleet_disk_total_gb', 1) +if disk_total > 0 and (disk_used / disk_total) > 0.85: + pct = round(disk_used / disk_total * 100) + alerts.append({ + 'severity': 'warning', + 'emoji': ':floppy_disk:', + 'text': f"*Fleet disk {pct}%*: {disk_used}/{disk_total} GB" + }) + +# Mac disk pressure +mac_pct = s.get('mac_disk_pct', 0) +if mac_pct > 85: + alerts.append({ + 'severity': 'warning', + 'emoji': ':computer:', + 'text': f"*Mac disk at {mac_pct}%*" + }) + +# Low autonomy score +score = s.get('autonomy_score', 0) +if score < 30: + alerts.append({ + 'severity': 'warning', + 'emoji': ':robot_face:', + 'text': f"*Low autonomy score*: {score}/100" + }) + +# Too many service restarts (possible crash loop) +restarts = s.get('service_restarts_today', 0) +if restarts > 100: + alerts.append({ + 'severity': 'warning', + 'emoji': ':repeat:', + 'text': f"*{restarts} service restarts today* — possible crash loop" + }) + +print(json.dumps(alerts)) +PYEOF +) + +if [ "$alerts" = "[]" ]; then + log "No alerts to send" + exit 0 +fi + +# Build and send alert payload +payload=$(python3 -c " +import json + +alerts = json.loads('''$alerts''') + +text_lines = [] +for a in alerts: + key = a['text'] + text_lines.append(f\"{a['emoji']} {a['text']}\") + +severity = 'critical' if any(a['severity'] == 'critical' for a in alerts) else 'warning' +header_emoji = ':rotating_light:' if severity == 'critical' else ':warning:' + +blocks = [ + {'type': 'header', 'text': {'type': 'plain_text', 'text': f'{header_emoji} BlackRoad Fleet Alert'}}, + {'type': 'section', 'text': {'type': 'mrkdwn', 'text': chr(10).join(text_lines)}}, + {'type': 'context', 'elements': [ + {'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | slack-alert.sh auto-detect'} + ]} +] +print(json.dumps({'blocks': blocks})) +") + +# Check dedup for the overall alert set +alert_key=$(echo "$alerts" | md5 2>/dev/null || echo "$alerts" | md5sum | cut -d' ' -f1) +if slack_dedup "$alert_key"; then + slack_alert "$payload" + ok "Alert posted to Slack ($(echo "$alerts" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))') issues)" +else + log "Alert suppressed (already sent within 1 hour)" +fi diff --git a/reports/slack-notify.sh b/reports/slack-notify.sh old mode 100644 new mode 100755 index 9e68ceb..3fb320b --- a/reports/slack-notify.sh +++ b/reports/slack-notify.sh @@ -1,87 +1,218 @@ #!/bin/bash # Post daily KPI report to Slack (blackroadosinc.slack.com) +# Enhanced: trend deltas, fleet health, git-agent status, alert severity +# # Requires SLACK_WEBHOOK_URL env var or ~/.blackroad/slack-webhook.env +# Optional: SLACK_ALERTS_WEBHOOK_URL for #alerts channel source "$(dirname "$0")/../lib/common.sh" +source "$(dirname "$0")/../lib/slack.sh" -# Load webhook URL -if [ -z "$SLACK_WEBHOOK_URL" ] && [ -f "$HOME/.blackroad/slack-webhook.env" ]; then - source "$HOME/.blackroad/slack-webhook.env" -fi +slack_load -if [ -z "$SLACK_WEBHOOK_URL" ]; then - err "No SLACK_WEBHOOK_URL set. Create ~/.blackroad/slack-webhook.env with:" - err " SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T.../B.../..." - err "" - err "To set up:" - err " 1. Go to blackroadosinc.slack.com → Apps → Incoming Webhooks" - err " 2. Add webhook to your #kpis or #general channel" - err " 3. Save URL to ~/.blackroad/slack-webhook.env" +if ! slack_ready; then + err "Slack not configured. Run: bash scripts/setup-slack.sh" exit 1 fi DAILY=$(today_file) if [ ! -f "$DAILY" ]; then - err "No daily data for $TODAY. Run: npm run collect" + err "No daily data for $TODAY. Run: bash collectors/collect-all.sh" exit 1 fi -# Build Slack message -payload=$(python3 << 'PYEOF' -import json, os +YESTERDAY=$(date -v-1d +%Y-%m-%d 2>/dev/null || date -d '1 day ago' +%Y-%m-%d) +YESTERDAY_FILE="$DATA_DIR/daily/${YESTERDAY}.json" +GIT_AGENT_LOG="$HOME/.blackroad/logs/git-agent.log" -daily_file = os.path.join(os.environ.get('DATA_DIR', 'data'), 'daily', os.environ.get('TODAY', '') + '.json') -# Fallback -import glob -files = sorted(glob.glob('data/daily/*.json')) -if not files: - files = sorted(glob.glob(os.path.expanduser('~/blackroad-os-kpis/data/daily/*.json'))) -if files: - daily_file = files[-1] +export DAILY YESTERDAY_FILE GIT_AGENT_LOG + +# ─── Build Slack payload ───────────────────────────────────────────── +payload=$(python3 << 'PYEOF' +import json, os, glob + +daily_file = os.environ.get('DAILY', '') +yesterday_file = os.environ.get('YESTERDAY_FILE', '') +git_log = os.environ.get('GIT_AGENT_LOG', '') with open(daily_file) as f: data = json.load(f) s = data['summary'] +# Yesterday's data for deltas +ys = {} +if yesterday_file and os.path.exists(yesterday_file): + with open(yesterday_file) as f: + ys = json.load(f).get('summary', {}) + +def delta(key, invert=False): + """Show delta from yesterday: +N or -N""" + curr = s.get(key, 0) + prev = ys.get(key, 0) + if not prev or not isinstance(curr, (int, float)): + return '' + diff = curr - prev + if diff == 0: + return '' + sign = '+' if diff > 0 else '' + emoji = '' + if invert: # lower is better (failed_units, throttled) + emoji = ' :small_red_triangle:' if diff > 0 else ' :small_red_triangle_down:' + else: + emoji = ' :chart_with_upwards_trend:' if diff > 0 else ' :chart_with_downwards_trend:' + return f" ({sign}{diff}{emoji})" + +def fmt(n): + if isinstance(n, float): + return f"{n:,.1f}" + if isinstance(n, int) and n >= 1000: + return f"{n:,}" + return str(n) + +# Fleet status emoji +fleet_online = s.get('fleet_online', 0) +fleet_total = s.get('fleet_total', 4) +fleet_emoji = ':large_green_circle:' if fleet_online == fleet_total else ':red_circle:' if fleet_online <= 1 else ':large_yellow_circle:' + +# Autonomy score bar +score = s.get('autonomy_score', 0) +filled = score // 10 +score_bar = ':black_large_square:' * filled + ':white_large_square:' * (10 - filled) + +# Git agent last patrol +git_status = 'No patrol data' +if git_log and os.path.exists(git_log): + with open(git_log) as f: + lines = f.readlines() + patrols = [l.strip() for l in lines if 'PATROL:' in l] + if patrols: + last = patrols[-1] + git_status = last.split('] ', 1)[-1] if '] ' in last else last + +# Alerts +alerts = [] +offline = s.get('fleet_offline', []) +if offline: + alerts.append(f":rotating_light: *Nodes offline*: {', '.join(offline)}") +if s.get('failed_units', 0) > 0: + alerts.append(f":warning: *{s['failed_units']} failed systemd units*") +throttled = s.get('throttled_nodes', []) +if throttled: + alerts.append(f":fire: *Throttled*: {', '.join(throttled)}") +if s.get('avg_temp_c', 0) > 70: + alerts.append(f":thermometer: *High temp*: {s['avg_temp_c']}C avg") +if fleet_online < fleet_total: + alerts.append(f":satellite: *Fleet degraded*: {fleet_online}/{fleet_total} online") + +alert_text = '\n'.join(alerts) if alerts else ':white_check_mark: All systems nominal' + +# Weekly trend (last 7 days of commits) +trend_commits = [] +daily_dir = os.path.dirname(daily_file) +for f in sorted(glob.glob(os.path.join(daily_dir, '*.json')))[-7:]: + try: + with open(f) as fh: + d = json.load(fh) + trend_commits.append(d.get('summary', {}).get('commits_today', 0)) + except: + pass + +sparkline = '' +if trend_commits: + max_c = max(trend_commits) or 1 + bars = ['_', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'] + sparkline = ''.join(bars[min(8, int(c / max_c * 8))] for c in trend_commits) + sparkline = f"`{sparkline}` (7d commits)" + blocks = [ { "type": "header", - "text": {"type": "plain_text", "text": f"🛣 BlackRoad OS KPIs — {data['date']}"} + "text": {"type": "plain_text", "text": f"BlackRoad OS — Daily KPIs {data['date']}"} }, + + # ── Alerts section ── + { + "type": "section", + "text": {"type": "mrkdwn", "text": alert_text} + }, + {"type": "divider"}, + + # ── Code velocity ── { "type": "section", "fields": [ - {"type": "mrkdwn", "text": f"*Commits Today*\n{s['commits_today']}"}, - {"type": "mrkdwn", "text": f"*PRs Merged Today*\n{s['prs_merged_today']}"}, - {"type": "mrkdwn", "text": f"*PRs Open*\n{s['prs_open']}"}, - {"type": "mrkdwn", "text": f"*Total LOC*\n{s['total_loc']:,}"}, + {"type": "mrkdwn", "text": f":rocket: *Code Velocity*\n" + f"Commits: *{s['commits_today']}*{delta('commits_today')}\n" + f"PRs merged: *{s['prs_merged_today']}*{delta('prs_merged_today')}\n" + f"PRs open: {s['prs_open']}{delta('prs_open')}\n" + f"Events: {s.get('github_events_today', 0)}{delta('github_events_today')}"}, + {"type": "mrkdwn", "text": f":bar_chart: *Scale*\n" + f"LOC: *{fmt(s['total_loc'])}*{delta('total_loc')}\n" + f"Repos: *{s['repos_total']}* ({s['repos_github']} GH + {s['repos_gitea']} Gitea){delta('repos_total')}\n" + f"Languages: {s.get('github_language_count', 0)}\n" + f"{sparkline}"}, ] }, + + # ── Fleet + Services ── { "type": "section", "fields": [ - {"type": "mrkdwn", "text": f"*Repos (GH + Gitea)*\n{s['repos_github']} + {s['repos_gitea']} = {s['repos_total']}"}, - {"type": "mrkdwn", "text": f"*Fleet*\n{s['fleet_online']}/{s['fleet_total']} online"}, - {"type": "mrkdwn", "text": f"*Docker*\n{s['docker_containers']} containers"}, - {"type": "mrkdwn", "text": f"*Ollama*\n{s['ollama_models']} models"}, + {"type": "mrkdwn", "text": f"{fleet_emoji} *Fleet*\n" + f"Online: *{fleet_online}/{fleet_total}*{delta('fleet_online')}\n" + f"Temp: {s.get('avg_temp_c', 0):.1f}C\n" + f"Mem: {s.get('fleet_mem_used_mb', 0)}/{s.get('fleet_mem_total_mb', 0)} MB\n" + f"Disk: {s.get('fleet_disk_used_gb', 0)}/{s.get('fleet_disk_total_gb', 0)} GB"}, + {"type": "mrkdwn", "text": f":gear: *Services*\n" + f"Ollama: *{s.get('ollama_models', 0)}* models ({s.get('ollama_size_gb', 0):.1f} GB)\n" + f"Docker: {s.get('docker_containers', 0)} containers\n" + f"Systemd: {s.get('systemd_services', 0)} svc / {s.get('systemd_timers', 0)} timers\n" + f"Nginx: {s.get('nginx_sites', 0)} sites"}, ] }, + + # ── Autonomy + Cloud ── { "type": "section", "fields": [ - {"type": "mrkdwn", "text": f"*Autonomy Score*\n{s['autonomy_score']}/100"}, - {"type": "mrkdwn", "text": f"*Avg Temp*\n{s['avg_temp_c']}°C"}, - {"type": "mrkdwn", "text": f"*Failed Units*\n{s['failed_units']}"}, - {"type": "mrkdwn", "text": f"*Throttled*\n{', '.join(s.get('throttled_nodes', [])) or 'None'}"}, + {"type": "mrkdwn", "text": f":robot_face: *Autonomy*\n" + f"Score: *{score}/100*{delta('autonomy_score')}\n" + f"{score_bar}\n" + f"Heals: {s.get('heal_events_today', 0)} | Restarts: {s.get('service_restarts_today', 0)}\n" + f"Crons: {s.get('fleet_cron_jobs', 0)} | Uptime: {s.get('max_uptime_days', 0)}d"}, + {"type": "mrkdwn", "text": f":cloud: *Cloudflare*\n" + f"Pages: {s.get('cf_pages', 0)}{delta('cf_pages')}\n" + f"D1: {s.get('cf_d1_databases', 0)} | KV: {s.get('cf_kv_namespaces', 0)}\n" + f"R2: {s.get('cf_r2_buckets', 0)}\n" + f"DBs total: {s.get('sqlite_dbs', 0)} SQLite + {s.get('postgres_dbs', 0)} PG + {s.get('cf_d1_databases', 0)} D1"}, + ] + }, + + # ── Local Mac ── + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f":computer: *Local Mac*\n" + f"CLI tools: {s.get('bin_tools', 0)} | Scripts: {s.get('home_scripts', 0)}\n" + f"Git repos: {s.get('local_git_repos', 0)}\n" + f"Disk: {s.get('mac_disk_used_gb', 0)} GB ({s.get('mac_disk_pct', 0)}%)\n" + f"Processes: {s.get('mac_processes', 0)}"}, + {"type": "mrkdwn", "text": f":file_cabinet: *Data*\n" + f"SQLite DBs: {s.get('sqlite_dbs', 0)}\n" + f"Total DB rows: {fmt(s.get('total_db_rows', 0))}\n" + f"FTS5 entries: {fmt(s.get('fts5_entries', 0))}\n" + f"Packages: {s.get('brew_packages', 0)} brew / {s.get('pip_packages', 0)} pip / {s.get('npm_global_packages', 0)} npm"}, ] }, {"type": "divider"}, + + # ── Git agent status ── { "type": "context", "elements": [ - {"type": "mrkdwn", "text": f"Collected at {data['collected_at']} | blackroad-os-kpis"} + {"type": "mrkdwn", "text": f":satellite_antenna: Git Agent: _{git_status}_ | Collected {data['collected_at']}"} ] } ] @@ -91,14 +222,15 @@ print(json.dumps(payload)) PYEOF ) -log "Posting to Slack..." -response=$(curl -sf -X POST -H 'Content-type: application/json' \ - --data "$payload" \ - "$SLACK_WEBHOOK_URL" 2>&1) - -if [ "$response" = "ok" ]; then - ok "Posted to Slack" -else - err "Slack post failed: $response" +if [ -z "$payload" ]; then + err "Failed to build Slack payload" exit 1 fi + +# ─── Post to Slack ─────────────────────────────────────────────────── +log "Posting daily report to Slack..." +if slack_post "$payload"; then + ok "Daily report posted to Slack" +else + err "Slack post failed" +fi diff --git a/reports/slack-weekly.sh b/reports/slack-weekly.sh new file mode 100755 index 0000000..88222ec --- /dev/null +++ b/reports/slack-weekly.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Weekly Slack digest — posts Sunday night summary of the week's KPIs +# Compares this week vs last week, shows trends and highlights + +source "$(dirname "$0")/../lib/common.sh" +source "$(dirname "$0")/../lib/slack.sh" + +slack_load + +if ! slack_ready; then + err "Slack not configured" + exit 1 +fi + +export DATA_DIR + +payload=$(python3 << 'PYEOF' +import json, os, glob +from datetime import datetime, timedelta + +data_dir = os.environ.get('DATA_DIR', 'data') +daily_dir = os.path.join(data_dir, 'daily') + +# Load all daily files +dailies = {} +for f in sorted(glob.glob(os.path.join(daily_dir, '*.json'))): + try: + with open(f) as fh: + d = json.load(fh) + dailies[d['date']] = d.get('summary', {}) + except: + pass + +if not dailies: + print('{}') + exit() + +today = datetime.now() +# This week = last 7 days, Last week = 7-14 days ago +this_week = [] +last_week = [] + +for i in range(7): + day = (today - timedelta(days=i)).strftime('%Y-%m-%d') + if day in dailies: + this_week.append(dailies[day]) + +for i in range(7, 14): + day = (today - timedelta(days=i)).strftime('%Y-%m-%d') + if day in dailies: + last_week.append(dailies[day]) + +def week_sum(data, key): + return sum(d.get(key, 0) for d in data) + +def week_avg(data, key): + vals = [d.get(key, 0) for d in data if d.get(key, 0)] + return round(sum(vals) / len(vals), 1) if vals else 0 + +def week_max(data, key): + vals = [d.get(key, 0) for d in data] + return max(vals) if vals else 0 + +def week_last(data, key): + return data[0].get(key, 0) if data else 0 + +def trend(curr, prev): + if not prev: + return '' + diff = curr - prev + pct = round(diff / prev * 100) if prev else 0 + if diff > 0: + return f' (+{diff}, +{pct}%) :chart_with_upwards_trend:' + elif diff < 0: + return f' ({diff}, {pct}%) :chart_with_downwards_trend:' + return ' (=)' + +# Key metrics +tw_commits = week_sum(this_week, 'commits_today') +lw_commits = week_sum(last_week, 'commits_today') +tw_prs = week_sum(this_week, 'prs_merged_today') +lw_prs = week_sum(last_week, 'prs_merged_today') +tw_events = week_sum(this_week, 'github_events_today') +lw_events = week_sum(last_week, 'github_events_today') + +# Latest values +latest = this_week[0] if this_week else {} +loc = latest.get('total_loc', 0) +repos = latest.get('repos_total', 0) +fleet = latest.get('fleet_online', 0) +fleet_total = latest.get('fleet_total', 4) +autonomy = latest.get('autonomy_score', 0) +models = latest.get('ollama_models', 0) + +# Sparkline for commits +commit_vals = [] +for i in range(6, -1, -1): + day = (today - timedelta(days=i)).strftime('%Y-%m-%d') + commit_vals.append(dailies.get(day, {}).get('commits_today', 0)) + +bars = ['_', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'] +max_c = max(commit_vals) or 1 +sparkline = ''.join(bars[min(8, int(c / max_c * 8))] for c in commit_vals) + +# Uptime percentage +fleet_days = [d.get('fleet_online', 0) for d in this_week] +fleet_totals = [d.get('fleet_total', 4) for d in this_week] +uptime_pct = round(sum(fleet_days) / sum(fleet_totals) * 100) if sum(fleet_totals) > 0 else 0 + +# Build blocks +week_start = (today - timedelta(days=6)).strftime('%b %d') +week_end = today.strftime('%b %d') + +blocks = [ + { + "type": "header", + "text": {"type": "plain_text", "text": f"BlackRoad OS — Weekly Digest ({week_start} - {week_end})"} + }, + + # Velocity + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": + f":rocket: *Code Velocity (7d)*\n" + f"Commits: *{tw_commits}*{trend(tw_commits, lw_commits)}\n" + f"PRs merged: *{tw_prs}*{trend(tw_prs, lw_prs)}\n" + f"GH events: *{tw_events}*{trend(tw_events, lw_events)}\n" + f"`{sparkline}` daily commits"}, + {"type": "mrkdwn", "text": + f":bar_chart: *Current State*\n" + f"LOC: *{loc:,}*\n" + f"Repos: *{repos}*\n" + f"Languages: {latest.get('github_language_count', 0)}\n" + f"Autonomy: *{autonomy}/100*"}, + ] + }, + + # Fleet + infra + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": + f":satellite: *Fleet*\n" + f"Online: {fleet}/{fleet_total}\n" + f"Uptime: {uptime_pct}% this week\n" + f"Avg temp: {week_avg(this_week, 'avg_temp_c')}C\n" + f"Max uptime: {week_max(this_week, 'max_uptime_days')}d"}, + {"type": "mrkdwn", "text": + f":gear: *Services*\n" + f"Ollama: {models} models\n" + f"Docker: {latest.get('docker_containers', 0)} containers\n" + f"Nginx: {latest.get('nginx_sites', 0)} sites\n" + f"DBs: {latest.get('sqlite_dbs', 0)} SQLite + {latest.get('cf_d1_databases', 0)} D1"}, + ] + }, + + # Highlights + { + "type": "section", + "text": {"type": "mrkdwn", "text": + f":sparkles: *Week Highlights*\n" + f"• {len(this_week)} days of data collected\n" + f"• Peak commits: {max(commit_vals)} in a single day\n" + f"• Total heal events: {week_sum(this_week, 'heal_events_today')}\n" + f"• Service restarts: {week_sum(this_week, 'service_restarts_today')}"} + }, + {"type": "divider"}, + { + "type": "context", + "elements": [ + {"type": "mrkdwn", "text": f":calendar: Week of {week_start} | blackroad-os-kpis weekly digest"} + ] + } +] + +print(json.dumps({"blocks": blocks})) +PYEOF +) + +if [ -z "$payload" ] || [ "$payload" = "{}" ]; then + err "Not enough data for weekly digest" + exit 1 +fi + +log "Posting weekly digest to Slack..." +if slack_post "$payload"; then + ok "Weekly digest posted" +else + err "Weekly digest failed" +fi diff --git a/reports/update-resumes.sh b/reports/update-resumes.sh new file mode 100644 index 0000000..34b1b8a --- /dev/null +++ b/reports/update-resumes.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Auto-update resume repo with latest verified metrics from KPI collection +# Runs after daily collection to keep all 20 resumes current + +source "$(dirname "$0")/../lib/common.sh" + +RESUME_DIR="$HOME/alexa-amundson-resume" +DAILY=$(today_file) + +if [ ! -f "$DAILY" ]; then + err "No daily data. Run collection first." + exit 1 +fi + +if [ ! -d "$RESUME_DIR" ]; then + err "Resume repo not found at $RESUME_DIR" + exit 1 +fi + +log "Updating resume metrics from KPI data..." + +# Get all-org stats +ALL_ORGS="$DATA_DIR/snapshots/${TODAY}-github-all-orgs.json" + +python3 << PYEOF +import json, os, glob, re + +# Load daily KPI data +with open('$DAILY') as f: + daily = json.load(f) +s = daily['summary'] + +# Load all-org data if available +all_orgs = {} +aof = '$ALL_ORGS' +if os.path.exists(aof): + with open(aof) as f: + all_orgs = json.load(f) + +t = all_orgs.get('totals', {}) +langs = all_orgs.get('languages', {}) +orgs = all_orgs.get('orgs', {}) + +# Build verified metrics +github_repos = t.get('repos', s.get('repos_github', 0)) +github_active = t.get('active', s.get('repos_active', 0)) +github_archived = t.get('archived', s.get('repos_archived', 0)) +github_size_mb = t.get('size_mb', s.get('github_size_mb', 0)) +github_size_gb = round(github_size_mb / 1024, 1) if github_size_mb else 0 +lang_count = t.get('language_count', 20) +org_count = t.get('org_count', 17) +gitea_repos = 207 # from memory, Octavia offline +total_repos = github_repos + gitea_repos +stars = t.get('stars', s.get('github_stars', 0)) + +# Write VERIFIED-METRICS.md +metrics = f"""# Verified BlackRoad OS Metrics ({daily['date']}) + +Source: blackroad-os-kpis automated collection + full GitHub API scan across all {org_count} owners. + +## Code +- {s['total_loc']:,} lines of code +- {s['commits_today']} commits/day, {s['prs_merged_total']:,} PRs merged all-time +- **{github_repos:,} GitHub repositories** across {org_count} owners ({github_active:,} active, {github_archived:,} archived) +- **{gitea_repos} Gitea repositories** across 7 organizations (self-hosted) +- **{total_repos:,} total repositories** +- {lang_count} languages: {', '.join(f'{l} ({c})' for l, c in sorted(langs.items(), key=lambda x: -x[1])[:15])} +- {github_size_gb} GB total GitHub repo size +- {stars} stars, {org_count} organizations + +## GitHub Organizations ({len(orgs)} owners) +""" +for org_name, count in sorted(orgs.items(), key=lambda x: -x[1]): + metrics += f"- {org_name}: {count:,} repos\\n" + +metrics += f""" +## Infrastructure +- 5 Raspberry Pi nodes (Pi 5 x 4, Pi 400 x 1), 2 DigitalOcean droplets +- 52 TOPS AI acceleration (2x Hailo-8 NPUs) +- {s.get('fleet_mem_total_mb', 20000) // 1000} GB fleet RAM, {s.get('fleet_disk_total_gb', 707)} GB fleet storage +- WireGuard mesh VPN across all nodes + +## AI/ML +- {s.get('ollama_models', 27)} Ollama models deployed ({s.get('ollama_size_gb', 48.1)} GB) +- 4 custom fine-tuned CECE models +- 2x Hailo-8 NPU (52 TOPS) + +## Cloud (Cloudflare) +- {s.get('cf_pages', 99)} Pages projects +- {s.get('cf_d1_databases', 22)} D1 databases +- {s.get('cf_kv_namespaces', 46)} KV namespaces +- {s.get('cf_r2_buckets', 11)} R2 buckets +- 48+ custom domains via 4 tunnels + +## Services +- {s.get('docker_containers', 14)} Docker containers +- {s.get('postgres_dbs', 11)} PostgreSQL databases +- {s.get('nginx_sites', 48)} Nginx sites +- {s.get('systemd_services', 256)} systemd services +- {s.get('systemd_timers', 35)} timers +- {s.get('tailscale_peers', 9)} Tailscale peers + +## Automation +- {s.get('bin_tools', 212)} CLI tools ({s.get('bin_size_mb', 121)} MB) +- {s.get('home_scripts', 91)} shell scripts +- {s.get('mac_cron_jobs', 17)} Mac crons + {s.get('systemd_timers', 35)} fleet timers = {s.get('mac_cron_jobs', 17) + s.get('systemd_timers', 35)} automated tasks +- {s.get('sqlite_dbs', 230)} SQLite databases ({s.get('blackroad_dir_mb', 1390)} MB) +- {s.get('systems_registered', 111)} registered systems +- 60+ KPIs tracked daily across 9 collectors +""" + +metrics_path = os.path.join('$RESUME_DIR', 'VERIFIED-METRICS.md') +with open(metrics_path, 'w') as f: + f.write(metrics) + +print(f"Updated VERIFIED-METRICS.md") + +# Update README quick stats +readme_path = os.path.join('$RESUME_DIR', 'README.md') +with open(readme_path) as f: + readme = f.read() + +# Replace the stats block +old_stats_start = readme.find('## Verified Metrics') +old_stats_end = readme.find('---', old_stats_start + 1) +if old_stats_start > 0 and old_stats_end > 0: + new_stats = f"""## Verified Metrics ({daily['date']}) + +All numbers collected by [blackroad-os-kpis](https://github.com/blackboxprogramming/blackroad-os-kpis). + +\`\`\` +CODE + Lines of code {s['total_loc']:>10,} + Commits/day {s['commits_today']:>10} + PRs merged (all time) {s['prs_merged_total']:>10,} + GitHub repos {github_repos:>10,} ({github_active} active, {org_count} orgs) + Gitea repos {gitea_repos:>10} (7 orgs) + Total repos {total_repos:>10,} + Languages {lang_count:>10} + GitHub size {github_size_mb:>8} GB + +INFRASTRUCTURE + Fleet nodes {s['fleet_total']:>10} + Systemd services {s.get('systemd_services', 256):>10} + Docker containers {s.get('docker_containers', 14):>10} + Nginx sites {s.get('nginx_sites', 48):>10} + Fleet storage {s.get('fleet_disk_total_gb', 707):>7} GB + Fleet RAM {s.get('fleet_mem_total_mb', 20000) // 1000:>7} GB + +AI + Models deployed {s.get('ollama_models', 27):>10} ({s.get('ollama_size_gb', 48.1)} GB) + AI acceleration {52:>7} TOPS + Custom models {4:>10} + +CLOUD (Cloudflare) + Pages projects {s.get('cf_pages', 99):>10} + D1 databases {s.get('cf_d1_databases', 22):>10} + KV namespaces {s.get('cf_kv_namespaces', 46):>10} + R2 buckets {s.get('cf_r2_buckets', 11):>10} + Domains 48+ + +DATA + Total databases {s.get('postgres_dbs', 11) + s.get('sqlite_dbs', 230) + s.get('cf_d1_databases', 22):>10} + PostgreSQL {s.get('postgres_dbs', 11):>10} + SQLite {s.get('sqlite_dbs', 230):>10} ({s.get('blackroad_dir_mb', 1390)} MB) + +AUTOMATION + CLI tools {s.get('bin_tools', 212):>10} ({s.get('bin_size_mb', 121)} MB) + Automated tasks {s.get('mac_cron_jobs', 17) + s.get('systemd_timers', 35):>10} + KPIs tracked 60+ + Data collectors {9:>10} +\`\`\` + +""" + readme = readme[:old_stats_start] + new_stats + readme[old_stats_end:] + with open(readme_path, 'w') as f: + f.write(readme) + print("Updated README.md stats block") + +PYEOF + +# Commit and push if there are changes +cd "$RESUME_DIR" +if ! git diff --quiet 2>/dev/null; then + git add -A + git commit -m "kpi: auto-update metrics $(date +%Y-%m-%d)" 2>/dev/null + git push 2>/dev/null + ok "Resume repo updated and pushed" +else + ok "Resume metrics already current" +fi diff --git a/scripts/setup-slack.sh b/scripts/setup-slack.sh new file mode 100755 index 0000000..529262a --- /dev/null +++ b/scripts/setup-slack.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Set up Slack webhooks for BlackRoad OS notifications +# Creates ~/.blackroad/slack-webhook.env with webhook URLs + +source "$(dirname "$0")/../lib/common.sh" + +ENV_FILE="$HOME/.blackroad/slack-webhook.env" +mkdir -p "$(dirname "$ENV_FILE")" + +echo -e "${PINK}BlackRoad OS — Slack Setup${RESET}" +echo +echo "This script configures Slack webhooks for:" +echo " • #kpis — daily KPI reports + weekly digests" +echo " • #alerts — fleet alerts + deploy status (optional)" +echo +echo -e "${AMBER}Setup Instructions:${RESET}" +echo " 1. Go to https://api.slack.com/apps → Create New App → From scratch" +echo " App name: BlackRoad OS, Workspace: BlackRoad OS Inc" +echo +echo " 2. In your app → Incoming Webhooks → Activate" +echo +echo " 3. Add New Webhook to Workspace → select #kpis channel" +echo " Copy the webhook URL" +echo +echo " 4. (Optional) Add another webhook → select #alerts channel" +echo + +# Check existing config +if [ -f "$ENV_FILE" ]; then + echo -e "${BLUE}Current config:${RESET}" + grep -v "^#" "$ENV_FILE" | grep -v "^$" | sed 's/=.*/=***/' + echo +fi + +echo -e "${GREEN}Enter your webhook URLs (or press Enter to skip):${RESET}" +echo + +read -rp " #kpis webhook URL: " kpi_url +read -rp " #alerts webhook URL (optional): " alert_url + +if [ -z "$kpi_url" ] && [ -z "$alert_url" ]; then + echo + err "No URLs provided. Run this script again when you have them." + exit 1 +fi + +# Write env file +cat > "$ENV_FILE" << EOF +# BlackRoad OS Slack Webhooks +# Generated $(date -u +%Y-%m-%dT%H:%M:%SZ) + +# Daily KPI reports, weekly digests, deploy notifications +SLACK_WEBHOOK_URL=${kpi_url:-https://hooks.slack.com/services/YOUR/WEBHOOK/URL} + +# Critical fleet alerts (falls back to SLACK_WEBHOOK_URL if not set) +SLACK_ALERTS_WEBHOOK_URL=${alert_url:-} +EOF + +chmod 600 "$ENV_FILE" + +echo +ok "Config saved to $ENV_FILE (chmod 600)" + +# Test the webhook +if [ -n "$kpi_url" ] && ! echo "$kpi_url" | grep -q "YOUR"; then + echo + read -rp " Send test message? [y/N] " test + if [[ "$test" =~ ^[yY] ]]; then + source "$ENV_FILE" + source "$(dirname "$0")/../lib/slack.sh" + slack_load + + if slack_notify ":white_check_mark:" "BlackRoad OS Connected" \ + "Slack integration is live. Daily KPIs at 6:05am, alerts every 30min."; then + ok "Test message sent!" + else + err "Test message failed — check your webhook URL" + fi + fi +fi + +echo +echo -e "${BLUE}Notification schedule:${RESET}" +echo " • Daily report: 6:05 AM (slack-notify.sh)" +echo " • Fleet alerts: every 30 min (slack-alert.sh)" +echo " • Git patrol: every 2 hours (git-agent patrol)" +echo " • Deploy status: after each deploy (git-agent deploy)" +echo " • Weekly digest: Sunday 8 PM (slack-weekly.sh)" +echo +echo -e "Run ${GREEN}slack-alert.sh \"test message\"${RESET} to send a custom alert"