Initial monorepo — everything BlackRoad in one place
bin/ 230 CLI tools (ask-*, br-*, agent-*, roadid, carpool) scripts/ 99 automation scripts fleet/ Node configs and deployment workers/ Cloudflare Worker sources (roadpay, road-search, squad webhooks) roadc/ RoadC programming language roadnet/ Mesh network (5 APs, WireGuard) operator/ Memory system scripts config/ System configs dotfiles/ Shell configs docs/ Documentation BlackRoad OS — Pave Tomorrow. RoadChain-SHA2048: d1a24f55318d338b RoadChain-Identity: alexa@sovereign RoadChain-Full: d1a24f55318d338b24b60bad7be39286379c76ae5470817482100cb0ddbbcb97e147d07ac7243da0a9f0363e4e5c833d612b9c0df3a3cd20802465420278ef74875a5b77f55af6fe42a931b8b635b3d0d0b6bde9abf33dc42eea52bc03c951406d8cbe49f1a3d29b26a94dade05e9477f34a7d4d4c6ec4005c3c2ac54e73a68440c512c8e83fd9b1fe234750b898ef8f4032c23db173961fe225e67a0432b5293a9714f76c5c57ed5fdf35b9fb40fd73c03ebf88b7253c6a0575f5afb6a6b49b3bda310602fb1ef676859962dad2aebbb2875814b30eee0a8ba195e482d4cbc91d8819e7f38f6db53e8063401649c77bb994371473cabfb917fb53e8cbe73d60
This commit is contained in:
225
bin/slack-alert
Executable file
225
bin/slack-alert
Executable file
@@ -0,0 +1,225 @@
|
||||
#!/bin/bash
|
||||
# Real-time Slack alerts for fleet/service issues
|
||||
# Posts to SLACK_ALERTS_WEBHOOK_URL (or falls back to SLACK_WEBHOOK_URL)
|
||||
#
|
||||
# Usage: slack-alert.sh — auto-detect issues from latest KPI data
|
||||
# slack-alert.sh "message" — post a custom alert
|
||||
# slack-alert.sh git-patrol — post git-agent patrol results
|
||||
|
||||
source "$(dirname "$0")/../lib/common.sh"
|
||||
source "$(dirname "$0")/../lib/slack.sh"
|
||||
|
||||
slack_load
|
||||
|
||||
if ! slack_ready; then
|
||||
err "Slack not configured. Run: bash scripts/setup-slack.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─── Custom message mode ─────────────────────────────────────────────
|
||||
if [ -n "${1:-}" ] && [ "$1" != "git-patrol" ]; then
|
||||
payload=$(python3 -c "
|
||||
import json, sys
|
||||
msg = ' '.join(sys.argv[1:])
|
||||
blocks = [
|
||||
{'type': 'section', 'text': {'type': 'mrkdwn', 'text': f':rotating_light: *BlackRoad Alert*\n{msg}'}},
|
||||
{'type': 'context', 'elements': [{'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | slack-alert.sh'}]}
|
||||
]
|
||||
print(json.dumps({'blocks': blocks}))
|
||||
" "$@")
|
||||
slack_alert "$payload"
|
||||
ok "Alert posted: $*"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ─── Git patrol mode ─────────────────────────────────────────────────
|
||||
if [ "${1:-}" = "git-patrol" ]; then
|
||||
AGENT_SCRIPT="$(dirname "$0")/../agents/git-agent.sh"
|
||||
if [ ! -x "$AGENT_SCRIPT" ]; then
|
||||
err "git-agent.sh not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
patrol_output=$(bash "$AGENT_SCRIPT" health 2>&1)
|
||||
fleet_output=$(bash "$AGENT_SCRIPT" fleet status 2>&1)
|
||||
|
||||
payload=$(python3 -c "
|
||||
import json, sys, re
|
||||
|
||||
patrol = '''$patrol_output'''
|
||||
fleet = '''$fleet_output'''
|
||||
|
||||
# Parse health output
|
||||
issues = []
|
||||
for line in patrol.split('\n'):
|
||||
if '✗' in line:
|
||||
# Strip ANSI codes
|
||||
clean = re.sub(r'\033\[[0-9;]*m', '', line).strip()
|
||||
if clean:
|
||||
issues.append(clean.lstrip('✗ '))
|
||||
|
||||
fleet_lines = []
|
||||
for line in fleet.split('\n'):
|
||||
clean = re.sub(r'\033\[[0-9;]*m', '', line).strip()
|
||||
if 'repos=' in clean:
|
||||
fleet_lines.append(clean.lstrip('✓ '))
|
||||
|
||||
health_text = '\n'.join(f'• {i}' for i in issues) if issues else ':white_check_mark: All repos clean'
|
||||
fleet_text = '\n'.join(f'• {l}' for l in fleet_lines) if fleet_lines else 'No fleet data'
|
||||
|
||||
blocks = [
|
||||
{'type': 'header', 'text': {'type': 'plain_text', 'text': 'Git Agent Patrol Report'}},
|
||||
{'type': 'section', 'fields': [
|
||||
{'type': 'mrkdwn', 'text': f':mag: *Local Repos*\n{health_text}'},
|
||||
{'type': 'mrkdwn', 'text': f':satellite: *Fleet Repos*\n{fleet_text}'},
|
||||
]},
|
||||
{'type': 'context', 'elements': [
|
||||
{'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | git-agent patrol'}
|
||||
]}
|
||||
]
|
||||
print(json.dumps({'blocks': blocks}))
|
||||
")
|
||||
|
||||
slack_alert "$payload"
|
||||
ok "Git patrol posted to Slack"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ─── Auto-detect mode — scan latest KPIs for alertable issues ────────
|
||||
DAILY=$(today_file)
|
||||
[ ! -f "$DAILY" ] && { err "No daily data"; exit 1; }
|
||||
|
||||
export DAILY
|
||||
alerts=$(python3 << 'PYEOF'
|
||||
import json, os
|
||||
|
||||
with open(os.environ['DAILY']) as f:
|
||||
s = json.load(f).get('summary', {})
|
||||
|
||||
alerts = []
|
||||
|
||||
# Fleet nodes down
|
||||
offline = s.get('fleet_offline', [])
|
||||
if offline:
|
||||
alerts.append({
|
||||
'severity': 'critical',
|
||||
'emoji': ':rotating_light:',
|
||||
'text': f"*Nodes offline*: {', '.join(offline)}"
|
||||
})
|
||||
|
||||
# Fleet degraded
|
||||
online = s.get('fleet_online', 0)
|
||||
total = s.get('fleet_total', 4)
|
||||
if online < total and not offline:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':large_yellow_circle:',
|
||||
'text': f"*Fleet degraded*: {online}/{total} online"
|
||||
})
|
||||
|
||||
# Failed systemd units
|
||||
failed = s.get('failed_units', 0)
|
||||
if failed > 0:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':warning:',
|
||||
'text': f"*{failed} failed systemd units*"
|
||||
})
|
||||
|
||||
# Throttled nodes (undervoltage/thermal)
|
||||
throttled = s.get('throttled_nodes', [])
|
||||
if throttled:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':zap:',
|
||||
'text': f"*Throttled nodes*: {', '.join(throttled)}"
|
||||
})
|
||||
|
||||
# High temperature
|
||||
temp = s.get('avg_temp_c', 0)
|
||||
if temp > 70:
|
||||
alerts.append({
|
||||
'severity': 'critical' if temp > 80 else 'warning',
|
||||
'emoji': ':fire:',
|
||||
'text': f"*High fleet temp*: {temp:.1f}C avg"
|
||||
})
|
||||
|
||||
# Disk pressure (fleet)
|
||||
disk_used = s.get('fleet_disk_used_gb', 0)
|
||||
disk_total = s.get('fleet_disk_total_gb', 1)
|
||||
if disk_total > 0 and (disk_used / disk_total) > 0.85:
|
||||
pct = round(disk_used / disk_total * 100)
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':floppy_disk:',
|
||||
'text': f"*Fleet disk {pct}%*: {disk_used}/{disk_total} GB"
|
||||
})
|
||||
|
||||
# Mac disk pressure
|
||||
mac_pct = s.get('mac_disk_pct', 0)
|
||||
if mac_pct > 85:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':computer:',
|
||||
'text': f"*Mac disk at {mac_pct}%*"
|
||||
})
|
||||
|
||||
# Low autonomy score
|
||||
score = s.get('autonomy_score', 0)
|
||||
if score < 30:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':robot_face:',
|
||||
'text': f"*Low autonomy score*: {score}/100"
|
||||
})
|
||||
|
||||
# Too many service restarts (possible crash loop)
|
||||
restarts = s.get('service_restarts_today', 0)
|
||||
if restarts > 100:
|
||||
alerts.append({
|
||||
'severity': 'warning',
|
||||
'emoji': ':repeat:',
|
||||
'text': f"*{restarts} service restarts today* — possible crash loop"
|
||||
})
|
||||
|
||||
print(json.dumps(alerts))
|
||||
PYEOF
|
||||
)
|
||||
|
||||
if [ "$alerts" = "[]" ]; then
|
||||
log "No alerts to send"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Build and send alert payload
|
||||
payload=$(python3 -c "
|
||||
import json
|
||||
|
||||
alerts = json.loads('''$alerts''')
|
||||
|
||||
text_lines = []
|
||||
for a in alerts:
|
||||
key = a['text']
|
||||
text_lines.append(f\"{a['emoji']} {a['text']}\")
|
||||
|
||||
severity = 'critical' if any(a['severity'] == 'critical' for a in alerts) else 'warning'
|
||||
header_emoji = ':rotating_light:' if severity == 'critical' else ':warning:'
|
||||
|
||||
blocks = [
|
||||
{'type': 'header', 'text': {'type': 'plain_text', 'text': f'{header_emoji} BlackRoad Fleet Alert'}},
|
||||
{'type': 'section', 'text': {'type': 'mrkdwn', 'text': chr(10).join(text_lines)}},
|
||||
{'type': 'context', 'elements': [
|
||||
{'type': 'mrkdwn', 'text': '$(date -u +%Y-%m-%dT%H:%M:%SZ) | slack-alert.sh auto-detect'}
|
||||
]}
|
||||
]
|
||||
print(json.dumps({'blocks': blocks}))
|
||||
")
|
||||
|
||||
# Check dedup for the overall alert set
|
||||
alert_key=$(echo "$alerts" | md5 2>/dev/null || echo "$alerts" | md5sum | cut -d' ' -f1)
|
||||
if slack_dedup "$alert_key"; then
|
||||
slack_alert "$payload"
|
||||
ok "Alert posted to Slack ($(echo "$alerts" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))') issues)"
|
||||
else
|
||||
log "Alert suppressed (already sent within 1 hour)"
|
||||
fi
|
||||
Reference in New Issue
Block a user