From ace988587c708e22ff77a81d37df8de8e8806f0c Mon Sep 17 00:00:00 2001 From: Alexa Amundson Date: Mon, 16 Mar 2026 14:00:02 -0500 Subject: [PATCH] =?UTF-8?q?sync:=202026-03-16=2014:00=20=E2=80=94=2026=20f?= =?UTF-8?q?iles=20from=20Alexandria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RoadChain-SHA2048: 4aa2baf238024b57 RoadChain-Identity: alexa@sovereign RoadChain-Full: 4aa2baf238024b57b5f97e1fc31d99a3d06cfcc89db1b65a250fc32fa730d3afa0285c603566aa3f33aac0002106f8ab433756c110d4e34669375312d27b7df5c322e705b22bdd1bf2e73621c88acb72005cf0e01c0fe43ebda842cd3ed46b38cbe2630d19265f446313fda488510c91b7e232f7d3df04730afbeae331e23a20cec0f3a21ea0a85a16e6468db8dba9e7dfeface2b9d8a6298655fe1704f4275724d039f0dd8c46828faacd451128832415dd7cc14e9239d9db1128ec560e5fb738e186a55c39304a9b362860038cff76345a238581780ff574f1eb8904250965adc0486318106cbfd890433d1787a4a98270eec49ca0341681a8f13dba86bd43 --- dotfiles/brew-formulae.txt | 2 + fleet/alice/autonomy-log.txt | 100 ++++++++++++------------- fleet/alice/heartbeat.json | 2 +- fleet/alice/ports.txt | 2 +- fleet/alice/services.txt | 43 +++++++++++ fleet/alice/system-info.json | 18 ++--- fleet/anastasia/autonomy-log.txt | 1 + fleet/anastasia/ports.txt | 18 +++++ fleet/anastasia/services.txt | 28 +++++++ fleet/anastasia/system-info.json | 22 ++++++ fleet/aria/status.json | 2 +- fleet/cecilia/autonomy-log.txt | 50 ------------- fleet/cecilia/ports.txt | 34 --------- fleet/cecilia/services.txt | 39 ---------- fleet/cecilia/status.json | 2 +- fleet/cecilia/system-info.json | 38 ---------- fleet/gematria/ports.txt | 8 +- fleet/gematria/services.txt | 1 - fleet/gematria/system-info.json | 16 ++-- fleet/lucidia/autonomy-log.txt | 100 ++++++++++++------------- fleet/lucidia/docker.txt | 28 +++---- fleet/lucidia/heartbeat.json | 2 +- fleet/lucidia/ports.txt | 1 + fleet/lucidia/system-info.json | 18 ++--- fleet/octavia/status.json | 2 +- scripts/blackroad-fleet-coordinator.sh | 25 +++++++ 26 files changed, 291 insertions(+), 311 deletions(-) diff --git a/dotfiles/brew-formulae.txt b/dotfiles/brew-formulae.txt index f32944a..8699726 100644 --- a/dotfiles/brew-formulae.txt +++ b/dotfiles/brew-formulae.txt @@ -178,6 +178,7 @@ opus opusfile osx-cpu-temp p11-kit +p7zip pango pcre2 perl @@ -227,6 +228,7 @@ tesseract theora tmux tree +unar unbound utf8proc uv diff --git a/fleet/alice/autonomy-log.txt b/fleet/alice/autonomy-log.txt index a71172b..095b69c 100644 --- a/fleet/alice/autonomy-log.txt +++ b/fleet/alice/autonomy-log.txt @@ -1,50 +1,50 @@ -[2026-03-16 00:15:01] [HEAL] [alice] Healed 1 services -[2026-03-16 00:15:12] [BEAT] [alice] load=1.44 mem=3295/3794MB temp=36.5C disk=81% -[2026-03-16 00:15:13] [BEAT] [alice] load=1.44 mem=3294/3794MB temp=35.5C disk=81% -[2026-03-16 00:15:46] [DIAL] [alice] Switchboard unreachable -[2026-03-16 00:16:02] [BEAT] [alice] load=1.19 mem=3303/3794MB temp=34.1C disk=81% -[2026-03-16 00:16:11] [DIAL] [alice] Switchboard unreachable -[2026-03-16 00:17:01] [BEAT] [alice] load=0.84 mem=3301/3794MB temp=34.6C disk=81% -[2026-03-16 00:18:01] [BEAT] [alice] load=1.19 mem=3307/3794MB temp=33.6C disk=81% -[2026-03-16 00:19:01] [BEAT] [alice] load=0.58 mem=3302/3794MB temp=34.6C disk=81% -[2026-03-16 00:20:01] [FLEET] [alice] Starting cross-node health check -[2026-03-16 00:20:01] [HEAL] [alice] Service blackroad-agent is DOWN — restarting -[2026-03-16 00:20:01] [BEAT] [alice] load=1.44 mem=3296/3794MB temp=34.6C disk=81% -[2026-03-16 00:20:02] [HEAL] [alice] Service blackroad-agent restarted successfully -[2026-03-16 00:20:03] [HEAL] [alice] Healed 1 services -[2026-03-16 00:20:03] [FLEET] [alice] octavia: UP temp=36C mem=5696MB disk=68% -[2026-03-16 00:20:04] [FLEET] [alice] cecilia: UP temp=58C mem=3795MB disk=20% -[2026-03-16 00:20:06] [FLEET] [alice] gematria: UP temp=C mem=4186MB disk=67% -[2026-03-16 00:20:07] [FLEET] [alice] lucidia: UP temp=68C mem=2467MB disk=33% -[2026-03-16 00:20:09] [FLEET] [alice] aria: DOWN (no ping response) -[2026-03-16 00:20:10] [FLEET] [alice] anastasia: UP temp=C mem=280MB disk=69% -[2026-03-16 00:20:23] [BEAT] [alice] load=1.03 mem=3306/3794MB temp=35.0C disk=81% -[2026-03-16 00:20:24] [BEAT] [alice] load=1.03 mem=3305/3794MB temp=34.6C disk=81% -[2026-03-16 00:21:01] [BEAT] [alice] load=0.73 mem=3300/3794MB temp=34.1C disk=81% -[2026-03-16 00:22:01] [BEAT] [alice] load=0.76 mem=3306/3794MB temp=33.1C disk=81% -[2026-03-16 00:23:01] [BEAT] [alice] load=1.22 mem=3292/3794MB temp=34.1C disk=81% -[2026-03-16 00:24:01] [BEAT] [alice] load=1.69 mem=3308/3794MB temp=34.1C disk=81% -[2026-03-16 00:25:02] [HEAL] [alice] Service blackroad-agent is DOWN — restarting -[2026-03-16 00:25:02] [BEAT] [alice] load=1.85 mem=3296/3794MB temp=36.5C disk=81% -[2026-03-16 00:25:02] [HEAL] [alice] Service blackroad-agent restarted successfully -[2026-03-16 00:25:02] [HEAL] [alice] Healed 1 services -[2026-03-16 00:25:34] [BEAT] [alice] load=1.46 mem=3304/3794MB temp=34.6C disk=81% -[2026-03-16 00:25:34] [BEAT] [alice] load=1.46 mem=3303/3794MB temp=34.1C disk=81% -[2026-03-16 00:26:01] [BEAT] [alice] load=1.10 mem=3306/3794MB temp=34.6C disk=81% -[2026-03-16 00:27:01] [BEAT] [alice] load=0.78 mem=3304/3794MB temp=35.5C disk=81% -[2026-03-16 00:28:01] [BEAT] [alice] load=0.78 mem=3303/3794MB temp=34.6C disk=81% -[2026-03-16 00:29:01] [BEAT] [alice] load=0.76 mem=3304/3794MB temp=35.0C disk=81% -[2026-03-16 00:30:01] [FLEET] [alice] Starting cross-node health check -[2026-03-16 00:30:01] [HEAL] [alice] Service blackroad-agent is DOWN — restarting -[2026-03-16 00:30:01] [BEAT] [alice] load=1.26 mem=3296/3794MB temp=33.6C disk=81% -[2026-03-16 00:30:01] [HEAL] [alice] Service blackroad-agent restarted successfully -[2026-03-16 00:30:02] [FLEET] [alice] octavia: UP temp=45C mem=5669MB disk=68% -[2026-03-16 00:30:02] [HEAL] [alice] Healed 1 services -[2026-03-16 00:30:05] [FLEET] [alice] cecilia: UP temp=58C mem=3632MB disk=21% -[2026-03-16 00:30:06] [FLEET] [alice] gematria: UP temp=C mem=4192MB disk=67% -[2026-03-16 00:30:07] [FLEET] [alice] lucidia: UP temp=68C mem=2508MB disk=33% -[2026-03-16 00:30:09] [FLEET] [alice] aria: DOWN (no ping response) -[2026-03-16 00:30:10] [FLEET] [alice] anastasia: UP temp=C mem=279MB disk=69% -[2026-03-16 00:30:44] [BEAT] [alice] load=0.94 mem=3296/3794MB temp=35.0C disk=81% -[2026-03-16 00:30:44] [BEAT] [alice] load=0.94 mem=3294/3794MB temp=35.5C disk=81% -[2026-03-16 00:30:47] [DIAL] [alice] Switchboard unreachable +[2026-03-16 13:40:01] [HEAL] [alice] Service blackroad-agent restarted successfully +[2026-03-16 13:40:02] [FLEET] [alice] octavia: UP temp=37C mem=6009MB disk=68% +[2026-03-16 13:40:02] [HEAL] [alice] Healed 1 services +[2026-03-16 13:40:04] [FLEET] [alice] cecilia: DOWN (no ping response) +[2026-03-16 13:40:07] [FLEET] [alice] gematria: UP temp=C mem=4166MB disk=68% +[2026-03-16 13:40:50] [FLEET] [alice] lucidia: UP temp=55C mem=1189MB disk=33% +[2026-03-16 13:40:52] [FLEET] [alice] aria: DOWN (no ping response) +[2026-03-16 13:40:54] [FLEET] [alice] anastasia: UP temp=C mem=346MB disk=69% +[2026-03-16 13:41:01] [BEAT] [alice] load=0.71 mem=3278/3794MB temp=34.1C disk=83% +[2026-03-16 13:42:01] [BEAT] [alice] load=0.91 mem=3282/3794MB temp=34.1C disk=83% +[2026-03-16 13:43:01] [BEAT] [alice] load=0.71 mem=3280/3794MB temp=33.1C disk=83% +[2026-03-16 13:44:01] [BEAT] [alice] load=5.22 mem=3283/3794MB temp=34.1C disk=83% +[2026-03-16 13:45:01] [HEAL] [alice] Service blackroad-agent is DOWN — restarting +[2026-03-16 13:45:02] [BEAT] [alice] load=2.03 mem=3266/3794MB temp=35.5C disk=83% +[2026-03-16 13:45:02] [HEAL] [alice] Service blackroad-agent restarted successfully +[2026-03-16 13:45:02] [HEAL] [alice] Healed 1 services +[2026-03-16 13:45:47] [DIAL] [alice] Switchboard unreachable +[2026-03-16 13:46:02] [BEAT] [alice] load=1.55 mem=3278/3794MB temp=34.1C disk=83% +[2026-03-16 13:46:34] [DIAL] [alice] Switchboard unreachable +[2026-03-16 13:47:01] [BEAT] [alice] load=0.98 mem=3280/3794MB temp=33.6C disk=83% +[2026-03-16 13:48:01] [BEAT] [alice] load=0.96 mem=3280/3794MB temp=32.6C disk=83% +[2026-03-16 13:48:17] [BEAT] [alice] load=0.75 mem=3279/3794MB temp=34.1C disk=83% +[2026-03-16 13:48:17] [BEAT] [alice] load=0.75 mem=3279/3794MB temp=33.6C disk=83% +[2026-03-16 13:49:01] [BEAT] [alice] load=0.75 mem=3278/3794MB temp=32.6C disk=83% +[2026-03-16 13:50:01] [FLEET] [alice] Starting cross-node health check +[2026-03-16 13:50:01] [HEAL] [alice] Service blackroad-agent is DOWN — restarting +[2026-03-16 13:50:01] [BEAT] [alice] load=0.82 mem=3270/3794MB temp=35.0C disk=83% +[2026-03-16 13:50:01] [HEAL] [alice] Service blackroad-agent restarted successfully +[2026-03-16 13:50:02] [FLEET] [alice] octavia: UP temp=39C mem=6044MB disk=68% +[2026-03-16 13:50:02] [HEAL] [alice] Healed 1 services +[2026-03-16 13:50:04] [FLEET] [alice] cecilia: DOWN (no ping response) +[2026-03-16 13:50:07] [FLEET] [alice] gematria: UP temp=C mem=4167MB disk=68% +[2026-03-16 13:50:08] [FLEET] [alice] lucidia: UP temp=60C mem=1175MB disk=33% +[2026-03-16 13:50:10] [FLEET] [alice] aria: DOWN (no ping response) +[2026-03-16 13:50:11] [FLEET] [alice] anastasia: UP temp=C mem=339MB disk=69% +[2026-03-16 13:51:01] [BEAT] [alice] load=0.67 mem=3275/3794MB temp=33.1C disk=83% +[2026-03-16 13:52:01] [BEAT] [alice] load=0.58 mem=3276/3794MB temp=34.1C disk=83% +[2026-03-16 13:53:02] [BEAT] [alice] load=0.71 mem=3278/3794MB temp=32.6C disk=83% +[2026-03-16 13:53:26] [BEAT] [alice] load=0.95 mem=3274/3794MB temp=35.0C disk=83% +[2026-03-16 13:53:26] [BEAT] [alice] load=0.95 mem=3275/3794MB temp=34.6C disk=83% +[2026-03-16 13:54:01] [BEAT] [alice] load=0.98 mem=3278/3794MB temp=33.1C disk=83% +[2026-03-16 13:55:01] [HEAL] [alice] Service blackroad-agent is DOWN — restarting +[2026-03-16 13:55:01] [BEAT] [alice] load=0.95 mem=3277/3794MB temp=34.1C disk=83% +[2026-03-16 13:55:01] [HEAL] [alice] Service blackroad-agent restarted successfully +[2026-03-16 13:55:01] [HEAL] [alice] Healed 1 services +[2026-03-16 13:56:01] [BEAT] [alice] load=0.85 mem=3279/3794MB temp=31.2C disk=83% +[2026-03-16 13:57:01] [BEAT] [alice] load=1.03 mem=3278/3794MB temp=32.6C disk=83% +[2026-03-16 13:58:01] [BEAT] [alice] load=1.03 mem=3279/3794MB temp=32.1C disk=83% +[2026-03-16 13:58:35] [BEAT] [alice] load=0.65 mem=3277/3794MB temp=31.6C disk=83% +[2026-03-16 13:58:35] [BEAT] [alice] load=0.65 mem=3277/3794MB temp=32.1C disk=83% diff --git a/fleet/alice/heartbeat.json b/fleet/alice/heartbeat.json index f87247f..33608ae 100644 --- a/fleet/alice/heartbeat.json +++ b/fleet/alice/heartbeat.json @@ -1 +1 @@ -{"node":"alice","ts":"2026-03-16T07:31:43Z","load":0.76,"mem_free_mb":3298,"mem_total_mb":3794,"temp_c":34.1,"disk_pct":81,"throttle":"0x0"} +{"node":"alice","ts":"2026-03-16T18:58:35Z","load":0.65,"mem_free_mb":3277,"mem_total_mb":3794,"temp_c":32.1,"disk_pct":83,"throttle":"0x0"} diff --git a/fleet/alice/ports.txt b/fleet/alice/ports.txt index 6af3286..27a2324 100644 --- a/fleet/alice/ports.txt +++ b/fleet/alice/ports.txt @@ -11,7 +11,7 @@ LISTEN 0 5 0.0.0.0:8013 0.0.0.0:* users:(("python3",pid LISTEN 0 5 0.0.0.0:8012 0.0.0.0:* LISTEN 0 5 0.0.0.0:8014 0.0.0.0:* LISTEN 0 2048 0.0.0.0:8001 0.0.0.0:* users:(("python3",pid=617,fd=6)) -LISTEN 0 5 0.0.0.0:7890 0.0.0.0:* users:(("python3",pid=2218,fd=5)) +LISTEN 0 5 0.0.0.0:7890 0.0.0.0:* users:(("python3",pid=26217,fd=7)) LISTEN 0 200 0.0.0.0:443 0.0.0.0:* LISTEN 0 1024 0.0.0.0:6333 0.0.0.0:* LISTEN 0 128 0.0.0.0:6334 0.0.0.0:* diff --git a/fleet/alice/services.txt b/fleet/alice/services.txt index e69de29..4c5e40e 100644 --- a/fleet/alice/services.txt +++ b/fleet/alice/services.txt @@ -0,0 +1,43 @@ +avahi-daemon.service +blackroad-agents-proxy.service +blackroad-agents.service +blackroad-nats-agent.service +blackroad-operator.service +blackroad-status.service +blackroad-task-queue-v2.service +blackroad-task-worker.service +bluetooth.service +cloudflared.service +containerd.service +cron.service +dbus.service +dhcpcd.service +docker.service +getty@tty1.service +hciuart.service +nginx.service +pi-fleet-dashboard.service +pihole-FTL.service +pm2-pi.service +polkit.service +postgresql@13-main.service +prism-agent.service +qdrant.service +redis-server.service +rng-tools-debian.service +road-phone.service +roadnet-failover.service +rsyslog.service +rtkit-daemon.service +serial-getty@ttyS0.service +ssh.service +stats-proxy.service +systemd-journald.service +systemd-logind.service +systemd-timesyncd.service +systemd-udevd.service +triggerhappy.service +user@1000.service +user@1001.service +user@1002.service +wpa_supplicant.service diff --git a/fleet/alice/system-info.json b/fleet/alice/system-info.json index b645095..2c390bd 100644 --- a/fleet/alice/system-info.json +++ b/fleet/alice/system-info.json @@ -1,19 +1,19 @@ { "hostname": "alice", - "ts": "2026-03-16T05:30:46Z", - "uptime_seconds": 9984, + "ts": "2026-03-16T18:58:37Z", + "uptime_seconds": 58455, "kernel": "6.1.21-v8+", - "temp_c": 36.5, + "temp_c": 32.1, "memory_mb": { "total": 3794, - "used": 398, - "free": 3295 + "used": 418, + "free": 3276 }, - "disk": "11G/15G (81%)", + "disk": "12G/15G (83%)", "load": [ - 1.03, - 1.05, - 1.08 + 0.65, + 0.92, + 1.06 ], "ollama_models": [ "qwen2.5:3b", diff --git a/fleet/anastasia/autonomy-log.txt b/fleet/anastasia/autonomy-log.txt index e69de29..5f579cf 100644 --- a/fleet/anastasia/autonomy-log.txt +++ b/fleet/anastasia/autonomy-log.txt @@ -0,0 +1 @@ +no autonomy log diff --git a/fleet/anastasia/ports.txt b/fleet/anastasia/ports.txt index e69de29..0a816bc 100644 --- a/fleet/anastasia/ports.txt +++ b/fleet/anastasia/ports.txt @@ -0,0 +1,18 @@ +LISTEN 0 4096 127.0.0.1:20241 0.0.0.0:* users:(("cloudflared",pid=4072379,fd=3)) +LISTEN 0 2048 0.0.0.0:8000 0.0.0.0:* users:(("uvicorn",pid=532097,fd=11)) +LISTEN 0 4096 127.0.0.1:9090 0.0.0.0:* users:(("headscale",pid=2341808,fd=13)) +LISTEN 0 4096 100.64.0.1:11434 0.0.0.0:* users:(("ollama",pid=2412347,fd=3)) +LISTEN 0 5 0.0.0.0:8888 0.0.0.0:* users:(("python3",pid=396502,fd=3)) +LISTEN 0 100 0.0.0.0:6379 0.0.0.0:* users:(("python3",pid=539748,fd=6)) +LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=991,fd=7)) +LISTEN 0 100 0.0.0.0:8766 0.0.0.0:* users:(("python3",pid=538117,fd=6)) +LISTEN 0 100 0.0.0.0:8765 0.0.0.0:* users:(("python3",pid=538116,fd=6)) +LISTEN 0 5 0.0.0.0:8787 0.0.0.0:* users:(("python3",pid=1122797,fd=3)) +LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("nginx",pid=3461172,fd=8),("nginx",pid=3461171,fd=8)) +LISTEN 0 4096 0.0.0.0:111 0.0.0.0:* users:(("rpcbind",pid=589,fd=4),("systemd",pid=1,fd=127)) +LISTEN 0 4096 *:8080 *:* users:(("headscale",pid=2341808,fd=12)) +LISTEN 0 511 *:3000 *:* users:(("node /srv/hello",pid=1895584,fd=19)) +LISTEN 0 511 *:3001 *:* users:(("node",pid=757,fd=21)) +LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=991,fd=8)) +LISTEN 0 511 [::]:80 [::]:* users:(("nginx",pid=3461172,fd=9),("nginx",pid=3461171,fd=9)) +LISTEN 0 4096 [::]:111 [::]:* users:(("rpcbind",pid=589,fd=6),("systemd",pid=1,fd=129)) diff --git a/fleet/anastasia/services.txt b/fleet/anastasia/services.txt index e69de29..9a0ec20 100644 --- a/fleet/anastasia/services.txt +++ b/fleet/anastasia/services.txt @@ -0,0 +1,28 @@ +auditd.service +blackroad-api.service +chronyd.service +cloudflared.service +containerd.service +crond.service +dbus-broker.service +docker.service +droplet-agent.service +fail2ban.service +firewalld.service +getty@tty1.service +gssproxy.service +headscale.service +lucidia-agent.service +NetworkManager.service +nginx.service +ollama.service +polkit.service +rpcbind.service +rsyslog.service +serial-getty@ttyS0.service +sshd.service +systemd-journald.service +systemd-logind.service +systemd-udevd.service +user@0.service +user@1003.service diff --git a/fleet/anastasia/system-info.json b/fleet/anastasia/system-info.json index e69de29..e1a63e3 100644 --- a/fleet/anastasia/system-info.json +++ b/fleet/anastasia/system-info.json @@ -0,0 +1,22 @@ +{ + "hostname": "anastasia", + "ts": "2026-03-16T18:58:36Z", + "uptime_seconds": 6829408, + "kernel": "5.14.0-651.el9.x86_64", + "temp_c": 0, + "memory_mb": { + "total": 765, + "used": 419, + "free": 345 + }, + "disk": "18G/25G (69%)", + "load": [ + 0.0, + 0.03, + 0.0 + ], + "ollama_models": [], + "throttle": "N/A", + "voltage": "N/A", + "services_running": 28 +} diff --git a/fleet/aria/status.json b/fleet/aria/status.json index 0c1c1c5..a9a3386 100644 --- a/fleet/aria/status.json +++ b/fleet/aria/status.json @@ -1 +1 @@ -{"node":"aria","status":"down","ts":"2026-03-16T07:31:43Z"} +{"node":"aria","status":"down","ts":"2026-03-16T18:58:35Z"} diff --git a/fleet/cecilia/autonomy-log.txt b/fleet/cecilia/autonomy-log.txt index 6971885..e69de29 100644 --- a/fleet/cecilia/autonomy-log.txt +++ b/fleet/cecilia/autonomy-log.txt @@ -1,50 +0,0 @@ -[2026-03-16 00:10:12] [FLEET] [cecilia] anastasia: UP temp=C mem=279MB disk=69% -[2026-03-16 00:11:01] [BEAT] [cecilia] load=4.05 mem=1029/8062MB temp=55.6C disk=20% -[2026-03-16 00:12:01] [BEAT] [cecilia] load=4.38 mem=1000/8062MB temp=56.8C disk=20% -[2026-03-16 00:13:01] [BEAT] [cecilia] load=4.63 mem=3933/8062MB temp=57.9C disk=20% -[2026-03-16 00:14:01] [BEAT] [cecilia] load=5.21 mem=3912/8062MB temp=55.1C disk=20% -[2026-03-16 00:15:01] [BEAT] [cecilia] load=5.66 mem=3912/8062MB temp=55.6C disk=20% -[2026-03-16 00:15:01] [HEAL] [cecilia] High swap: 585MB — clearing inactive -[2026-03-16 00:15:12] [BEAT] [cecilia] load=6.04 mem=3049/8062MB temp=56.2C disk=20% -[2026-03-16 00:15:13] [BEAT] [cecilia] load=6.04 mem=3049/8062MB temp=56.2C disk=20% -[2026-03-16 00:15:47] [DIAL] [cecilia] Switchboard unreachable -[2026-03-16 00:16:01] [BEAT] [cecilia] load=8.39 mem=3168/8062MB temp=57.3C disk=20% -[2026-03-16 00:17:01] [BEAT] [cecilia] load=9.39 mem=3132/8062MB temp=57.9C disk=20% -[2026-03-16 00:18:01] [BEAT] [cecilia] load=8.99 mem=3093/8062MB temp=57.9C disk=20% -[2026-03-16 00:19:01] [BEAT] [cecilia] load=6.60 mem=3836/8062MB temp=55.6C disk=20% -[2026-03-16 00:20:01] [FLEET] [cecilia] Starting cross-node health check -[2026-03-16 00:20:01] [BEAT] [cecilia] load=5.38 mem=3805/8062MB temp=55.6C disk=20% -[2026-03-16 00:20:01] [HEAL] [cecilia] High swap: 574MB — clearing inactive -[2026-03-16 00:20:04] [FLEET] [cecilia] alice: UP temp=34C mem=3297MB disk=81% -[2026-03-16 00:20:06] [FLEET] [cecilia] octavia: UP temp=36C mem=5681MB disk=68% -[2026-03-16 00:20:07] [DIAL] [cecilia] Switchboard unreachable -[2026-03-16 00:20:10] [FLEET] [cecilia] gematria: UP temp=C mem=4186MB disk=67% -[2026-03-16 00:20:13] [FLEET] [cecilia] lucidia: UP temp=69C mem=2496MB disk=33% -[2026-03-16 00:20:15] [FLEET] [cecilia] aria: DOWN (no ping response) -[2026-03-16 00:20:17] [FLEET] [cecilia] anastasia: UP temp=C mem=280MB disk=69% -[2026-03-16 00:20:24] [BEAT] [cecilia] load=5.16 mem=3838/8062MB temp=57.3C disk=21% -[2026-03-16 00:20:24] [BEAT] [cecilia] load=5.16 mem=3837/8062MB temp=57.9C disk=21% -[2026-03-16 00:21:01] [BEAT] [cecilia] load=5.22 mem=3872/8062MB temp=60.0C disk=21% -[2026-03-16 00:22:01] [BEAT] [cecilia] load=4.80 mem=6052/8062MB temp=58.4C disk=21% -[2026-03-16 00:23:01] [BEAT] [cecilia] load=5.02 mem=6048/8062MB temp=56.8C disk=21% -[2026-03-16 00:24:01] [BEAT] [cecilia] load=2.77 mem=6889/8062MB temp=46.3C disk=21% -[2026-03-16 00:25:01] [BEAT] [cecilia] load=3.56 mem=3907/8062MB temp=56.2C disk=21% -[2026-03-16 00:25:01] [HEAL] [cecilia] High swap: 565MB — clearing inactive -[2026-03-16 00:25:34] [BEAT] [cecilia] load=4.20 mem=3935/8062MB temp=56.8C disk=21% -[2026-03-16 00:25:34] [BEAT] [cecilia] load=4.20 mem=3933/8062MB temp=55.6C disk=21% -[2026-03-16 00:26:01] [BEAT] [cecilia] load=4.13 mem=3965/8062MB temp=56.2C disk=21% -[2026-03-16 00:27:01] [BEAT] [cecilia] load=4.26 mem=3956/8062MB temp=57.3C disk=21% -[2026-03-16 00:28:01] [BEAT] [cecilia] load=4.22 mem=885/8062MB temp=57.3C disk=21% -[2026-03-16 00:29:01] [BEAT] [cecilia] load=4.53 mem=903/8062MB temp=57.9C disk=21% -[2026-03-16 00:30:01] [FLEET] [cecilia] Starting cross-node health check -[2026-03-16 00:30:01] [BEAT] [cecilia] load=4.17 mem=3679/8062MB temp=57.3C disk=21% -[2026-03-16 00:30:01] [HEAL] [cecilia] High swap: 627MB — clearing inactive -[2026-03-16 00:30:05] [FLEET] [cecilia] alice: UP temp=35C mem=3300MB disk=81% -[2026-03-16 00:30:09] [FLEET] [cecilia] octavia: UP temp=46C mem=5704MB disk=68% -[2026-03-16 00:30:12] [FLEET] [cecilia] gematria: UP temp=C mem=4249MB disk=67% -[2026-03-16 00:30:14] [FLEET] [cecilia] lucidia: UP temp=69C mem=2599MB disk=33% -[2026-03-16 00:30:16] [FLEET] [cecilia] aria: DOWN (no ping response) -[2026-03-16 00:30:17] [FLEET] [cecilia] anastasia: UP temp=C mem=280MB disk=69% -[2026-03-16 00:30:44] [BEAT] [cecilia] load=4.55 mem=3735/8062MB temp=58.4C disk=21% -[2026-03-16 00:30:44] [BEAT] [cecilia] load=4.55 mem=3734/8062MB temp=58.4C disk=21% -[2026-03-16 00:30:47] [DIAL] [cecilia] Switchboard unreachable diff --git a/fleet/cecilia/ports.txt b/fleet/cecilia/ports.txt index 2a57647..e69de29 100644 --- a/fleet/cecilia/ports.txt +++ b/fleet/cecilia/ports.txt @@ -1,34 +0,0 @@ -LISTEN 0 4096 127.0.0.1:9000 0.0.0.0:* -LISTEN 0 128 0.0.0.0:34001 0.0.0.0:* -LISTEN 0 32 127.0.0.1:53 0.0.0.0:* -LISTEN 0 2048 0.0.0.0:8788 0.0.0.0:* users:(("python3",pid=1429,fd=16)) -LISTEN 0 5 0.0.0.0:8787 0.0.0.0:* users:(("python3",pid=1562,fd=3)) -LISTEN 0 5 0.0.0.0:9100 0.0.0.0:* -LISTEN 0 128 0.0.0.0:5002 0.0.0.0:* users:(("python3",pid=1566,fd=3)) -LISTEN 0 128 0.0.0.0:5001 0.0.0.0:* users:(("python3",pid=1572,fd=3)) -LISTEN 0 511 0.0.0.0:80 0.0.0.0:* -LISTEN 0 128 0.0.0.0:22 0.0.0.0:* -LISTEN 0 200 127.0.0.1:5432 0.0.0.0:* -LISTEN 0 32 192.168.4.96:53 0.0.0.0:* -LISTEN 0 5 0.0.0.0:7890 0.0.0.0:* users:(("python3",pid=222532,fd=5)) -LISTEN 0 5 0.0.0.0:4010 0.0.0.0:* users:(("python3",pid=1035,fd=3)) -LISTEN 0 511 0.0.0.0:8080 0.0.0.0:* -LISTEN 0 511 0.0.0.0:3100 0.0.0.0:* -LISTEN 0 5 0.0.0.0:11435 0.0.0.0:* users:(("python3",pid=1412,fd=3)) -LISTEN 0 4096 127.0.0.1:39001 0.0.0.0:* -LISTEN 0 4096 127.0.0.1:8088 0.0.0.0:* -LISTEN 0 4096 127.0.0.1:20241 0.0.0.0:* -LISTEN 0 5 0.0.0.0:3001 0.0.0.0:* -LISTEN 0 5 0.0.0.0:3000 0.0.0.0:* users:(("python3",pid=1563,fd=3)) -LISTEN 0 4096 127.0.0.1:36339 0.0.0.0:* -LISTEN 0 16 *:5900 *:* -LISTEN 0 4096 *:9001 *:* -LISTEN 0 4096 *:9000 *:* -LISTEN 0 511 [::]:80 [::]:* -LISTEN 0 128 [::]:22 [::]:* -LISTEN 0 4096 [::1]:9000 [::]:* -LISTEN 0 4096 *:8086 *:* -LISTEN 0 511 [::]:8080 [::]:* -LISTEN 0 4096 *:11434 *:* -LISTEN 0 32 [::1]:53 [::]:* -LISTEN 0 200 [::1]:5432 [::]:* diff --git a/fleet/cecilia/services.txt b/fleet/cecilia/services.txt index 9c35f53..e69de29 100644 --- a/fleet/cecilia/services.txt +++ b/fleet/cecilia/services.txt @@ -1,39 +0,0 @@ -accounts-daemon.service -avahi-daemon.service -blackroad-agent.service -blackroad-dashboard.service -blackroad-monitor.service -blackroad-nats-agent.service -bluetooth.service -cece-api.service -cece-heartbeat.service -cloudflared.service -containerd.service -cron.service -dbus.service -dnsmasq.service -docker.service -getty@tty1.service -hailort.service -influxdb.service -minio.service -ModemManager.service -NetworkManager.service -ollama.service -pironman5.service -polkit.service -postgresql@17-main.service -roadnet-failover.service -serial-getty@ttyAMA10.service -ssh.service -stats-proxy.service -systemd-journald.service -systemd-logind.service -systemd-timesyncd.service -systemd-udevd.service -udisks2.service -user@1000.service -user@1001.service -wayvnc-control.service -wayvnc.service -wpa_supplicant.service diff --git a/fleet/cecilia/status.json b/fleet/cecilia/status.json index f9094a2..90aa01a 100644 --- a/fleet/cecilia/status.json +++ b/fleet/cecilia/status.json @@ -1 +1 @@ -{"node":"cecilia","status":"down","ts":"2026-03-16T02:45:34Z"} +{"node":"cecilia","status":"down","ts":"2026-03-16T18:58:35Z"} diff --git a/fleet/cecilia/system-info.json b/fleet/cecilia/system-info.json index e4feef1..e69de29 100644 --- a/fleet/cecilia/system-info.json +++ b/fleet/cecilia/system-info.json @@ -1,38 +0,0 @@ -{ - "hostname": "cecilia", - "ts": "2026-03-16T05:30:46Z", - "uptime_seconds": 9756, - "kernel": "6.12.62+rpt-rpi-2712", - "temp_c": 58.4, - "memory_mb": { - "total": 8062, - "used": 4331, - "free": 3731 - }, - "disk": "90G/457G (21%)", - "load": [ - 4.55, - 4.49, - 4.32 - ], - "ollama_models": [ - "deepseek-r1:1.5b", - "nomic-embed-text:latest", - "hf.co/mradermacher/OpenELM-1_1B-Instruct-GGUF:Q4_K_M", - "hf.co/mradermacher/OpenELM-3B-Instruct-GGUF:Q4_K_M", - "cece2:latest", - "qwen3:8b", - "llama3:8b-instruct-q4_K_M", - "cece:latest", - "deepseek-coder:1.3b", - "qwen2.5-coder:3b", - "llama3.2:3b", - "tinyllama:latest", - "llama3.2:latest", - "codellama:7b", - "llama3.2:1b" - ], - "throttle": "0x50000", - "voltage": "0.8732V", - "services_running": 39 -} diff --git a/fleet/gematria/ports.txt b/fleet/gematria/ports.txt index 21f2ba6..b66daab 100644 --- a/fleet/gematria/ports.txt +++ b/fleet/gematria/ports.txt @@ -1,11 +1,13 @@ LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=682329,fd=14)) -LISTEN 0 4096 127.0.0.1:2019 0.0.0.0:* users:(("caddy",pid=26445,fd=13)) +LISTEN 0 4096 127.0.0.1:2019 0.0.0.0:* users:(("caddy",pid=26445,fd=7)) +LISTEN 0 5 0.0.0.0:11435 0.0.0.0:* users:(("python3",pid=147655,fd=3)) LISTEN 0 5 0.0.0.0:8787 0.0.0.0:* users:(("python3",pid=577215,fd=3)) LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=3335327,fd=3)) LISTEN 0 4096 127.0.0.1:11434 0.0.0.0:* users:(("ollama",pid=2498181,fd=3)) -LISTEN 0 4096 127.0.0.1:20241 0.0.0.0:* users:(("cloudflared",pid=24118,fd=3)) +LISTEN 0 4096 127.0.0.1:20241 0.0.0.0:* users:(("cloudflared",pid=156470,fd=3)) LISTEN 0 128 127.0.0.1:8011 0.0.0.0:* users:(("sshd",pid=4152649,fd=5)) -LISTEN 0 4096 *:443 *:* users:(("caddy",pid=26445,fd=4)) +LISTEN 0 4096 *:11436 *:* users:(("caddy",pid=26445,fd=10)) +LISTEN 0 4096 *:443 *:* users:(("caddy",pid=26445,fd=13)) LISTEN 0 4096 *:80 *:* users:(("caddy",pid=26445,fd=14)) LISTEN 0 4096 *:4222 *:* users:(("nats-server",pid=1353183,fd=6)) LISTEN 0 4096 *:8222 *:* users:(("nats-server",pid=1353183,fd=3)) diff --git a/fleet/gematria/services.txt b/fleet/gematria/services.txt index d2f5ec4..be6423c 100644 --- a/fleet/gematria/services.txt +++ b/fleet/gematria/services.txt @@ -12,7 +12,6 @@ lucidia-agent.service lucidia-watchdog.service multipathd.service networkd-dispatcher.service -nginx.service ollama.service packagekit.service polkit.service diff --git a/fleet/gematria/system-info.json b/fleet/gematria/system-info.json index b337e6b..af854d5 100644 --- a/fleet/gematria/system-info.json +++ b/fleet/gematria/system-info.json @@ -1,18 +1,18 @@ { "hostname": "gematria", - "ts": "2026-03-16T05:30:48Z", - "uptime_seconds": 5438240, + "ts": "2026-03-16T18:58:38Z", + "uptime_seconds": 5486710, "kernel": "5.15.0-113-generic", "temp_c": 0, "memory_mb": { "total": 7937, - "used": 3285, - "free": 4210 + "used": 3309, + "free": 4178 }, - "disk": "52G/78G (67%)", + "disk": "52G/78G (68%)", "load": [ - 3.18, - 3.09, + 3.11, + 3.1, 3.09 ], "ollama_models": [ @@ -26,5 +26,5 @@ ], "throttle": "N/A", "voltage": "N/A", - "services_running": 32 + "services_running": 31 } diff --git a/fleet/lucidia/autonomy-log.txt b/fleet/lucidia/autonomy-log.txt index bf5ddb9..7dd19c8 100644 --- a/fleet/lucidia/autonomy-log.txt +++ b/fleet/lucidia/autonomy-log.txt @@ -1,50 +1,50 @@ -[2026-03-16 00:10:13] [FLEET] [lucidia] anastasia: UP temp=C mem=279MB disk=69% -[2026-03-16 00:11:30] [BEAT] [lucidia] load=42.63 mem=2623/8063MB temp=65.0C disk=33% -[2026-03-16 00:12:01] [BEAT] [lucidia] load=29.47 mem=2591/8063MB temp=67.2C disk=33% -[2026-03-16 00:13:01] [BEAT] [lucidia] load=16.28 mem=2582/8063MB temp=68.3C disk=33% -[2026-03-16 00:14:01] [BEAT] [lucidia] load=11.28 mem=2645/8063MB temp=68.8C disk=33% -[2026-03-16 00:15:02] [BEAT] [lucidia] load=7.13 mem=2523/8063MB temp=67.2C disk=33% -[2026-03-16 00:15:02] [HEAL] [lucidia] High swap: 741MB — clearing inactive -[2026-03-16 00:15:12] [BEAT] [lucidia] load=7.04 mem=2510/8063MB temp=67.8C disk=33% -[2026-03-16 00:15:12] [BEAT] [lucidia] load=7.04 mem=2509/8063MB temp=68.3C disk=33% -[2026-03-16 00:15:48] [DIAL] [lucidia] Switchboard unreachable -[2026-03-16 00:16:01] [BEAT] [lucidia] load=5.89 mem=2536/8063MB temp=62.8C disk=33% -[2026-03-16 00:17:01] [BEAT] [lucidia] load=3.32 mem=2471/8063MB temp=62.8C disk=33% -[2026-03-16 00:18:01] [BEAT] [lucidia] load=2.02 mem=2476/8063MB temp=55.6C disk=33% -[2026-03-16 00:19:02] [BEAT] [lucidia] load=3.30 mem=2498/8063MB temp=65.0C disk=33% -[2026-03-16 00:19:11] [DIAL] [lucidia] Switchboard unreachable -[2026-03-16 00:20:01] [FLEET] [lucidia] Starting cross-node health check -[2026-03-16 00:20:01] [BEAT] [lucidia] load=5.14 mem=2559/8063MB temp=68.3C disk=33% -[2026-03-16 00:20:01] [HEAL] [lucidia] High swap: 730MB — clearing inactive -[2026-03-16 00:20:02] [FLEET] [lucidia] alice: UP temp=34C mem=3296MB disk=81% -[2026-03-16 00:20:04] [FLEET] [lucidia] octavia: DOWN (no ping response) -[2026-03-16 00:20:07] [FLEET] [lucidia] cecilia: UP temp=57C mem=3819MB disk=20% -[2026-03-16 00:20:09] [FLEET] [lucidia] gematria: UP temp=C mem=4186MB disk=67% -[2026-03-16 00:20:11] [FLEET] [lucidia] aria: DOWN (no ping response) -[2026-03-16 00:20:14] [FLEET] [lucidia] anastasia: UP temp=C mem=280MB disk=69% -[2026-03-16 00:20:23] [BEAT] [lucidia] load=5.37 mem=2511/8063MB temp=69.4C disk=33% -[2026-03-16 00:20:23] [BEAT] [lucidia] load=5.37 mem=2509/8063MB temp=70.0C disk=33% -[2026-03-16 00:21:14] [BEAT] [lucidia] load=17.93 mem=2511/8063MB temp=59.0C disk=33% -[2026-03-16 00:22:01] [BEAT] [lucidia] load=11.93 mem=2511/8063MB temp=66.1C disk=33% -[2026-03-16 00:23:01] [BEAT] [lucidia] load=6.69 mem=2547/8063MB temp=59.0C disk=33% -[2026-03-16 00:24:01] [BEAT] [lucidia] load=4.85 mem=2555/8063MB temp=57.3C disk=33% -[2026-03-16 00:25:01] [BEAT] [lucidia] load=3.84 mem=2526/8063MB temp=63.4C disk=33% -[2026-03-16 00:25:02] [HEAL] [lucidia] High swap: 727MB — clearing inactive -[2026-03-16 00:25:34] [BEAT] [lucidia] load=2.75 mem=2506/8063MB temp=56.8C disk=33% -[2026-03-16 00:25:34] [BEAT] [lucidia] load=2.75 mem=2506/8063MB temp=56.8C disk=33% -[2026-03-16 00:26:12] [BEAT] [lucidia] load=17.73 mem=2508/8063MB temp=55.6C disk=33% -[2026-03-16 00:27:01] [BEAT] [lucidia] load=10.20 mem=2507/8063MB temp=57.9C disk=33% -[2026-03-16 00:28:01] [BEAT] [lucidia] load=5.49 mem=2537/8063MB temp=62.8C disk=33% -[2026-03-16 00:29:01] [BEAT] [lucidia] load=5.46 mem=2534/8063MB temp=65.0C disk=33% -[2026-03-16 00:30:01] [FLEET] [lucidia] Starting cross-node health check -[2026-03-16 00:30:01] [BEAT] [lucidia] load=5.62 mem=2523/8063MB temp=68.8C disk=33% -[2026-03-16 00:30:02] [HEAL] [lucidia] High swap: 727MB — clearing inactive -[2026-03-16 00:30:03] [FLEET] [lucidia] alice: UP temp=34C mem=3292MB disk=81% -[2026-03-16 00:30:05] [FLEET] [lucidia] octavia: DOWN (no ping response) -[2026-03-16 00:30:09] [FLEET] [lucidia] cecilia: UP temp=58C mem=3670MB disk=21% -[2026-03-16 00:30:11] [FLEET] [lucidia] gematria: UP temp=C mem=4180MB disk=67% -[2026-03-16 00:30:13] [FLEET] [lucidia] aria: DOWN (no ping response) -[2026-03-16 00:30:14] [FLEET] [lucidia] anastasia: UP temp=C mem=279MB disk=69% -[2026-03-16 00:30:44] [BEAT] [lucidia] load=5.78 mem=2561/8063MB temp=68.3C disk=33% -[2026-03-16 00:30:44] [BEAT] [lucidia] load=5.78 mem=2559/8063MB temp=70.0C disk=33% -[2026-03-16 00:30:47] [DIAL] [lucidia] Switchboard unreachable +[2026-03-16 13:36:01] [BEAT] [lucidia] load=2.50 mem=1236/8063MB temp=54.0C disk=33% +[2026-03-16 13:37:01] [BEAT] [lucidia] load=2.20 mem=1197/8063MB temp=55.6C disk=33% +[2026-03-16 13:37:22] [DIAL] [lucidia] Switchboard unreachable +[2026-03-16 13:38:22] [BEAT] [lucidia] load=27.54 mem=1187/8063MB temp=55.1C disk=33% +[2026-03-16 13:39:01] [BEAT] [lucidia] load=15.46 mem=1182/8063MB temp=54.0C disk=33% +[2026-03-16 13:40:01] [FLEET] [lucidia] Starting cross-node health check +[2026-03-16 13:40:02] [BEAT] [lucidia] load=6.65 mem=1233/8063MB temp=54.0C disk=33% +[2026-03-16 13:40:02] [HEAL] [lucidia] High swap: 2163MB — clearing inactive +[2026-03-16 13:40:03] [FLEET] [lucidia] alice: UP temp=37C mem=3273MB disk=83% +[2026-03-16 13:40:05] [FLEET] [lucidia] octavia: DOWN (no ping response) +[2026-03-16 13:40:07] [FLEET] [lucidia] cecilia: DOWN (no ping response) +[2026-03-16 13:40:48] [FLEET] [lucidia] gematria: UP temp=C mem=4169MB disk=68% +[2026-03-16 13:40:50] [FLEET] [lucidia] aria: DOWN (no ping response) +[2026-03-16 13:40:52] [FLEET] [lucidia] anastasia: UP temp=C mem=346MB disk=69% +[2026-03-16 13:41:01] [BEAT] [lucidia] load=22.41 mem=1169/8063MB temp=60.0C disk=33% +[2026-03-16 13:42:01] [BEAT] [lucidia] load=9.64 mem=1177/8063MB temp=54.5C disk=33% +[2026-03-16 13:43:01] [BEAT] [lucidia] load=4.96 mem=1201/8063MB temp=53.5C disk=33% +[2026-03-16 13:44:01] [BEAT] [lucidia] load=4.00 mem=1157/8063MB temp=56.8C disk=33% +[2026-03-16 13:45:01] [BEAT] [lucidia] load=1.87 mem=1228/8063MB temp=58.4C disk=33% +[2026-03-16 13:45:01] [HEAL] [lucidia] High swap: 2163MB — clearing inactive +[2026-03-16 13:45:47] [DIAL] [lucidia] Switchboard unreachable +[2026-03-16 13:46:01] [BEAT] [lucidia] load=2.63 mem=1190/8063MB temp=55.1C disk=33% +[2026-03-16 13:47:01] [BEAT] [lucidia] load=2.79 mem=1166/8063MB temp=63.4C disk=33% +[2026-03-16 13:48:01] [BEAT] [lucidia] load=2.52 mem=1193/8063MB temp=55.6C disk=33% +[2026-03-16 13:48:16] [BEAT] [lucidia] load=3.29 mem=1203/8063MB temp=56.2C disk=33% +[2026-03-16 13:48:16] [BEAT] [lucidia] load=3.29 mem=1201/8063MB temp=56.8C disk=33% +[2026-03-16 13:49:01] [BEAT] [lucidia] load=3.75 mem=1161/8063MB temp=58.4C disk=33% +[2026-03-16 13:50:01] [FLEET] [lucidia] Starting cross-node health check +[2026-03-16 13:50:01] [BEAT] [lucidia] load=1.92 mem=1192/8063MB temp=59.0C disk=33% +[2026-03-16 13:50:02] [HEAL] [lucidia] High swap: 2163MB — clearing inactive +[2026-03-16 13:50:03] [FLEET] [lucidia] alice: UP temp=36C mem=3275MB disk=83% +[2026-03-16 13:50:05] [FLEET] [lucidia] octavia: DOWN (no ping response) +[2026-03-16 13:50:07] [FLEET] [lucidia] cecilia: DOWN (no ping response) +[2026-03-16 13:50:08] [FLEET] [lucidia] gematria: UP temp=C mem=4162MB disk=68% +[2026-03-16 13:50:10] [FLEET] [lucidia] aria: DOWN (no ping response) +[2026-03-16 13:50:11] [FLEET] [lucidia] anastasia: UP temp=C mem=339MB disk=69% +[2026-03-16 13:51:01] [BEAT] [lucidia] load=10.05 mem=1182/8063MB temp=55.1C disk=33% +[2026-03-16 13:52:01] [BEAT] [lucidia] load=4.88 mem=1223/8063MB temp=54.0C disk=33% +[2026-03-16 13:52:23] [DIAL] [lucidia] Switchboard unreachable +[2026-03-16 13:53:01] [BEAT] [lucidia] load=2.75 mem=1222/8063MB temp=59.0C disk=33% +[2026-03-16 13:53:25] [BEAT] [lucidia] load=2.86 mem=1185/8063MB temp=52.4C disk=33% +[2026-03-16 13:53:25] [BEAT] [lucidia] load=2.86 mem=1184/8063MB temp=51.2C disk=33% +[2026-03-16 13:54:02] [BEAT] [lucidia] load=1.73 mem=1239/8063MB temp=49.6C disk=33% +[2026-03-16 13:55:01] [BEAT] [lucidia] load=2.72 mem=1190/8063MB temp=49.6C disk=33% +[2026-03-16 13:55:01] [HEAL] [lucidia] High swap: 2162MB — clearing inactive +[2026-03-16 13:56:01] [BEAT] [lucidia] load=3.99 mem=1203/8063MB temp=55.1C disk=33% +[2026-03-16 13:57:01] [BEAT] [lucidia] load=3.13 mem=1229/8063MB temp=48.0C disk=33% +[2026-03-16 13:58:01] [BEAT] [lucidia] load=2.88 mem=1186/8063MB temp=48.0C disk=33% +[2026-03-16 13:58:34] [BEAT] [lucidia] load=3.09 mem=1197/8063MB temp=46.3C disk=33% +[2026-03-16 13:58:34] [BEAT] [lucidia] load=3.09 mem=1195/8063MB temp=47.4C disk=33% diff --git a/fleet/lucidia/docker.txt b/fleet/lucidia/docker.txt index eb91d18..0015c91 100644 --- a/fleet/lucidia/docker.txt +++ b/fleet/lucidia/docker.txt @@ -1,14 +1,14 @@ -blackroad-gitea Up 3 hours -road-pdns-admin Up 3 hours (healthy) -road-pdns Up 3 hours -road-dns-db Up 3 hours -roadauth Up 3 hours -roadapi Up 3 hours -blackroad-edge-agent Up 3 hours -blackroad.systems Up 3 hours -blackroadai.com Up 3 hours -blackroad-auth-gateway Up 3 hours -blackroad-metaverse Up 3 hours -blackroad-os Up 3 hours -blackroad-os-carpool Up 3 hours -pi-my-agent-1 Up 3 hours (healthy) +blackroad-gitea Up 16 hours +road-pdns-admin Up 16 hours (healthy) +road-pdns Up 16 hours +road-dns-db Up 16 hours +roadauth Up 16 hours +roadapi Up 16 hours +blackroad-edge-agent Up 16 hours +blackroad.systems Up 16 hours +blackroadai.com Up 16 hours +blackroad-auth-gateway Up 16 hours +blackroad-metaverse Up 16 hours +blackroad-os Up 16 hours +blackroad-os-carpool Up 16 hours +pi-my-agent-1 Up 16 hours (healthy) diff --git a/fleet/lucidia/heartbeat.json b/fleet/lucidia/heartbeat.json index 2486c6f..25b1820 100644 --- a/fleet/lucidia/heartbeat.json +++ b/fleet/lucidia/heartbeat.json @@ -1 +1 @@ -{"node":"lucidia","ts":"2026-03-16T07:31:43Z","load":1.97,"mem_free_mb":1703,"mem_total_mb":8063,"temp_c":57.3,"disk_pct":33,"throttle":"0x0"} +{"node":"lucidia","ts":"2026-03-16T18:58:34Z","load":3.09,"mem_free_mb":1195,"mem_total_mb":8063,"temp_c":47.4,"disk_pct":33,"throttle":"0x0"} diff --git a/fleet/lucidia/ports.txt b/fleet/lucidia/ports.txt index de607b0..17dff48 100644 --- a/fleet/lucidia/ports.txt +++ b/fleet/lucidia/ports.txt @@ -8,6 +8,7 @@ LISTEN 0 128 0.0.0.0:8889 0.0.0.0:* LISTEN 0 5 0.0.0.0:8787 0.0.0.0:* LISTEN 0 4096 0.0.0.0:9090 0.0.0.0:* LISTEN 0 4096 0.0.0.0:9192 0.0.0.0:* +LISTEN 0 4096 127.0.0.1:38955 0.0.0.0:* LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 511 0.0.0.0:80 0.0.0.0:* LISTEN 0 200 127.0.0.1:5432 0.0.0.0:* diff --git a/fleet/lucidia/system-info.json b/fleet/lucidia/system-info.json index 3e03a4f..ceee86f 100644 --- a/fleet/lucidia/system-info.json +++ b/fleet/lucidia/system-info.json @@ -1,19 +1,19 @@ { "hostname": "octavia", - "ts": "2026-03-16T05:30:46Z", - "uptime_seconds": 9804, + "ts": "2026-03-16T18:58:35Z", + "uptime_seconds": 58273, "kernel": "6.12.62+rpt-rpi-2712", - "temp_c": 70.5, + "temp_c": 45.2, "memory_mb": { "total": 8063, - "used": 5488, - "free": 2574 + "used": 6909, + "free": 1153 }, - "disk": "73G/235G (33%)", + "disk": "74G/235G (33%)", "load": [ - 5.87, - 7.38, - 12.7 + 3.09, + 3.59, + 5.08 ], "ollama_models": [ "qwen2.5:3b", diff --git a/fleet/octavia/status.json b/fleet/octavia/status.json index 5300c35..62023c8 100644 --- a/fleet/octavia/status.json +++ b/fleet/octavia/status.json @@ -1 +1 @@ -{"node":"octavia","status":"down","ts":"2026-03-16T07:31:43Z"} +{"node":"octavia","status":"down","ts":"2026-03-16T18:58:35Z"} diff --git a/scripts/blackroad-fleet-coordinator.sh b/scripts/blackroad-fleet-coordinator.sh index 07b09b9..61cbc15 100755 --- a/scripts/blackroad-fleet-coordinator.sh +++ b/scripts/blackroad-fleet-coordinator.sh @@ -22,13 +22,38 @@ SLACK_URL="https://blackroad-slack.amundsonalexa.workers.dev" log() { printf "[%s] %s\n" "$(date '+%Y-%m-%d %H:%M:%S')" "$1"; } +ALERT_COOLDOWN=1800 # 30 minutes between repeat alerts for same issue +ALERT_STATE_DIR="$STATE_DIR/alerts" +mkdir -p "$ALERT_STATE_DIR" + slack_alert() { + local key=$(echo "$1" | md5sum | cut -c1-12 2>/dev/null || echo "$(date +%s)") + local cooldown_file="$ALERT_STATE_DIR/$key" + # Skip if same alert was sent within cooldown period + if [[ -f "$cooldown_file" ]]; then + local last=$(cat "$cooldown_file" 2>/dev/null) + local now=$(date +%s) + if (( now - last < ALERT_COOLDOWN )); then + return 0 # Suppress duplicate + fi + fi + date +%s > "$cooldown_file" curl -s --max-time 5 -X POST "$SLACK_URL/alert" \ -H "Content-Type: application/json" \ -d "{\"text\":\"$1\"}" >/dev/null 2>&1 || true } slack_post() { + local key=$(echo "$1" | md5sum | cut -c1-12 2>/dev/null || echo "$(date +%s)") + local cooldown_file="$ALERT_STATE_DIR/$key" + if [[ -f "$cooldown_file" ]]; then + local last=$(cat "$cooldown_file" 2>/dev/null) + local now=$(date +%s) + if (( now - last < ALERT_COOLDOWN )); then + return 0 + fi + fi + date +%s > "$cooldown_file" curl -s --max-time 5 -X POST "$SLACK_URL/post" \ -H "Content-Type: application/json" \ -d "{\"text\":\"$1\"}" >/dev/null 2>&1 || true