blackroad/bin/blackroad-experiment-orchestrator.sh

#!/usr/bin/env bash
# ============================================================================
# BLACKROAD OS, INC. - PROPRIETARY AND CONFIDENTIAL
# Copyright (c) 2025-2026 BlackRoad OS, Inc. All Rights Reserved.
#
# This code is the intellectual property of BlackRoad OS, Inc.
# AI-assisted development does not transfer ownership to AI providers.
# Unauthorized use, copying, or distribution is prohibited.
# NOT licensed for AI training or data extraction.
# ============================================================================
# blackroad-experiment-orchestrator.sh
# Comprehensive capability testing for BlackRoad infrastructure
# Tests all devices to discover what they can do without external dependencies

# Don't exit on errors - we want to continue testing even if some tests fail
set +e

source "$HOME/.blackroad/config/nodes.sh" 2>/dev/null || true

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
MAGENTA='\033[0;35m'
PINK='\033[38;5;205m'
BOLD='\033[1m'
NC='\033[0m'

# Use ALL_NODES from config (falls back to defaults if config missing)
DEVICE_NAMES=("${ALL_NODES[@]:-alice cecilia octavia aria lucidia}")

RESULTS_DIR="/tmp/blackroad-experiments-$(date +%s)"
mkdir -p "$RESULTS_DIR"

# Experiment tracking
EXPERIMENTS_TOTAL=0
EXPERIMENTS_PASSED=0
EXPERIMENTS_FAILED=0

echo -e "${BOLD}${PINK}"
cat << "EOF"
╔══════════════════════════════════════════════════════════════╗
║                                                              ║
║   🧪 BLACKROAD EXPERIMENT ORCHESTRATOR 🧪                   ║
║                                                              ║
║   Discovering Infrastructure Capabilities                    ║
║   No External Dependencies Required                          ║
║                                                              ║
╚══════════════════════════════════════════════════════════════╝
EOF
echo -e "${NC}"

# Helper function to run command on device (uses nodes.sh config)
run_on_device() {
    local device=$1
    local cmd=$2
    local ip="${NODE_IP[$device]:-}"
    local user="${NODE_USER[$device]:-}"

    if [[ -z "$ip" ]]; then
        echo "Device $device not found in config" >&2
        return 1
    fi

    ssh -o ConnectTimeout=5 -o BatchMode=yes "${user}@${ip}" "$cmd" 2>/dev/null
}

# Helper function to log experiment result
log_experiment() {
    local name=$1
    local device=$2
    local status=$3
    local result=$4

    ((EXPERIMENTS_TOTAL++))

    if [[ "$status" == "PASS" ]]; then
        ((EXPERIMENTS_PASSED++))
        echo -e "${GREEN}✓${NC} [$device] $name: $result"
    else
        ((EXPERIMENTS_FAILED++))
        echo -e "${RED}✗${NC} [$device] $name: $result"
    fi

    echo "$device,$name,$status,$result" >> "$RESULTS_DIR/experiments.csv"
}

# Experiment 1: Hardware Discovery
experiment_hardware_discovery() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 1: Hardware Discovery ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        local ip="${NODE_IP[$device]:-}"
        [[ -z "$ip" ]] && continue
        echo -e "${YELLOW}Testing $device ($ip)...${NC}"

        # CPU info
        cpu_info=$(run_on_device "$device" "nproc 2>/dev/null || echo '0'")
        if [[ "$cpu_info" != "0" ]]; then
            log_experiment "CPU_COUNT" "$device" "PASS" "${cpu_info} cores"
        else
            log_experiment "CPU_COUNT" "$device" "FAIL" "Unable to detect"
        fi

        # Memory info
        mem_info=$(run_on_device "$device" "free -h | grep Mem | awk '{print \$2}'")
        if [[ -n "$mem_info" ]]; then
            log_experiment "MEMORY_TOTAL" "$device" "PASS" "$mem_info"
        else
            log_experiment "MEMORY_TOTAL" "$device" "FAIL" "Unable to detect"
        fi

        # Disk space
        disk_info=$(run_on_device "$device" "df -h / | tail -1 | awk '{print \$2}'")
        if [[ -n "$disk_info" ]]; then
            log_experiment "DISK_SPACE" "$device" "PASS" "$disk_info"
        else
            log_experiment "DISK_SPACE" "$device" "FAIL" "Unable to detect"
        fi

        # Architecture
        arch_info=$(run_on_device "$device" "uname -m")
        if [[ -n "$arch_info" ]]; then
            log_experiment "ARCHITECTURE" "$device" "PASS" "$arch_info"
        else
            log_experiment "ARCHITECTURE" "$device" "FAIL" "Unable to detect"
        fi
    done
}

# Experiment 2: Software Inventory
experiment_software_inventory() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 2: Software Inventory ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Checking $device...${NC}"

        # Docker
        if run_on_device "$device" "command -v docker" &>/dev/null; then
            version=$(run_on_device "$device" "docker --version | cut -d' ' -f3 | tr -d ','")
            log_experiment "DOCKER" "$device" "PASS" "v$version"
        else
            log_experiment "DOCKER" "$device" "FAIL" "Not installed"
        fi

        # Python
        if run_on_device "$device" "command -v python3" &>/dev/null; then
            version=$(run_on_device "$device" "python3 --version | cut -d' ' -f2")
            log_experiment "PYTHON3" "$device" "PASS" "v$version"
        else
            log_experiment "PYTHON3" "$device" "FAIL" "Not installed"
        fi

        # Node.js
        if run_on_device "$device" "command -v node" &>/dev/null; then
            version=$(run_on_device "$device" "node --version | tr -d 'v'")
            log_experiment "NODEJS" "$device" "PASS" "v$version"
        else
            log_experiment "NODEJS" "$device" "FAIL" "Not installed"
        fi

        # Git
        if run_on_device "$device" "command -v git" &>/dev/null; then
            version=$(run_on_device "$device" "git --version | cut -d' ' -f3")
            log_experiment "GIT" "$device" "PASS" "v$version"
        else
            log_experiment "GIT" "$device" "FAIL" "Not installed"
        fi

        # Kubernetes
        if run_on_device "$device" "command -v kubectl" &>/dev/null; then
            log_experiment "KUBERNETES" "$device" "PASS" "kubectl available"
        else
            log_experiment "KUBERNETES" "$device" "FAIL" "Not installed"
        fi
    done
}

# Experiment 3: CPU Performance Test
experiment_cpu_performance() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 3: CPU Performance Test ━━━${NC}\n"
    echo -e "${YELLOW}Running CPU benchmark (1000 iterations)...${NC}"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        # Simple CPU test: count to 1000000
        start_time=$(date +%s)
        result=$(run_on_device "$device" "
            count=0
            for i in \$(seq 1 1000); do
                count=\$((count + i))
            done
            echo \$count
        " 2>/dev/null || echo "0")
        end_time=$(date +%s)
        duration=$((end_time - start_time))

        if [[ "$result" != "0" ]]; then
            log_experiment "CPU_BENCH" "$device" "PASS" "${duration}s (result: $result)"
        else
            log_experiment "CPU_BENCH" "$device" "FAIL" "Benchmark failed"
        fi
    done
}

# Experiment 4: Network Speed Test
experiment_network_speed() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 4: Network Speed Test ━━━${NC}\n"
    echo -e "${YELLOW}Testing inter-device network performance...${NC}"

    # Create 1MB test file (smaller for faster tests)
    dd if=/dev/zero of=/tmp/nettest bs=1M count=1 2>/dev/null

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        local ip="${NODE_IP[$device]:-}"
        local user="${NODE_USER[$device]:-}"

        start_time=$(date +%s)
        if scp -o ConnectTimeout=5 -o StrictHostKeyChecking=no /tmp/nettest "${user}@${ip}:/tmp/nettest" &>/dev/null; then
            end_time=$(date +%s)
            duration=$((end_time - start_time))

            # Calculate approximate speed
            if [[ $duration -gt 0 ]]; then
                log_experiment "NETWORK_UPLOAD" "$device" "PASS" "~${duration}s for 1MB"
            else
                log_experiment "NETWORK_UPLOAD" "$device" "PASS" "<1s for 1MB"
            fi

            # Cleanup
            run_on_device "$device" "rm -f /tmp/nettest" &>/dev/null
        else
            log_experiment "NETWORK_UPLOAD" "$device" "FAIL" "Transfer failed"
        fi
    done

    rm -f /tmp/nettest
}

# Experiment 5: Container Capability Test
experiment_container_capability() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 5: Container Capability Test ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Testing $device...${NC}"

        # Check if docker is running
        if ! run_on_device "$device" "docker ps" &>/dev/null; then
            log_experiment "DOCKER_RUNNING" "$device" "FAIL" "Docker not running"
            continue
        fi

        log_experiment "DOCKER_RUNNING" "$device" "PASS" "Docker daemon active"

        # Count running containers
        container_count=$(run_on_device "$device" "docker ps -q | wc -l")
        log_experiment "RUNNING_CONTAINERS" "$device" "PASS" "$container_count containers"

        # List networks
        network_count=$(run_on_device "$device" "docker network ls -q | wc -l")
        log_experiment "DOCKER_NETWORKS" "$device" "PASS" "$network_count networks"

        # Check available images
        image_count=$(run_on_device "$device" "docker images -q | wc -l")
        log_experiment "DOCKER_IMAGES" "$device" "PASS" "$image_count images cached"
    done
}

# Experiment 6: Parallel Processing Test
experiment_parallel_processing() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 6: Parallel Processing Test ━━━${NC}\n"
    echo -e "${YELLOW}Testing distributed workload processing...${NC}"

    # Create a test script that each device will run
    test_script='
#!/bin/bash
count=0
for i in $(seq 1 1000); do
    count=$((count + i))
done
echo $count
'

    # Start all devices simultaneously
    pids=()
    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        (
            result=$(run_on_device "$device" "$test_script")
            echo "$device:$result" >> "$RESULTS_DIR/parallel_results.txt"
        ) &
        pids+=($!)
    done

    # Wait for all to complete
    start_time=$(date +%s)
    for pid in "${pids[@]}"; do
        wait "$pid"
    done
    end_time=$(date +%s)

    total_time=$((end_time - start_time))
    device_count=${#DEVICE_NAMES[@]}

    log_experiment "PARALLEL_PROCESSING" "cluster" "PASS" "${device_count} devices completed in ${total_time}s"
}

# Experiment 7: Storage I/O Performance
experiment_storage_io() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 7: Storage I/O Performance ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Testing $device storage I/O...${NC}"

        # Write test
        write_speed=$(run_on_device "$device" "
            dd if=/dev/zero of=/tmp/iotest bs=1M count=100 2>&1 | \
            grep -oP '\d+(\.\d+)? MB/s' | tail -1
        " || echo "0 MB/s")

        if [[ "$write_speed" != "0 MB/s" ]]; then
            log_experiment "STORAGE_WRITE" "$device" "PASS" "$write_speed"
        else
            log_experiment "STORAGE_WRITE" "$device" "FAIL" "Unable to test"
        fi

        # Read test
        read_speed=$(run_on_device "$device" "
            dd if=/tmp/iotest of=/dev/null bs=1M 2>&1 | \
            grep -oP '\d+(\.\d+)? MB/s' | tail -1
        " || echo "0 MB/s")

        if [[ "$read_speed" != "0 MB/s" ]]; then
            log_experiment "STORAGE_READ" "$device" "PASS" "$read_speed"
        else
            log_experiment "STORAGE_READ" "$device" "FAIL" "Unable to test"
        fi

        # Cleanup
        run_on_device "$device" "rm -f /tmp/iotest" &>/dev/null
    done
}

# Experiment 8: Service Discovery
experiment_service_discovery() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 8: Active Service Discovery ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Discovering services on $device...${NC}"

        # Check for common listening ports
        ports=$(run_on_device "$device" "ss -tuln 2>/dev/null | grep LISTEN | awk '{print \$5}' | cut -d':' -f2 | sort -n | uniq | tr '\n' ',' | sed 's/,$//'")

        if [[ -n "$ports" ]]; then
            log_experiment "LISTENING_PORTS" "$device" "PASS" "$ports"
        else
            log_experiment "LISTENING_PORTS" "$device" "FAIL" "No ports detected"
        fi

        # Check systemd services (if available)
        service_count=$(run_on_device "$device" "systemctl list-units --type=service --state=running 2>/dev/null | grep -c '.service' || echo '0'")

        if [[ "$service_count" != "0" ]]; then
            log_experiment "ACTIVE_SERVICES" "$device" "PASS" "$service_count systemd services"
        else
            log_experiment "ACTIVE_SERVICES" "$device" "FAIL" "Unable to count services"
        fi
    done
}

# Experiment 9: Compilation Test
experiment_compilation_capability() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 9: Native Compilation Test ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Testing compilation on $device...${NC}"

        # Create simple C program
        c_program='
#include <stdio.h>
int main() {
    printf("BlackRoad Compilation Test\\n");
    return 0;
}
'

        # Test GCC
        if run_on_device "$device" "command -v gcc" &>/dev/null; then
            compile_result=$(run_on_device "$device" "
                echo '$c_program' > /tmp/test.c && \
                gcc /tmp/test.c -o /tmp/test 2>&1 && \
                /tmp/test && \
                rm -f /tmp/test.c /tmp/test
            " 2>/dev/null)

            if [[ "$compile_result" == *"BlackRoad Compilation Test"* ]]; then
                log_experiment "GCC_COMPILE" "$device" "PASS" "Native compilation working"
            else
                log_experiment "GCC_COMPILE" "$device" "FAIL" "Compilation failed"
            fi
        else
            log_experiment "GCC_COMPILE" "$device" "FAIL" "GCC not installed"
        fi
    done
}

# Experiment 10: Memory Stress Test
experiment_memory_stress() {
    echo -e "\n${BOLD}${PINK}━━━ Experiment 10: Memory Capacity Test ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        echo -e "${YELLOW}Testing $device memory...${NC}"

        # Get available memory
        mem_available=$(run_on_device "$device" "free -m | grep Mem | awk '{print \$7}'")

        if [[ -n "$mem_available" ]]; then
            log_experiment "MEMORY_AVAILABLE" "$device" "PASS" "${mem_available}MB free"

            # Test memory allocation (allocate 100MB)
            mem_test=$(run_on_device "$device" "
                python3 -c 'import sys; data = bytearray(100 * 1024 * 1024); print(len(data))' 2>/dev/null || echo '0'
            ")

            if [[ "$mem_test" != "0" ]]; then
                log_experiment "MEMORY_ALLOCATION" "$device" "PASS" "Can allocate 100MB"
            else
                log_experiment "MEMORY_ALLOCATION" "$device" "FAIL" "Allocation test failed"
            fi
        else
            log_experiment "MEMORY_AVAILABLE" "$device" "FAIL" "Unable to detect"
        fi
    done
}

# Generate comprehensive report
generate_report() {
    echo -e "\n${BOLD}${PINK}"
    cat << "EOF"
╔══════════════════════════════════════════════════════════════╗
║                 📊 EXPERIMENT RESULTS 📊                    ║
╚══════════════════════════════════════════════════════════════╝
EOF
    echo -e "${NC}"

    echo -e "${BOLD}Total Experiments:${NC} $EXPERIMENTS_TOTAL"
    echo -e "${GREEN}Passed:${NC} $EXPERIMENTS_PASSED"
    echo -e "${RED}Failed:${NC} $EXPERIMENTS_FAILED"

    pass_rate=$(echo "scale=2; ($EXPERIMENTS_PASSED * 100) / $EXPERIMENTS_TOTAL" | bc)
    echo -e "${BOLD}Pass Rate:${NC} ${pass_rate}%"

    echo -e "\n${BOLD}Results saved to:${NC} $RESULTS_DIR"
    echo -e "  • experiments.csv - Detailed results"
    echo -e "  • parallel_results.txt - Parallel processing data"

    # Generate device capability matrix
    echo -e "\n${BOLD}${PINK}━━━ Device Capability Matrix ━━━${NC}\n"

    for device in "${DEVICE_NAMES[@]}"; do
        # device set by loop
        local ip="${NODE_IP[$device]:-}"
        echo -e "${YELLOW}$device ($ip):${NC}"
        grep "^$device," "$RESULTS_DIR/experiments.csv" | while IFS=, read -r dev exp status result; do
            if [[ "$status" == "PASS" ]]; then
                echo -e "  ${GREEN}✓${NC} $exp: $result"
            fi
        done
        echo ""
    done
}

# Main execution
main() {
    echo -e "${PINK}Starting comprehensive infrastructure experiments...${NC}\n"
    echo -e "${YELLOW}Testing ${#DEVICE_NAMES[@]} devices:${NC} ${DEVICE_NAMES[*]}\n"

    # Run all experiments
    experiment_hardware_discovery
    experiment_software_inventory
    experiment_cpu_performance
    experiment_network_speed
    experiment_container_capability
    experiment_parallel_processing
    experiment_storage_io
    experiment_service_discovery
    experiment_compilation_capability
    experiment_memory_stress

    # Generate final report
    generate_report

    # Log to memory system
    ~/memory-system.sh log experiment "[INFRASTRUCTURE]+[CAPABILITY_TEST] Experiment Orchestrator Complete" \
        "Ran $EXPERIMENTS_TOTAL experiments across ${#DEVICE_NAMES[@]} devices. Pass rate: ${pass_rate}%. Results: $RESULTS_DIR" \
        "infrastructure,testing,experiments" 2>/dev/null || true

    echo -e "\n${GREEN}${BOLD}🎉 All experiments complete! 🎉${NC}\n"
}

# Handle script arguments
case "${1:-}" in
    help|--help|-h)
        echo "Usage: $0 [command]"
        echo ""
        echo "Commands:"
        echo "  (no args)   - Run all experiments"
        echo "  hardware    - Hardware discovery only"
        echo "  software    - Software inventory only"
        echo "  network     - Network tests only"
        echo "  containers  - Container tests only"
        echo "  help        - Show this help"
        ;;
    hardware)
        experiment_hardware_discovery
        ;;
    software)
        experiment_software_inventory
        ;;
    network)
        experiment_network_speed
        ;;
    containers)
        experiment_container_capability
        ;;
    *)
        main
        ;;
esac