improvements to wizard UX,

implemented HealthMonitor to broadcast ws when a sensor goes inadvertedly offline,
implemented /api/v1/offline for graceful shut down of sensors to instantly mark them offline,
implemented graceful shutdown in sensor sdks
This commit is contained in:
AndReicscs
2026-05-22 12:22:12 +00:00
parent 555bcdbd3c
commit cf78b45472
17 changed files with 284 additions and 43 deletions
+8 -4
View File
@@ -40,8 +40,13 @@ func main() {
}
defer dbStore.DB.Close()
// 1. Establish Root Context for all background workers
rootCtx, rootCancel := context.WithCancel(context.Background())
defer rootCancel()
sessionStore := auth.NewSessionStore()
r, err := api.SetupRouter(cfg, dbStore, sessionStore)
h := api.NewHandler(dbStore, cfg, sessionStore)
r, err := api.SetupRouter(h)
if err != nil {
log.Fatalf("[FATAL] Router setup failed: %v", err)
}
@@ -49,6 +54,7 @@ func main() {
// 1. Start External Workers
notify.StartWorker()
siem.StartWorker()
go h.StartHealthMonitor(rootCtx)
// 2. Load Runtime Configurations Safely
isArmed := loadConfigSafe(dbStore, "is_armed", "false") == "true"
@@ -68,8 +74,6 @@ func main() {
}
// 3. Start Database Retention Worker (cancelable)
rootCtx, rootCancel := context.WithCancel(context.Background())
defer rootCancel()
go startRetentionWorker(rootCtx, dbStore)
// 4. Start HTTP Server
@@ -137,4 +141,4 @@ func startRetentionWorker(ctx context.Context, s *store.SQLiteStore) {
}
}
}
}
}
+39 -2
View File
@@ -1,6 +1,7 @@
package api
import (
"context"
"encoding/json"
"log"
"net"
@@ -56,7 +57,7 @@ func SendJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(data); err != nil {
log.Printf("[ERROR] SendJSON encode error: %v\n", err)
log.Printf("[ERROR] SendJSON encode error: %v\n", err)
}
}
@@ -132,4 +133,40 @@ func (h *Handler) startChartSyncBroadcaster() {
// and they should refresh their time-series charts.
h.broadcastWS("SYNC_CHARTS", nil)
}
}
}
// StartHealthMonitor runs a background task to periodically check for offline nodes and sensors.
// It should be invoked with a context from main.go to ensure graceful shutdown.
func (h *Handler) StartHealthMonitor(ctx context.Context) {
log.Println("[INFO] Starting background health monitor...")
tickerPeriod := 30 * time.Second
ticker := time.NewTicker(tickerPeriod)
defer ticker.Stop()
// Offset lastCheck by the ticker period so the first run catches recent drops
lastCheck := time.Now().UTC().Add(-tickerPeriod)
for {
select {
case <-ctx.Done():
log.Println("[INFO] Health monitor stopped")
return
case t := <-ticker.C:
offlineThreshold := 60 * time.Second
updatedNodeIDs, err := h.Store.GetTransitionedOfflineNodes(offlineThreshold, lastCheck)
if err == nil {
for nodeID := range updatedNodeIDs {
// Send a lightweight signal to instruct the UI to securely refresh this node's state
h.broadcastWS("UPDATE_NODE", map[string]interface{}{
"id": nodeID,
"trigger_refresh": true,
})
}
}
lastCheck = t.UTC()
}
}
}
+3 -7
View File
@@ -9,9 +9,6 @@ import (
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/honeywire/hub/internal/auth"
"github.com/honeywire/hub/internal/config"
"github.com/honeywire/hub/internal/store"
"github.com/honeywire/hub/ui"
)
@@ -29,14 +26,12 @@ func ErrorOnlyLogger(next http.Handler) http.Handler {
})
}
func SetupRouter(cfg *config.Config, s *store.SQLiteStore, sessionStore *auth.SessionStore) (*chi.Mux, error) {
func SetupRouter(h *Handler) (*chi.Mux, error) {
r := chi.NewRouter()
r.Use(ErrorOnlyLogger)
r.Use(middleware.Recoverer)
h := NewHandler(s, cfg, sessionStore)
// Public Endpoints
r.Get("/api/v1/version", h.HandleVersion)
r.Post("/login", h.Login)
@@ -46,7 +41,7 @@ func SetupRouter(cfg *config.Config, s *store.SQLiteStore, sessionStore *auth.Se
// UI Endpoints (Protected by Cookies)
r.Group(func(r chi.Router) {
r.Use(UIAuthMiddleware(sessionStore))
r.Use(UIAuthMiddleware(h.SessionStore))
r.Get("/api/v1/ws", h.ServeWS)
@@ -92,6 +87,7 @@ func SetupRouter(cfg *config.Config, s *store.SQLiteStore, sessionStore *auth.Se
r.Get("/api/v1/nodes/me", h.GetCurrentNode) // node whoami based on api key.
r.Get("/api/v1/nodes/compose", h.GetNodeCompose) // aggreagates all generated compose files for a node's sensors
r.Post("/api/v1/heartbeat", h.ReceiveHeartbeat)
r.Post("/api/v1/offline", h.ReceiveOffline)
r.Post("/api/v1/event", h.ReceiveEvent)
r.Get("/api/v1/manifests", h.GetManifests) // Fetches catalog (Dual Auth)
+40
View File
@@ -78,6 +78,46 @@ func (h *Handler) ReceiveHeartbeat(w http.ResponseWriter, r *http.Request) {
SendJSON(w, http.StatusOK, map[string]string{"status": "alive"})
}
func (h *Handler) ReceiveOffline(w http.ResponseWriter, r *http.Request) {
var req struct {
SensorID string `json:"sensor_id"`
Reason string `json:"reason"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
RespondError(w, "Invalid JSON body", http.StatusBadRequest)
return
}
if req.SensorID == "" {
RespondError(w, "sensor_id is required", http.StatusBadRequest)
return
}
nodeID, err := h.authenticateNodeRequest(r)
if err != nil {
RespondError(w, "Unauthorized", http.StatusUnauthorized)
return
}
// Push last_heartbeat 2 hours into the past to instantly force deriveStatus() to return "down"
offlineTime := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339)
if err := h.Store.MarkSensorOffline(nodeID, req.SensorID, offlineTime); err != nil {
log.Printf("[ERROR] Failed to set offline status for node %s sensor %s: %v", nodeID, req.SensorID, err)
RespondError(w, "Database error", http.StatusInternalServerError)
return
}
log.Printf("[INFO] Sensor %s on node %s went offline gracefully (Reason: %s)", req.SensorID, nodeID, req.Reason)
h.broadcastWS("UPDATE_NODE", map[string]interface{}{
"id": nodeID,
"trigger_refresh": true,
})
SendJSON(w, http.StatusOK, map[string]string{"status": "offline_acknowledged"})
}
func (h *Handler) GetUptime(w http.ResponseWriter, r *http.Request) {
timeframe := r.URL.Query().Get("timeframe")
if timeframe == "" {
+1 -1
View File
@@ -142,7 +142,7 @@ func GenerateUptimeResult(timeframe string, now time.Time, params UptimeParams,
})
}
isLive := now.Sub(s.LastSeen) < 90*time.Second
isLive := now.Sub(s.LastSeen) < 60*time.Second
if isLive {
blocks[len(blocks)-1]["status"] = "up"
blocks[len(blocks)-1]["label"] = "Online (Live)"
+55
View File
@@ -0,0 +1,55 @@
package store
import (
"time"
)
// GetTransitionedOfflineNodes checks all nodes and sensors to see if their
// last_heartbeat crossed the offline threshold since the last check.
// It returns a set of Node IDs that transitioned to 'down' so the Hub can broadcast them.
func (s *SQLiteStore) GetTransitionedOfflineNodes(offlineThreshold time.Duration, lastCheck time.Time) (map[string]bool, error) {
now := time.Now().UTC()
// The heartbeat must be older than the current threshold...
cutoffNow := now.Add(-offlineThreshold).Format(time.RFC3339)
// ...but newer than or equal to the threshold was at the time of the last check.
cutoffPrev := lastCheck.UTC().Add(-offlineThreshold).Format(time.RFC3339)
updatedNodes := make(map[string]bool)
// 1. Identify nodes that have gone offline
rows, err := s.DB.Query(`
SELECT id FROM nodes
WHERE last_heartbeat < ? AND last_heartbeat >= ? AND last_heartbeat != ''
`, cutoffNow, cutoffPrev)
if err == nil {
defer rows.Close()
for rows.Next() {
var id string
if err := rows.Scan(&id); err == nil {
updatedNodes[id] = true
}
}
} else {
return nil, err
}
// 2. Identify sensors that have gone offline
sensorRows, err := s.DB.Query(`
SELECT node_id FROM node_sensors
WHERE last_heartbeat < ? AND last_heartbeat >= ? AND last_heartbeat != ''
`, cutoffNow, cutoffPrev)
if err == nil {
defer sensorRows.Close()
for sensorRows.Next() {
var nid string
if err := sensorRows.Scan(&nid); err == nil {
updatedNodes[nid] = true
}
}
} else {
return nil, err
}
return updatedNodes, nil
}
+2 -2
View File
@@ -62,8 +62,8 @@ func deriveStatus(lastHeartbeat *string) string {
if err != nil {
return "down"
}
// If heartbeat is older than 90 seconds, consider it offline
if time.Now().UTC().Sub(t) > 90*time.Second {
// If heartbeat is older than 60 seconds, consider it offline
if time.Now().UTC().Sub(t) > 60*time.Second {
return "down"
}
return "up"
+6
View File
@@ -170,3 +170,9 @@ func (s *SQLiteStore) UpdateNodeLastHeartbeat(nodeID, sensorID, timestamp string
return tx.Commit()
}
// MarkSensorOffline artificially ages a sensor's heartbeat to instantly mark it as 'down'
func (s *SQLiteStore) MarkSensorOffline(nodeID, sensorID, offlineTime string) error {
_, err := s.DB.Exec(`UPDATE node_sensors SET last_heartbeat = ?, updated_at = ? WHERE node_id = ? AND sensor_id = ?`, offlineTime, offlineTime, nodeID, sensorID)
return err
}
+1
View File
@@ -36,6 +36,7 @@ type DataStore interface {
GetHeartbeatsSince(cutoffStr string) ([]HeartbeatData, error)
UpdateSensorSilence(nodeID, sensorID string, silenceVal int) error
DeleteSensor(nodeID, sensorID string) (int64, error)
MarkSensorOffline(nodeID, sensorID, offlineTime string) error
// Provisioning
InsertPairingToken(token string, expiresAt time.Time, createdAt time.Time) error
+8 -2
View File
@@ -528,8 +528,14 @@ export const useFleetStore = defineStore('fleet', () => {
}
if (type === 'UPDATE_NODE') {
const node = getNode(payload.id)
if (node) mergeNode(node, normalizeNode(payload))
if (payload.trigger_refresh) {
fetchNodeDetails(payload.id)
return
}
const normalized = normalizeNode(payload)
const targetNode = getNode(normalized.id)
if (targetNode) mergeNode(targetNode, normalized)
return
}
+35 -5
View File
@@ -52,8 +52,9 @@ func (s *Sensor) Start() error {
return nil
}
// Stop gracefully shuts down the heartbeat goroutine.
// Stop gracefully shuts down the heartbeat goroutine and notifies the Hub.
func (s *Sensor) Stop() {
s.GoOffline("graceful_shutdown")
close(s.stopCh)
}
@@ -131,7 +132,7 @@ func (s *Sensor) sendHeartbeat() {
"HW_CONFIG_REV": s.ConfigRev,
},
}
resp, err := s.postToHub("/api/v1/heartbeat", payload)
if err != nil {
log.Printf("[-] Heartbeat failed to send: %v", err)
@@ -162,16 +163,45 @@ func (s *Sensor) ReportEvent(severity, trigger, source, target string, details m
return false
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
log.Printf("[-] Hub rejected event report (HTTP %d).", resp.StatusCode)
return false
}
log.Printf("[+] Event successfully reported to Hub.")
return true
}
func (s *Sensor) GoOffline(reason string) {
log.Printf("[*] Sending graceful offline status (reason: %s)...", reason)
// Strict 2-second timeout: best-effort, never hang the container shutdown
fastClient := &http.Client{Timeout: 2 * time.Second}
payload := map[string]any{
"sensor_id": s.SensorID,
"reason": reason,
}
jsonData, err := json.Marshal(payload)
if err != nil {
return
}
req, err := http.NewRequest("POST", s.HubEndpoint+"/api/v1/offline", bytes.NewReader(jsonData))
if err != nil {
return
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+s.HubKey)
if resp, err := fastClient.Do(req); err == nil {
resp.Body.Close()
}
}
func (s *Sensor) postToHub(path string, data map[string]any) (*http.Response, error) {
jsonData, err := json.Marshal(data)
if err != nil {
@@ -194,4 +224,4 @@ func getEnv(key, fallback string) string {
return value
}
return fallback
}
}
@@ -137,6 +137,19 @@ class HoneyWireSensor(ABC):
)
return success
def go_offline(self, reason: str):
print(f"[*] Sending graceful offline status (reason: {reason})...")
payload = {
"sensor_id": self.sensor_id,
"reason": reason
}
try:
# Strict 2-second timeout: best-effort, never hang the container shutdown
self._post_to_hub("/api/v1/offline", payload, timeout=2)
except Exception:
# Fail silently; this is a best-effort optimization
pass
@abstractmethod
async def monitor(self):
pass
@@ -147,4 +160,5 @@ class HoneyWireSensor(ABC):
await self.monitor()
def stop(self):
self.go_offline("graceful_shutdown")
self._stop_event.set()
+26 -8
View File
@@ -1,27 +1,41 @@
.PHONY: build test test-gateway test-backend test-chaos
.PHONY: all build build-linux deploy test clean
-include .env
BINARY_NAME=wizard
BUILD_DIR=build
GATEWAY_IP ?= 127.0.0.1
BACKEND_IP ?= 127.0.0.1
CHAOS_IP ?= 127.0.0.1
# Remote deployment defaults (override via `make deploy TARGET_IP=192.168.1.X`)
TARGET_USER?=root
TARGET_IP?=192.168.1.11
TARGET_PATH?=/root/wizard
# URL remote nodes use
HUB_URL ?= http://localhost:8080
all: clean build
# Local API URL
LOCAL_HUB_URL ?= http://127.0.0.1:8080
build:
@echo "[*] Compiling Wizard locally..."
@mkdir -p $(BUILD_DIR)
go build -o $(BUILD_DIR)/$(BINARY_NAME) cmd/wizard/main.go
# Dashboard password for provisioning
HUB_PASSWORD ?= defaultpass
build-linux:
@echo "[*] Compiling Wizard for Linux (amd64)..."
@mkdir -p $(BUILD_DIR)
GOOS=linux GOARCH=amd64 go build -o $(BUILD_DIR)/$(BINARY_NAME)_linux cmd/wizard/main.go
ENV ?= dev
ifeq ($(ENV),public)
MANIFEST_FILE = ../Sensors/official/manifests.json
else
MANIFEST_FILE = ../Sensors/official/manifests.dev.json
endif
deploy: build-linux
@echo "[*] Pushing Wizard to $(TARGET_USER)@$(TARGET_IP)..."
scp $(BUILD_DIR)/$(BINARY_NAME)_linux $(TARGET_USER)@$(TARGET_IP):$(TARGET_PATH)
@echo "✅ Deployment complete! You can now run $(TARGET_PATH) on the remote node."
build:
mkdir -p build
@@ -29,12 +43,13 @@ build:
test:
go test ./...
@echo "[*] Running tests..."
go test ./... -v
define deploy_target
@echo "[*] TARGETING $(1) ($(ENV))"
scp build/wizard root@$($(2)):/root/wizard
scp $(MANIFEST_FILE) root@$($(2)):/root/manifests.json
ssh root@$($(2)) '\
chmod +x /root/wizard && \
@@ -54,4 +69,7 @@ test-backend: build
.ONESHELL:
test-chaos: build
$(call deploy_target,chaos,CHAOS_IP)
$(call deploy_target,chaos,CHAOS_IP)
clean:
@echo "[*] Cleaning build directory..."
rm -rf $(BUILD_DIR)
+1 -1
View File
@@ -55,7 +55,7 @@ Build and run the Wizard on the host:
```bash
make build
./build/wizard --registry https://raw.githubusercontent.com/andreicscs/HoneyWire/main/Sensors/official/manifests.json
./build/wizard
```
Run the module test suite:
+21 -9
View File
@@ -109,14 +109,14 @@ func (c *HubClient) AuthenticateDashboard(ctx context.Context, password string)
return cookieValue, nil
}
func (c *HubClient) CreateNode(ctx context.Context, alias string, tags []string, cookie string) (string, string, error) {
func (c *HubClient) CreateNode(ctx context.Context, alias string, tags []string, cookie string) (string, error) {
payload := createNodeRequest{
Alias: alias,
Tags: tags,
}
body, err := json.Marshal(payload)
if err != nil {
return "", "", fmt.Errorf("failed to marshal create node request: %w", err)
return "", fmt.Errorf("failed to marshal create node request: %w", err)
}
resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/nodes", bytes.NewReader(body), map[string]string{
@@ -124,24 +124,37 @@ func (c *HubClient) CreateNode(ctx context.Context, alias string, tags []string,
"Cookie": "hw_auth=" + cookie,
})
if err != nil {
return "", "", fmt.Errorf("network error: %w", err)
return "", fmt.Errorf("network error: %w", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
return "", "", fmt.Errorf("hub rejected request (HTTP %d): %s", resp.StatusCode, readBodyTruncated(resp))
return "", fmt.Errorf("hub rejected request (HTTP %d): %s", resp.StatusCode, readBodyTruncated(resp))
}
data, err := readBody(resp)
if err != nil {
return "", "", err
return "", err
}
var result createNodeResponse
var result map[string]interface{}
if err := json.Unmarshal(data, &result); err != nil {
return "", "", fmt.Errorf("failed to parse hub response: %w", err)
return "", fmt.Errorf("failed to parse hub response: %w", err)
}
return result.NodeID, result.APIKey, nil
var apiKey string
if val, ok := result["api_key"].(string); ok && val != "" {
apiKey = val
} else if val, ok := result["apiKey"].(string); ok && val != "" {
apiKey = val
} else if val, ok := result["key"].(string); ok && val != "" {
apiKey = val
}
if apiKey == "" {
return "", fmt.Errorf("hub did not return an API key. Raw response: %s", string(data))
}
return apiKey, nil
}
func (c *HubClient) GetCurrentNode(ctx context.Context, apiKey string) (*NodeInfo, error) {
@@ -231,7 +244,6 @@ type createNodeRequest struct {
}
type createNodeResponse struct {
NodeID string `json:"id"`
APIKey string `json:"api_key"`
Alias string `json:"alias"`
}
+11
View File
@@ -7,6 +7,7 @@ import (
"github.com/honeywire/wizard/internal/cli"
"github.com/honeywire/wizard/internal/deploy"
"gopkg.in/yaml.v3"
)
func HandleApply() error {
@@ -26,6 +27,16 @@ func HandleApply() error {
return fmt.Errorf("failed to fetch deployment bundle: %w", err)
}
var compose struct {
Services map[string]interface{} `yaml:"services"`
}
// If the compose file is empty or has no services, there's nothing to do.
if err := yaml.Unmarshal(composeData, &compose); err != nil || len(compose.Services) == 0 {
fmt.Printf("\n %sNothing to reconcile. No sensors are configured for this node.%s\n", cli.Yellow, cli.Reset)
fmt.Printf(" %sUse 'wizard discover' or the Hub Dashboard to add sensors first.%s\n\n", cli.Dim, cli.Reset)
return nil
}
if err := deploy.Apply(ctx, composeData); err != nil {
return fmt.Errorf("reconciliation failed: %w", err)
}
+13 -2
View File
@@ -69,7 +69,7 @@ func HandleRelink(args []string) error {
case "1":
return provisionNewNode(hubURL, "", "")
case "2":
apiKey, err = cli.PromptInput(" API Key: ")
apiKey, err = cli.ReadPasswordMasked(" API Key: ")
if err != nil {
return fmt.Errorf("failed to read API key: %w", err)
}
@@ -113,6 +113,7 @@ func linkExistingNode(hubURL, apiKey string) error {
}
fmt.Printf("\n%s✅ Linked to Hub as '%s'.%s\n", cli.Green, nodeInfo.Alias, cli.Reset)
fmt.Printf(" %sRun 'wizard apply' to deploy this node's sensors.%s\n\n", cli.Dim, cli.Reset)
return nil
}
@@ -151,10 +152,19 @@ func provisionNewNode(hubURL, customAlias, tagsStr string) error {
}
}
nodeID, apiKey, err := hub.CreateNode(ctx, customAlias, tags, cookie)
apiKey, err := hub.CreateNode(ctx, customAlias, tags, cookie)
if err != nil {
return fmt.Errorf("node creation failed: %w", err)
}
// The Hub's CreateNode API response currently only returns the API Key and Alias.
// We must use the new API Key to fetch the authoritative node identity (including its NodeID).
nodeInfo, err := hub.GetCurrentNode(ctx, apiKey)
if err != nil {
return fmt.Errorf("failed to retrieve node ID after creation: %w", err)
}
nodeID := nodeInfo.NodeID
fmt.Printf("%s[*] Node created: %s (ID: %s)%s\n", cli.Green, customAlias, nodeID, cli.Reset)
cfg := app.NodeConfig{
@@ -172,5 +182,6 @@ func provisionNewNode(hubURL, customAlias, tagsStr string) error {
}
fmt.Printf("\n%s✅ Node provisioned and identity saved.%s\n", cli.Green, cli.Reset)
fmt.Printf(" %sRun 'wizard apply' to deploy existing sensors, or 'wizard discover' to add new ones.%s\n\n", cli.Dim, cli.Reset)
return nil
}