diff --git a/Docs/API.md b/Docs/API.md index 388de48..ba722c2 100644 --- a/Docs/API.md +++ b/Docs/API.md @@ -158,7 +158,7 @@ Creates a new node entry and returns the generated node credentials. #### GET /api/v1/nodes -Returns all registered nodes and their installed sensors. Each node includes current status, heartbeat metadata, and pending config state. +Returns all registered nodes and their installed sensors. Each node includes current status and pending config state. **Response:** ```json @@ -546,18 +546,15 @@ These endpoints are used by sensor agents and require the node API key bearer to ### POST /api/v1/heartbeat -Reports a sensor heartbeat and updates runtime metadata. +Reports a sensor heartbeat and updates its versioning state. **Request:** ```json { "sensorId": "alpha-node-01", - "metadata": { - "agent_version": "1.0.0", - "contract_version": "1.0", - "sensor_type": "tarpit", - "HW_CONFIG_REV": "rev_abc123" - } + "agentVersion": "1.0.0", + "contractVersion": "1.0", + "configRev": "rev_abc123" } ``` diff --git a/Docs/architecture/dataContracts.md b/Docs/architecture/dataContracts.md index b5cf6f5..34c6438 100644 --- a/Docs/architecture/dataContracts.md +++ b/Docs/architecture/dataContracts.md @@ -51,20 +51,17 @@ This payload is emitted continuously (typically every 30 seconds) by sensors to ```json { "sensorId": "alpha-node-01", - "metadata": { - "agent_version": "1.0.0", - "contract_version": "1.0", - "HW_CONFIG_REV": "rev_abc123" - } + "agentVersion": "1.0.0", + "contractVersion": "1.0", + "configRev": "rev_abc123" } ``` ### Field Definitions - `sensorId` (string, required): The unique identifier of the sensor. -- `metadata` (object, required): - - `agent_version` (string): The version of the language SDK or agent. - - `contract_version` (string): The API contract version in use. - - `HW_CONFIG_REV` (string): The current configuration revision the sensor is running. Used by the Hub to determine if the node has synchronized its desired state. +- `agentVersion` (string, required): The version of the language SDK or agent. +- `contractVersion` (string, required): The API contract version in use. +- `configRev` (string, required): The current configuration revision the sensor is running. Used by the Hub to determine if the node has synchronized its desired state. --- diff --git a/Docs/architecture/hub/backend/overview.md b/Docs/architecture/hub/backend/overview.md index ecc0305..1c5bc97 100644 --- a/Docs/architecture/hub/backend/overview.md +++ b/Docs/architecture/hub/backend/overview.md @@ -65,6 +65,11 @@ The data access layer. - Operates in Write-Ahead Log (WAL) mode with connection pooling. - Contains **no** business logic. +#### Schema Migrations & Constraints +- The Hub uses an embedded, automated migration system. +- **CRITICAL:** Do NOT use the legacy `CREATE new_table -> DROP old_table -> RENAME` workaround for altering schemas if the table is referenced by foreign keys with `ON DELETE CASCADE`. Doing so while foreign keys are enforced will instantly trigger the cascade and wipe out dependent data (e.g., dropping `node_sensors` deletes all `events`). +- **Solution:** HoneyWire uses a modern SQLite driver (3.35.0+) that natively supports `ALTER TABLE ... DROP COLUMN`. Always use native `ALTER TABLE` operations to guarantee schema mutations are non-destructive and isolated. + ### 5. Read/Analytics Layer (`internal/projections`) A specialized CQRS (Command Query Responsibility Segregation) pattern used for heavy dashboard analytics. - Used for high-volume aggregations like Threat Velocity and Threat Severity Distributions. diff --git a/Hub/internal/api/config.go b/Hub/internal/api/config.go index e63bcea..53d58db 100644 --- a/Hub/internal/api/config.go +++ b/Hub/internal/api/config.go @@ -58,18 +58,7 @@ func (h *ConfigHandler) GetConfig(w http.ResponseWriter, r *http.Request) { return } - SendJSON(w, http.StatusOK, map[string]interface{}{ - "hubEndpoint": cfg.HubEndpoint, - "registryUrl": cfg.RegistryURL, - "autoArchiveDays": cfg.AutoArchiveDays, - "autoPurgeDays": cfg.AutoPurgeDays, - "webhookType": cfg.WebhookType, - "webhookUrl": cfg.WebhookURL, - "webhookEvents": cfg.WebhookEvents, - "siemAddress": cfg.SiemAddress, - "siemProtocol": cfg.SiemProtocol, - "whitelistedSources": cfg.WhitelistedSources, - }) + SendJSON(w, http.StatusOK, cfg) } func (h *ConfigHandler) UpdateConfig(w http.ResponseWriter, r *http.Request) { diff --git a/Hub/internal/api/sensors.go b/Hub/internal/api/sensors.go index 25befdd..e7c1625 100644 --- a/Hub/internal/api/sensors.go +++ b/Hub/internal/api/sensors.go @@ -39,7 +39,7 @@ func (h *SensorHandler) ReceiveHeartbeat(w http.ResponseWriter, r *http.Request) } // Hand off to the Service layer - if err := h.service.ProcessHeartbeat(nodeID, hb.SensorID, hb.Metadata); err != nil { + if err := h.service.ProcessHeartbeat(nodeID, hb.SensorID, hb.AgentVersion, hb.ContractVersion, hb.ConfigRev); err != nil { if errors.Is(err, sensor.ErrSensorNotRegistered) { RespondError(w, "Sensor not registered on this node", http.StatusNotFound) return diff --git a/Hub/internal/models/models.go b/Hub/internal/models/models.go index 7fb46a1..58751cc 100644 --- a/Hub/internal/models/models.go +++ b/Hub/internal/models/models.go @@ -12,16 +12,16 @@ type SetupPayload struct { // ConfigPayload represents the runtime configuration of the Hub type ConfigPayload struct { - HubEndpoint string `json:"hubEndpoint"` - RegistryURL string `json:"registryUrl"` - AutoArchiveDays int `json:"autoArchiveDays"` - AutoPurgeDays int `json:"autoPurgeDays"` - WebhookURL string `json:"webhookUrl"` - WebhookType string `json:"webhookType"` - WebhookEvents []string `json:"webhookEvents"` - SiemAddress string `json:"siemAddress"` - SiemProtocol string `json:"siemProtocol"` - WhitelistedSources string `json:"whitelistedSources"` + HubEndpoint string `json:"hubEndpoint"` + RegistryURL string `json:"registryUrl"` + AutoArchiveDays int `json:"autoArchiveDays"` + AutoPurgeDays int `json:"autoPurgeDays"` + WebhookURL string `json:"webhookUrl"` + WebhookType string `json:"webhookType"` + WebhookEvents []string `json:"webhookEvents"` + SiemAddress string `json:"siemAddress"` + SiemProtocol string `json:"siemProtocol"` + WhitelistedSources string `json:"whitelistedSources"` } type Event struct { @@ -42,8 +42,10 @@ type Event struct { // Heartbeat represents a routine ping from a sensor type Heartbeat struct { - SensorID string `json:"sensorId"` - Metadata map[string]interface{} `json:"metadata"` // Contains HW_CONFIG_REV + SensorID string `json:"sensorId"` + AgentVersion string `json:"agentVersion"` + ContractVersion string `json:"contractVersion"` + ConfigRev string `json:"configRev"` } // Node represents a physical server managing sensors @@ -74,7 +76,6 @@ type NodeSensor struct { IsSilenced bool `json:"isSilenced"` Events24h int `json:"events24h"` EnvVars map[string]interface{} `json:"envVars"` - Metadata map[string]interface{} `json:"metadata"` DeployedVersion string `json:"deployedVersion"` UpdateAvailable bool `json:"updateAvailable"` } diff --git a/Hub/internal/services/sensor/service.go b/Hub/internal/services/sensor/service.go index 6de5494..a320726 100644 --- a/Hub/internal/services/sensor/service.go +++ b/Hub/internal/services/sensor/service.go @@ -2,7 +2,6 @@ package sensor import ( "context" - "encoding/json" "errors" "log" "strings" @@ -11,7 +10,7 @@ import ( // Store defines exactly what the Sensor service needs from internal/store type Store interface { - ProcessHeartbeat(nodeID, sensorID, agentRevision, nowStr, metadataJSON string) (bool, error) + ProcessHeartbeat(nodeID, sensorID, agentVersion, contractVersion, configRev, nowStr string) (bool, error) InsertHeartbeat(nodeID, sensorID, minuteBucket string) error MarkSensorOffline(nodeID, sensorID, offlineTime string) error UpdateSensorSilence(nodeID, sensorID string, silenceVal int) error @@ -38,24 +37,16 @@ func NewService(store Store, broadcaster Broadcaster) *Service { } // ProcessHeartbeat handles the core logic of a sensor checking in -func (s *Service) ProcessHeartbeat(nodeID, sensorID string, metadata map[string]interface{}) error { - now := time.Now().UTC() - nowStr := now.Format(time.RFC3339) - minuteBucket := now.Truncate(time.Minute).Format(time.RFC3339) +func (s *Service) ProcessHeartbeat(nodeID, sensorID, agentVersion, contractVersion, configRev string) error { + nowStr := time.Now().UTC().Format(time.RFC3339) - agentRevision := "" - if rev, ok := metadata["HW_CONFIG_REV"].(string); ok { - agentRevision = rev - } - - metadataJSON, _ := json.Marshal(metadata) - - justSynced, err := s.store.ProcessHeartbeat(nodeID, sensorID, agentRevision, nowStr, string(metadataJSON)) + justSynced, err := s.store.ProcessHeartbeat(nodeID, sensorID, agentVersion, contractVersion, configRev, nowStr) if err != nil { log.Printf("[ERROR] Heartbeat DB update failed for node %s: %v", nodeID, err) return err } + minuteBucket := time.Now().UTC().Truncate(time.Minute).Format(time.RFC3339) if err := s.store.InsertHeartbeat(nodeID, sensorID, minuteBucket); err != nil { if strings.Contains(err.Error(), "FOREIGN KEY") { return ErrSensorNotRegistered diff --git a/Hub/internal/store/migrations.go b/Hub/internal/store/migrations.go index c366647..41a4030 100644 --- a/Hub/internal/store/migrations.go +++ b/Hub/internal/store/migrations.go @@ -36,7 +36,9 @@ CREATE TABLE IF NOT EXISTS node_sensors ( sensor_id TEXT NOT NULL, custom_name TEXT NOT NULL, config_values TEXT NOT NULL DEFAULT '{}', - metadata TEXT NOT NULL DEFAULT '{}', + agent_version TEXT NOT NULL DEFAULT '', + contract_version TEXT NOT NULL DEFAULT '', + config_rev TEXT NOT NULL DEFAULT '', deployed_version TEXT NOT NULL DEFAULT '', last_heartbeat TEXT, is_silenced INTEGER NOT NULL DEFAULT 0, @@ -87,6 +89,18 @@ CREATE INDEX IF NOT EXISTS idx_sensors_node ON node_sensors(node_id); CREATE INDEX IF NOT EXISTS idx_heartbeats_time ON sensor_heartbeats(time_bucket); ` +// WARNING: Schema Migrations & Foreign Keys +// +// HoneyWire uses ON DELETE CASCADE for critical tables (e.g., deleting a node_sensor +// automatically deletes all its events). +// +// When altering schemas, NEVER use the legacy SQLite workaround of creating a new table, +// copying data, and dropping the old table. If foreign keys are enabled, dropping the +// old table will instantly trigger the cascade and permanently delete all dependent rows. +// +// ALWAYS use native SQLite 3.35.0+ ALTER TABLE statements (e.g., ADD COLUMN or DROP COLUMN) +// to mutate schemas safely without destroying dependent data. + var migrations = []Migration{ { Version: 1, diff --git a/Hub/internal/store/nodes.go b/Hub/internal/store/nodes.go index dce3e79..8a42eeb 100644 --- a/Hub/internal/store/nodes.go +++ b/Hub/internal/store/nodes.go @@ -128,7 +128,7 @@ func (s *SQLiteStore) GetNodes() ([]models.Node, error) { // Fetch sensors for i := range nodes { rows, err := s.DB.Query(` - SELECT sensor_id, custom_name, config_values, metadata, deployed_version, is_silenced, last_heartbeat + SELECT sensor_id, custom_name, config_values, deployed_version, is_silenced, last_heartbeat FROM node_sensors WHERE node_id = ?`, nodes[i].ID) if err != nil { return nil, err @@ -137,11 +137,10 @@ func (s *SQLiteStore) GetNodes() ([]models.Node, error) { for rows.Next() { var ns models.NodeSensor var configStr string - var metaStr string var silencedInt int var lastHb sql.NullString - if err := rows.Scan(&ns.Name, &ns.Display, &configStr, &metaStr, &ns.DeployedVersion, &silencedInt, &lastHb); err != nil { + if err := rows.Scan(&ns.Name, &ns.Display, &configStr, &ns.DeployedVersion, &silencedInt, &lastHb); err != nil { rows.Close() return nil, err } @@ -197,7 +196,7 @@ func (s *SQLiteStore) GetNodeDetails(nodeID string) (*models.Node, error) { // Fetch Installed Sensors (Added metadata and last_heartbeat) rows, err := s.DB.Query(` - SELECT sensor_id, custom_name, config_values, metadata, deployed_version, is_silenced, last_heartbeat + SELECT sensor_id, custom_name, config_values, deployed_version, is_silenced, last_heartbeat FROM node_sensors WHERE node_id = ?`, nodeID) if err != nil { return nil, err @@ -207,11 +206,10 @@ func (s *SQLiteStore) GetNodeDetails(nodeID string) (*models.Node, error) { for rows.Next() { var ns models.NodeSensor var configStr string - var metaStr string var silencedInt int var lastHb sql.NullString - if err := rows.Scan(&ns.Name, &ns.Display, &configStr, &metaStr, &ns.DeployedVersion, &silencedInt, &lastHb); err != nil { + if err := rows.Scan(&ns.Name, &ns.Display, &configStr, &ns.DeployedVersion, &silencedInt, &lastHb); err != nil { return nil, err } @@ -227,7 +225,6 @@ func (s *SQLiteStore) GetNodeDetails(nodeID string) (*models.Node, error) { ns.Status = deriveStatus(ns.LastHeartbeat) json.Unmarshal([]byte(configStr), &ns.EnvVars) - json.Unmarshal([]byte(metaStr), &ns.Metadata) // Count events in the last 24h for this specific sensor yesterday := time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339) diff --git a/Hub/internal/store/sensors.go b/Hub/internal/store/sensors.go index 4cb391d..82354bb 100644 --- a/Hub/internal/store/sensors.go +++ b/Hub/internal/store/sensors.go @@ -2,7 +2,6 @@ package store import ( "database/sql" - "encoding/json" "time" ) @@ -20,7 +19,7 @@ type HeartbeatData struct { } // ProcessHeartbeat safely handles node updates and config reconciliation -func (s *SQLiteStore) ProcessHeartbeat(nodeID, sensorID, agentRevision, nowStr, metadataStr string) (bool, error) { +func (s *SQLiteStore) ProcessHeartbeat(nodeID, sensorID, agentVersion, contractVersion, configRev, nowStr string) (bool, error) { tx, err := s.DB.Begin() if err != nil { return false, err @@ -35,9 +34,9 @@ func (s *SQLiteStore) ProcessHeartbeat(nodeID, sensorID, agentRevision, nowStr, // Update the specific Sensor's heartbeat AND metadata if _, err := tx.Exec(` UPDATE node_sensors - SET metadata = ?, last_heartbeat = ?, updated_at = ? + SET agent_version = ?, contract_version = ?, config_rev = ?, last_heartbeat = ?, updated_at = ? WHERE node_id = ? AND sensor_id = ?`, - metadataStr, nowStr, nowStr, nodeID, sensorID); err != nil { + agentVersion, contractVersion, configRev, nowStr, nowStr, nodeID, sensorID); err != nil { return false, err } @@ -54,22 +53,17 @@ func (s *SQLiteStore) ProcessHeartbeat(nodeID, sensorID, agentRevision, nowStr, targetRevision = desiredRevision.String } - if targetRevision != "" && targetRevision == agentRevision { - rows, err := tx.Query("SELECT metadata FROM node_sensors WHERE node_id = ?", nodeID) + if targetRevision != "" && targetRevision == configRev { + rows, err := tx.Query("SELECT config_rev FROM node_sensors WHERE node_id = ?", nodeID) if err == nil { allMatched := true for rows.Next() { - var metaStr string - if err := rows.Scan(&metaStr); err != nil { + var rev string + if err := rows.Scan(&rev); err != nil { allMatched = false break } - var meta map[string]interface{} - if err := json.Unmarshal([]byte(metaStr), &meta); err != nil { - allMatched = false - break - } - if rev, ok := meta["HW_CONFIG_REV"].(string); !ok || rev != targetRevision { + if rev != targetRevision { allMatched = false break } diff --git a/Hub/ui/src/stores/Fleet/fleet.ts b/Hub/ui/src/stores/Fleet/fleet.ts index 6b4528c..09ccf92 100644 --- a/Hub/ui/src/stores/Fleet/fleet.ts +++ b/Hub/ui/src/stores/Fleet/fleet.ts @@ -8,7 +8,6 @@ export interface RawSensorPayload { status?: string isSilenced?: boolean envVars?: Record - metadata?: Record deployedVersion?: string updateAvailable?: boolean lastHeartbeat?: string | null @@ -61,7 +60,6 @@ export interface InstalledSensor { status: string isSilenced: boolean envVars: Record - metadata: Record deployedVersion?: string updateAvailable?: boolean lastHeartbeat: string | null @@ -206,8 +204,8 @@ export const useFleetStore = defineStore('fleet', () => { return { ...sensor, display: manifest?.name || sensor.display || sensor.name || '', - icon: manifest?.icon_svg || sensor.metadata?.icon || DEFAULT_SENSOR_ICON, - osi: manifest?.osi_layer || sensor.metadata?.osi || 'Other', + icon: manifest?.icon_svg || DEFAULT_SENSOR_ICON, + osi: manifest?.osi_layer || 'Other', status: (node.status === 'down' && sensor.status === 'pending') ? 'down' : sensor.status } }) @@ -373,7 +371,6 @@ export const useFleetStore = defineStore('fleet', () => { deployedVersion: raw.deployedVersion || '', updateAvailable: raw.updateAvailable || false, envVars: raw.envVars || {}, - metadata: raw.metadata || {}, lastHeartbeat: raw.lastHeartbeat || null, } } diff --git a/SDKs/go-honeywire/sdk.go b/SDKs/go-honeywire/sdk.go index d414e47..ababe51 100644 --- a/SDKs/go-honeywire/sdk.go +++ b/SDKs/go-honeywire/sdk.go @@ -390,12 +390,10 @@ func (s *Sensor) heartbeatLoop() { func (s *Sensor) sendHeartbeat() (*http.Response, error) { payload := map[string]any{ - "sensorId": s.SensorID, - "metadata": map[string]string{ - "agent_version": s.AgentVersion, - "contract_version": s.hubContractVersion, - "HW_CONFIG_REV": s.ConfigRev, - }, + "sensorId": s.SensorID, + "agentVersion": s.AgentVersion, + "contractVersion": s.hubContractVersion, + "configRev": s.ConfigRev, } return s.postToHub("/api/v1/heartbeat", payload) } diff --git a/SDKs/python-honeywire/honeywire/__init__.py b/SDKs/python-honeywire/honeywire/__init__.py index 47d720c..747bbf8 100644 --- a/SDKs/python-honeywire/honeywire/__init__.py +++ b/SDKs/python-honeywire/honeywire/__init__.py @@ -317,11 +317,9 @@ class HoneyWireSensor(ABC): payload = { "sensorId": self.sensor_id, - "metadata": { - "agent_version": self.agent_version, - "contract_version": self._hub_contract_version, - "HW_CONFIG_REV": self.config_rev - } + "agentVersion": self.agent_version, + "contractVersion": self._hub_contract_version, + "configRev": self.config_rev } resp, exc = None, None