remove minimap adjust resolution of event log warnings

This commit is contained in:
its-a-feature
2026-05-12 16:00:11 -06:00
parent ff6e856988
commit b8000b9630
12 changed files with 212 additions and 85 deletions
@@ -14,7 +14,7 @@ import { Typography } from '@mui/material';
import { ReactFlow,
EdgeLabelRenderer,getBezierPath, BaseEdge,
Handle, Position, useReactFlow, ReactFlowProvider, Panel,
MiniMap, Controls, ControlButton, useUpdateNodeInternals,
Controls, ControlButton, useUpdateNodeInternals,
getConnectedEdges, useNodesState, useEdgesState
} from '@xyflow/react';
import '@xyflow/react/dist/style.css';
@@ -2687,7 +2687,6 @@ export const DrawC2PathElementsFlow = ({edges, panel, view_config, contextMenu,
}
}, [graphData, view_config, setNodes, setEdgeFlow, updateNodeInternals, fitView]);
const onlyRenderVisibleGraphElements = nodes.length > 20;
const showMiniMap = nodes.length <= 250;
return (
<div className="mythic-graph-canvas mythic-c2-flow-canvas" style={{height: "100%", width: "100%", position: "relative"}} ref={viewportRef}>
<ReactFlow
@@ -2715,9 +2714,6 @@ export const DrawC2PathElementsFlow = ({edges, panel, view_config, contextMenu,
<InsertPhotoIcon />
</ControlButton>
</Controls>
{showMiniMap &&
<MiniMap pannable={true} zoomable={true} />
}
</ReactFlow>
{openContextMenu && typeof document !== "undefined" && createPortal(
<div style={{...contextMenuCoord, position: "fixed"}} className="context-menu mythic-graph-context-menu">
@@ -3437,7 +3433,6 @@ export const DrawBrowserScriptElementsFlow = ({edges, panel, view_config, theme,
<RestartAltIcon />
</ControlButton>
</Controls>
<MiniMap pannable={true} zoomable={true} />
</ReactFlow>
{openContextMenu && typeof document !== "undefined" && createPortal(
<div style={{...contextMenuCoord, position: "fixed"}} className="context-menu mythic-graph-context-menu">
@@ -88,6 +88,10 @@ on "public"."mythictree" using btree (operation_id, tree_type, "timestamp");
create index if not exists mythictree_tree_deleted_timestamp_idx
on "public"."mythictree" using btree (tree_type, deleted, "timestamp");
create index if not exists operationeventlog_unresolved_warning_source_operation_level_idx
on "public"."operationeventlog" using btree (source, operation_id, "level")
where warning=true and resolved=false and deleted=false;
alter table "public"."task"
add column if not exists subtask_callback_function_started boolean not null default false,
add column if not exists group_callback_function_started boolean not null default false,
@@ -222,6 +226,7 @@ drop index if exists "public"."mythictree_tree_deleted_timestamp_idx";
drop index if exists "public"."mythictree_operation_tree_timestamp_idx";
drop index if exists "public"."mythictree_operation_tree_host_full_callback_idx";
drop index if exists "public"."mythictree_operation_tree_host_parent_callback_idx";
drop index if exists "public"."operationeventlog_unresolved_warning_source_operation_level_idx";
alter table "public"."task"
drop column if exists completed_callback_function_started,
drop column if exists group_callback_function_started,
@@ -313,10 +313,10 @@ func checkContainerStatus() {
if !running {
SendAllOperationsMessage(
getDownContainerMessage(container),
0, fmt.Sprintf("%s_container_down", container), database.MESSAGE_LEVEL_INFO, true)
0, getDownContainerSource(container), database.MESSAGE_LEVEL_INFO, true)
go updateDownContainerBuildingPayloads(container)
} else {
go ResolveAllOperationsMessage(getDownContainerMessage(container), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(container), 0)
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(container)
}
} else {
@@ -346,9 +346,9 @@ func checkContainerStatus() {
UpdateC2ProfileRunningStatus(c2profilesToCheck[container], false)
SendAllOperationsMessage(
getDownContainerMessage(container),
0, fmt.Sprintf("%s_container_down", container), database.MESSAGE_LEVEL_INFO, true)
0, getDownContainerSource(container), database.MESSAGE_LEVEL_INFO, true)
} else {
go ResolveAllOperationsMessage(getDownContainerMessage(container), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(container), 0)
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(container)
}
} else {
@@ -377,9 +377,9 @@ func checkContainerStatus() {
if !running {
SendAllOperationsMessage(
getDownContainerMessage(container),
0, fmt.Sprintf("%s_container_down", container), database.MESSAGE_LEVEL_INFO, true)
0, getDownContainerSource(container), database.MESSAGE_LEVEL_INFO, true)
} else {
go ResolveAllOperationsMessage(getDownContainerMessage(container), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(container), 0)
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(container)
}
} else {
@@ -409,9 +409,9 @@ func checkContainerStatus() {
if !running {
SendAllOperationsMessage(
getDownContainerMessage(container),
0, fmt.Sprintf("%s_container_down", container), database.MESSAGE_LEVEL_INFO, true)
0, getDownContainerSource(container), database.MESSAGE_LEVEL_INFO, true)
} else {
go ResolveAllOperationsMessage(getDownContainerMessage(container), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(container), 0)
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(container)
}
} else {
@@ -440,9 +440,9 @@ func checkContainerStatus() {
if !running {
SendAllOperationsMessage(
getDownContainerMessage(container),
0, fmt.Sprintf("%s_container_down", container), database.MESSAGE_LEVEL_INFO, true)
0, getDownContainerSource(container), database.MESSAGE_LEVEL_INFO, true)
} else {
go ResolveAllOperationsMessage(getDownContainerMessage(container), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(container), 0)
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(container)
}
} else {
@@ -461,6 +461,10 @@ func getDownContainerMessage(containerName string) string {
return fmt.Sprintf("Error: Can't contact %s", containerName)
}
func getDownContainerSource(containerName string) string {
return fmt.Sprintf("%s_container_down", containerName)
}
func UpdateC2ProfileRunningStatus(c2Profile databaseStructs.C2profile, running bool) {
if _, err := database.DB.Exec(`UPDATE c2profile SET running=$1 WHERE id=$2`, running, c2Profile.ID); err != nil {
logging.LogError(err, "Failed to update C2 profile running status", "c2_profile", c2Profile.ID)
+1 -1
View File
@@ -191,7 +191,7 @@ func c2Sync(in C2SyncMessage) error {
return err
}
go SendAllOperationsMessage(fmt.Sprintf("Successfully synced %s with container version %s", c2Profile.Name, in.ContainerVersion), 0, "debug", database.MESSAGE_LEVEL_DEBUG, false)
go ResolveAllOperationsMessage(getDownContainerMessage(c2Profile.Name), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(c2Profile.Name), 0)
go autoStartC2Profile(c2Profile, false)
if newProfile {
go reSyncPayloadTypes()
@@ -134,7 +134,7 @@ func consumingServicesSync(in ConsumingContainerSyncMessage) error {
}
go SendAllOperationsMessage(fmt.Sprintf("Successfully synced %s with container version %s",
consumingContainer.Name, in.ContainerVersion), 0, "debug", database.MESSAGE_LEVEL_DEBUG, false)
go ResolveAllOperationsMessage(getDownContainerMessage(consumingContainer.Name), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(consumingContainer.Name), 0)
checkContainerStatusAddConsumingContainerChannel <- consumingContainer
// update eventgroup consumingcontainer mappings
eventing.UpdateEventGroupConsumingContainersMappingByConsumingContainer(consumingContainer)
@@ -190,7 +190,7 @@ func customBrowserSync(in CustomBrowserSyncMessage) error {
return err
}
go SendAllOperationsMessage(fmt.Sprintf("Successfully synced %s with container version %s", customSyncBrowser.Name, in.ContainerVersion), 0, "debug", database.MESSAGE_LEVEL_DEBUG, false)
go ResolveAllOperationsMessage(getDownContainerMessage(customSyncBrowser.Name), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(customSyncBrowser.Name), 0)
checkContainerStatusAddCustomBrowserChannel <- customSyncBrowser
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(customSyncBrowser.Name)
+1 -1
View File
@@ -450,7 +450,7 @@ func payloadTypeSync(in PayloadTypeSyncMessage) error {
InvalidateAllCachedUUIDInfo()
}
go SendAllOperationsMessage(fmt.Sprintf("Successfully synced %s with container version %s", payloadtype.Name, in.ContainerVersion), 0, "debug", database.MESSAGE_LEVEL_DEBUG, false)
go ResolveAllOperationsMessage(getDownContainerMessage(payloadtype.Name), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(payloadtype.Name), 0)
checkContainerStatusAddPtChannel <- payloadtype
if !in.ForcedSync {
go CreateGraphQLSpectatorAPITokenAndSendOnStartMessage(payloadtype.Name)
+1 -1
View File
@@ -56,7 +56,7 @@ func processTrSyncMessages(msg amqp.Delivery) interface{} {
// successfully synced
response.Success = true
go SendAllOperationsMessage(fmt.Sprintf("Successfully synced %s with container version %s", trSyncMsg.Name, trSyncMsg.ContainerVersion), 0, "debug", database.MESSAGE_LEVEL_DEBUG, false)
go ResolveAllOperationsMessage(getDownContainerMessage(trSyncMsg.Name), 0)
go ResolveAllOperationsMessageBySource(getDownContainerSource(trSyncMsg.Name), 0)
logging.LogDebug("Successfully synced", "service", trSyncMsg.Name)
}
return response
@@ -6,10 +6,10 @@ import (
"errors"
"fmt"
"math"
"sort"
"time"
"github.com/its-a-feature/Mythic/database"
databaseStructs "github.com/its-a-feature/Mythic/database/structs"
"github.com/its-a-feature/Mythic/logging"
"github.com/jmoiron/sqlx"
)
@@ -196,6 +196,18 @@ func shouldAgentMessageGetDelegateTasks(incoming map[string]interface{}) bool {
return parsedGetDelegateTasks
}
func selectAgentMessageTaskIDsForIssue(taskIDs []int, taskingSize int) []int {
if len(taskIDs) == 0 || taskingSize == 0 {
return nil
}
taskIDsToIssue := cloneTaskIDs(taskIDs)
sort.Ints(taskIDsToIssue)
if taskingSize < 0 || taskingSize >= len(taskIDsToIssue) {
return taskIDsToIssue
}
return taskIDsToIssue[:taskingSize]
}
func handleAgentMessageGetTasking(incoming *map[string]interface{}, callbackID int) (map[string]interface{}, error) {
// got message:
/*
@@ -211,63 +223,33 @@ func handleAgentMessageGetTasking(incoming *map[string]interface{}, callbackID i
1. check for direct tasks
2. check for delegate tasks
*/
currentTasks := []databaseStructs.Task{}
if taskIDs := submittedTasksAwaitingFetching.getTasksForCallbackId(callbackID); len(taskIDs) > 0 {
query, args, err := sqlx.Named(`SELECT
agent_task_id, "timestamp", command_name, params, id, token_id
FROM task WHERE id IN (:ids) ORDER BY id ASC`, map[string]interface{}{
"ids": taskIDs,
})
if err != nil {
logging.LogError(err, "Failed to make named statement when searching for tasks")
return nil, errors.New("failed to make statement to search for tasks")
}
query, args, err = sqlx.In(query, args...)
if err != nil {
logging.LogError(err, "Failed to do sqlx.In")
return nil, errors.New("failed to make query to search for tasks")
}
query = database.DB.Rebind(query)
err = database.DB.Select(&currentTasks, query, args...)
if err != nil {
logging.LogError(err, "Failed to exec sqlx.IN modified statement")
return nil, errors.New("failed to search for tasks")
}
}
agentMessage, err := decodeAgentMessageGetTasking(*incoming)
if err != nil {
logging.LogError(err, "Failed to decode agent message into struct")
return nil, errors.New(fmt.Sprintf("Failed to decode agent message into agentMessageGetTasking struct: %s", err.Error()))
}
tasksToIssue := []agentMessageGetTaskingTask{}
currentTaskCount := 0
issuedTaskIDs := []int{}
currentTasks := []agentMessageTaskRow{}
if taskIDs := selectAgentMessageTaskIDsForIssue(submittedTasksAwaitingFetching.getTasksForCallbackId(callbackID), agentMessage.TaskingSize); len(taskIDs) > 0 {
currentTasks, err = getAgentMessageTaskRows(taskIDs)
if err != nil {
logging.LogError(err, "Failed to fetch direct tasking")
return nil, errors.New("failed to search for tasks")
}
}
for _, task := range currentTasks {
if currentTaskCount < agentMessage.TaskingSize || agentMessage.TaskingSize < 0 {
newTask := agentMessageGetTaskingTask{
Command: task.CommandName,
Parameters: task.Params,
ID: task.AgentTaskID,
Timestamp: task.Timestamp.Unix(),
tasksToIssue = append(tasksToIssue, buildAgentMessageTask(task))
issuedTaskIDs = append(issuedTaskIDs, task.ID)
}
if len(issuedTaskIDs) > 0 {
if err := markAgentMessageTasksProcessing(issuedTaskIDs, time.Now().UTC()); err != nil {
logging.LogError(err, "Failed to update direct task status to processing")
} else {
submittedTasksAwaitingFetching.removeTasksAfterProcessingUpdate(issuedTaskIDs)
for _, taskID := range issuedTaskIDs {
go addMitreAttackTaskMapping(taskID)
}
if task.TokenID.Valid {
var tokenID int
if err := database.DB.Get(&tokenID, `SELECT token_id FROM token WHERE id=$1`, task.TokenID.Int64); err != nil {
logging.LogError(err, "failed to get token information")
} else {
newTask.Token = &tokenID
}
}
tasksToIssue = append(tasksToIssue, newTask)
if _, err := database.DB.Exec(`UPDATE task SET
status=$2, status_timestamp_processing=$3
WHERE id=$1`, task.ID, PT_TASK_FUNCTION_STATUS_PROCESSING, time.Now().UTC()); err != nil {
logging.LogError(err, "Failed to update task status to processing")
} else {
submittedTasksAwaitingFetching.removeTask(task.ID)
go addMitreAttackTaskMapping(task.ID)
}
currentTaskCount += 1
}
}
response := map[string]interface{}{}
@@ -2,6 +2,7 @@ package rabbitmq
import (
"fmt"
"slices"
"testing"
"github.com/mitchellh/mapstructure"
@@ -113,6 +114,26 @@ func TestHandleAgentMessageGetTaskingReflectsOnlyUnusedKeys(t *testing.T) {
}
}
func TestSelectAgentMessageTaskIDsForIssueMatchesDirectTaskingOrder(t *testing.T) {
taskIDs := []int{5, 2, 9, 1}
if selected := selectAgentMessageTaskIDsForIssue(taskIDs, 0); len(selected) != 0 {
t.Fatalf("expected no issued task IDs for tasking_size 0, got %#v", selected)
}
if selected := selectAgentMessageTaskIDsForIssue(taskIDs, 2); !slices.Equal(selected, []int{1, 2}) {
t.Fatalf("expected first two task IDs in ascending DB order, got %#v", selected)
}
if selected := selectAgentMessageTaskIDsForIssue(taskIDs, -1); !slices.Equal(selected, []int{1, 2, 5, 9}) {
t.Fatalf("expected all task IDs in ascending DB order, got %#v", selected)
}
if selected := selectAgentMessageTaskIDsForIssue(taskIDs, 99); !slices.Equal(selected, []int{1, 2, 5, 9}) {
t.Fatalf("expected all task IDs when tasking_size exceeds pending count, got %#v", selected)
}
if !slices.Equal(taskIDs, []int{5, 2, 9, 1}) {
t.Fatalf("expected original task ID slice to remain unchanged, got %#v", taskIDs)
}
}
func BenchmarkDecodeAgentMessagePostResponseMessage(b *testing.B) {
benchmarks := []struct {
name string
+88 -14
View File
@@ -1369,6 +1369,21 @@ func addUnresolvedError(operationID int, source string, level string, messageID
func removeUnresolvedError(operationID int) {
delete(unresolvedErrors, operationID)
}
func removeUnresolvedErrorSource(operationID int, source string) {
if _, ok := unresolvedErrors[operationID]; !ok {
return
}
delete(unresolvedErrors[operationID], source)
if len(unresolvedErrors[operationID]) == 0 {
delete(unresolvedErrors, operationID)
}
}
func normalizeOperationMessageWarningLevel(level database.MESSAGE_TYPE, warning bool) (database.MESSAGE_TYPE, bool) {
if level == "warning" {
return database.MESSAGE_LEVEL_INFO, true
}
return level, warning
}
type operationMessage struct {
action string
@@ -1411,40 +1426,41 @@ func listenForOperationsMessages() {
if sourceString == "" {
sourceString = uuid.NewString()
}
storedMessageLevel, warning := normalizeOperationMessageWarningLevel(msg.messageLevel, msg.warning)
// hitting concurrency issues where like-messages aren't getting collapsed like they should
var err error
for _, operation := range operations {
if msg.operationID == 0 || operation.ID == msg.operationID {
// this is the operation we're interested in
if msg.messageLevel == "warning" || msg.warning {
if warning {
existingMessage := databaseStructs.Operationeventlog{}
existingMessage.ID = checkUnresolvedError(operation.ID, msg.source, msg.messageLevel)
existingMessage.ID = checkUnresolvedError(operation.ID, sourceString, storedMessageLevel)
if existingMessage.ID == 0 {
err = database.DB.Get(&existingMessage, `
SELECT id, count, "message", source, "level" FROM operationeventlog WHERE
warning=true and resolved=false and deleted=false and source=$1 and operation_id=$2 and "level"=$3
`, sourceString, operation.ID, msg.messageLevel)
`, sourceString, operation.ID, storedMessageLevel)
if !errors.Is(err, sql.ErrNoRows) && err != nil {
logging.LogError(err, "Failed to query existing event log message")
continue
}
if errors.Is(err, sql.ErrNoRows) {
var utf8Message = strings.ToValidUTF8(string(bytes.ReplaceAll([]byte(msg.message), []byte{0}, []byte{})), "*")
if msg.messageLevel == "warning" {
msg.messageLevel = database.MESSAGE_LEVEL_INFO
}
newMessage := databaseStructs.Operationeventlog{
Source: sourceString,
Level: msg.messageLevel,
Warning: msg.warning,
Level: storedMessageLevel,
Warning: warning,
Message: utf8Message,
OperationID: operation.ID,
Count: 1,
}
_, err = database.DB.NamedExec(`INSERT INTO operationeventlog
err = database.DB.QueryRowx(`INSERT INTO operationeventlog
(source, "level", "message", operation_id, count, warning)
VALUES
(:source, :level, :message, :operation_id, :count, :warning)`, newMessage)
($1, $2, $3, $4, $5, $6)
RETURNING id`,
newMessage.Source, newMessage.Level, newMessage.Message, newMessage.OperationID, newMessage.Count, newMessage.Warning,
).Scan(&newMessage.ID)
if err != nil {
logging.LogError(err, "Failed to create new operationeventlog message")
continue
@@ -1470,10 +1486,10 @@ func listenForOperationsMessages() {
"alert": newMessage.Message,
},
}
addUnresolvedError(operation.ID, msg.source, msg.messageLevel, existingMessage.ID)
addUnresolvedError(operation.ID, sourceString, storedMessageLevel, newMessage.ID)
continue
}
addUnresolvedError(operation.ID, msg.source, msg.messageLevel, existingMessage.ID)
addUnresolvedError(operation.ID, sourceString, storedMessageLevel, existingMessage.ID)
}
if time.Now().Sub(lastUpdateTime).Seconds() > 1 {
lastUpdateTime = time.Now()
@@ -1508,8 +1524,8 @@ func listenForOperationsMessages() {
var utf8Message = strings.ToValidUTF8(string(bytes.ReplaceAll([]byte(msg.message), []byte{0}, []byte{})), "*")
newMessage := databaseStructs.Operationeventlog{
Source: sourceString,
Level: msg.messageLevel,
Warning: msg.warning,
Level: storedMessageLevel,
Warning: warning,
Message: utf8Message,
OperationID: operation.ID,
Count: 1,
@@ -1541,6 +1557,55 @@ func listenForOperationsMessages() {
removeUnresolvedError(operation.ID)
}
}
case "remove_source":
if msg.source == "" {
continue
}
if msg.operationID == 0 {
rows, err := database.DB.Queryx(`UPDATE operationeventlog SET resolved=true
WHERE warning=true AND resolved=false AND deleted=false AND source=$1
RETURNING operation_id`, msg.source)
if err != nil {
logging.LogError(err, "Failed to resolve messages by source")
continue
}
for rows.Next() {
operationID := 0
if err := rows.Scan(&operationID); err != nil {
logging.LogError(err, "Failed to read resolved operation id")
continue
}
removeUnresolvedErrorSource(operationID, msg.source)
}
if err := rows.Err(); err != nil {
logging.LogError(err, "Failed to iterate resolved operation ids")
}
if err := rows.Close(); err != nil {
logging.LogError(err, "Failed to close resolve messages by source rows")
}
continue
}
rows, err := database.DB.Queryx(`UPDATE operationeventlog SET resolved=true
WHERE warning=true AND resolved=false AND deleted=false AND source=$1 AND operation_id=$2
RETURNING operation_id`, msg.source, msg.operationID)
if err != nil {
logging.LogError(err, "Failed to resolve operation message by source")
continue
}
for rows.Next() {
operationID := 0
if err := rows.Scan(&operationID); err != nil {
logging.LogError(err, "Failed to read resolved operation id")
continue
}
removeUnresolvedErrorSource(operationID, msg.source)
}
if err := rows.Err(); err != nil {
logging.LogError(err, "Failed to iterate resolved operation ids")
}
if err := rows.Close(); err != nil {
logging.LogError(err, "Failed to close resolve operation message by source rows")
}
}
case <-timer.C:
if len(unresolvedErrors) > 0 {
@@ -1571,3 +1636,12 @@ func ResolveAllOperationsMessage(message string, operationID int) {
}:
}
}
func ResolveAllOperationsMessageBySource(source string, operationID int) {
select {
case operationMessageChannel <- operationMessage{
action: "remove_source",
operationID: operationID,
source: source,
}:
}
}
@@ -0,0 +1,46 @@
package rabbitmq
import (
"testing"
"github.com/its-a-feature/Mythic/database"
)
func TestOperationMessageWarningLevelNormalization(t *testing.T) {
level, warning := normalizeOperationMessageWarningLevel("warning", false)
if level != database.MESSAGE_LEVEL_INFO || !warning {
t.Fatalf("expected warning level to normalize to info warning=true, got level=%q warning=%v", level, warning)
}
level, warning = normalizeOperationMessageWarningLevel(database.MESSAGE_LEVEL_AGENT_MESSGAGE, true)
if level != database.MESSAGE_LEVEL_AGENT_MESSGAGE || !warning {
t.Fatalf("expected explicit warning flag to preserve level, got level=%q warning=%v", level, warning)
}
}
func TestUnresolvedErrorCacheUsesSourceAndLevel(t *testing.T) {
previousUnresolvedErrors := unresolvedErrors
unresolvedErrors = make(map[int]map[string]map[string]int)
t.Cleanup(func() {
unresolvedErrors = previousUnresolvedErrors
})
addUnresolvedError(10, "container_down", database.MESSAGE_LEVEL_INFO, 42)
if messageID := checkUnresolvedError(10, "container_down", database.MESSAGE_LEVEL_INFO); messageID != 42 {
t.Fatalf("expected cached message ID 42, got %d", messageID)
}
if messageID := checkUnresolvedError(10, "container_down", database.MESSAGE_LEVEL_AGENT_MESSGAGE); messageID != 0 {
t.Fatalf("expected different level to miss cache, got %d", messageID)
}
removeUnresolvedErrorSource(10, "container_down")
if messageID := checkUnresolvedError(10, "container_down", database.MESSAGE_LEVEL_INFO); messageID != 0 {
t.Fatalf("expected source removal to clear cache, got %d", messageID)
}
}
func TestDownContainerSourceIsStable(t *testing.T) {
if source := getDownContainerSource("poseidon"); source != "poseidon_container_down" {
t.Fatalf("expected stable down-container source, got %q", source)
}
}