diff --git a/.env b/.env index aff3d58..b862381 100644 --- a/.env +++ b/.env @@ -9,3 +9,5 @@ CONFIG_DIR=./deployment CONFIG_FILE=./config.hjson SYSLOG_ADDRESS=localhost:514 DB_ADDRESS=localhost:9000 +LOGGING_ENABLED=true +LOG_LEVEL=1 diff --git a/.env.production b/.env.production index 698e5bd..f57b870 100644 --- a/.env.production +++ b/.env.production @@ -9,3 +9,6 @@ CONFIG_FILE=/etc/rita/config.hjson SYSLOG_ADDRESS=syslogng:5514 APP_LOGS=/var/log/rita DB_ADDRESS=db:9000 +LOGGING_ENABLED=true +LOG_LEVEL=1 + diff --git a/analysis/analysis.go b/analysis/analysis.go index f8737d6..4fc3140 100644 --- a/analysis/analysis.go +++ b/analysis/analysis.go @@ -283,10 +283,14 @@ func (analyzer *Analyzer) runAnalysis() error { timeSince := relativeTime.Sub(entry.FirstSeenHistorical) daysSinceFirstSeen := float32(timeSince.Hours() / 24) - if daysSinceFirstSeen <= analyzer.Config.Modifiers.FirstSeenIncreaseThreshold { - mixtape.FirstSeenScore = analyzer.Config.Modifiers.FirstSeenScoreIncrease - } else if daysSinceFirstSeen >= analyzer.Config.Modifiers.FirstSeenDecreaseThreshold { - mixtape.FirstSeenScore = -1 * analyzer.Config.Modifiers.FirstSeenScoreDecrease + // Historical First Seen Scoring + // only apply to rolling datasets + if analyzer.Database.Rolling { + if daysSinceFirstSeen <= analyzer.Config.Modifiers.FirstSeenIncreaseThreshold { + mixtape.FirstSeenScore = analyzer.Config.Modifiers.FirstSeenScoreIncrease + } else if daysSinceFirstSeen >= analyzer.Config.Modifiers.FirstSeenDecreaseThreshold { + mixtape.FirstSeenScore = -1 * analyzer.Config.Modifiers.FirstSeenScoreDecrease + } } // Prevalence Scoring diff --git a/config/config.go b/config/config.go index 99c826c..061fb57 100644 --- a/config/config.go +++ b/config/config.go @@ -130,9 +130,6 @@ type ( Modifiers Modifiers `json:"modifiers"` ThreatIntel ThreatIntel `json:"threat_intel"` - - LogLevel int `json:"log_level"` - LoggingEnabled bool `json:"logging_enabled"` } ) @@ -473,11 +470,6 @@ func (cfg *Config) verifyConfig() error { return fmt.Errorf("the MIME type/URI mismatch score increase must be between 0 and 1, got %v", cfg.Modifiers.MIMETypeMismatchScoreIncrease) } - // validate log level - if cfg.LogLevel < -1 || cfg.LogLevel > 5 { - return fmt.Errorf("the LogLevel must be between -1 and 5 (inclusive)") - } - return nil } @@ -651,7 +643,5 @@ func defaultConfig() Config { OnlineFeeds: []string{}, CustomFeedsDirectory: "/etc/rita/threat_intel_feeds", }, - LogLevel: 1, // INFO level is default - LoggingEnabled: true, // enable logging by default } } diff --git a/config/config_test.go b/config/config_test.go index d91a790..483e090 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -136,8 +136,6 @@ func TestParseJSON(t *testing.T) { c2_over_dns_direct_conn_score_increase: 0.9, mime_type_mismatch_score_increase: 0.6 }, - log_level: 3, - logging_enabled: false, } `), expectedConfig: Config{ @@ -240,8 +238,6 @@ func TestParseJSON(t *testing.T) { OnlineFeeds: []string{"https://example.com/feed1", "https://example.com/feed2"}, CustomFeedsDirectory: "/path/to/custom/feeds", }, - LogLevel: 3, - LoggingEnabled: false, }, expectedError: nil, }, @@ -379,9 +375,6 @@ func TestParseJSON(t *testing.T) { require.InDelta(test.expectedConfig.Modifiers.RareSignatureScoreIncrease, cfg.Modifiers.RareSignatureScoreIncrease, 0.00001, "RareSignatureScoreIncrease should match expected value") require.InDelta(test.expectedConfig.Modifiers.C2OverDNSDirectConnScoreIncrease, cfg.Modifiers.C2OverDNSDirectConnScoreIncrease, 0.00001, "C2OverDNSDirectConnScoreIncrease should match expected value") require.InDelta(test.expectedConfig.Modifiers.MIMETypeMismatchScoreIncrease, cfg.Modifiers.MIMETypeMismatchScoreIncrease, 0.00001, "MIMETypeMismatchScoreIncrease should match expected value") - - require.Equal(test.expectedConfig.LogLevel, cfg.LogLevel, "LogLevel should match expected value") - require.Equal(test.expectedConfig.LoggingEnabled, cfg.LoggingEnabled, "LoggingEnabled should match expected value") }) } } @@ -655,8 +648,6 @@ func TestGetDefaultConfig(t *testing.T) { require.Equal(origConfigVar.Scoring, cfg.Scoring, "config scoring should match expected value") require.Equal(origConfigVar.Modifiers, cfg.Modifiers, "config modifiers should match expected value") require.Equal(origConfigVar.ThreatIntel, cfg.ThreatIntel, "config threat intel should match expected value") - require.Equal(origConfigVar.LogLevel, cfg.LogLevel, "config log level should match expected value") - require.Equal(origConfigVar.LoggingEnabled, cfg.LoggingEnabled, "config logging enabled should match expected value") // match the whole object just in case require.Equal(origConfigVar, cfg, "config should match expected value") diff --git a/database/analysis_tables.go b/database/analysis_tables.go index 477d087..ef37ff7 100644 --- a/database/analysis_tables.go +++ b/database/analysis_tables.go @@ -257,13 +257,16 @@ func (db *DB) createRareSignatureTable(ctx context.Context) error { hour DateTime(), src IPv6, src_nuid UUID, + dst IPv6, + dst_nuid UUID, + fqdn String, signature String, is_ja3 Bool, times_used_dst AggregateFunction(uniqExact, IPv6), times_used_fqdn AggregateFunction(uniqExact, String) ) ENGINE = AggregatingMergeTree() - PRIMARY KEY (src_nuid, src, signature ) + PRIMARY KEY (src_nuid, src, dst, dst_nuid, fqdn, signature ) `) if err != nil { @@ -278,13 +281,14 @@ func (db *DB) createRareSignatureTable(ctx context.Context) error { toStartOfHour(ts) as hour, src, src_nuid, + host AS fqdn, useragent as signature, false as is_ja3, uniqExactState(dst) as times_used_dst, uniqExactState(host) as times_used_fqdn FROM {database:Identifier}.http WHERE length(useragent) > 0 - GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3) + GROUP BY (import_hour, hour, src, src_nuid, fqdn, signature, is_ja3) `) if err != nil { @@ -299,13 +303,14 @@ func (db *DB) createRareSignatureTable(ctx context.Context) error { toStartOfHour(ts) as hour, src, src_nuid, + server_name AS fqdn, ja3 as signature, true as is_ja3, uniqExactState(dst) as times_used_dst, uniqExactState(server_name) as times_used_fqdn FROM {database:Identifier}.ssl WHERE length(ja3) > 0 - GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3) + GROUP BY (import_hour, hour, src, src_nuid, fqdn, signature, is_ja3) `) if err != nil { return err @@ -319,12 +324,14 @@ func (db *DB) createRareSignatureTable(ctx context.Context) error { toStartOfHour(ts) as hour, src, src_nuid, + dst, + dst_nuid, missing_host_useragent as signature, false as is_ja3, uniqExactState(if(src_local, dst, src)) as times_used_dst FROM {database:Identifier}.conn WHERE length(missing_host_useragent) > 0 AND missing_host_header = true - GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3) + GROUP BY (import_hour, hour, src, src_nuid, dst, dst_nuid, signature, is_ja3) `) if err != nil { return err @@ -333,7 +340,6 @@ func (db *DB) createRareSignatureTable(ctx context.Context) error { return err } - func (db *DB) createPortInfoTable(ctx context.Context) error { if err := db.Conn.Exec(ctx, `--sql diff --git a/default_config.hjson b/default_config.hjson index 297a0a6..0903b2f 100644 --- a/default_config.hjson +++ b/default_config.hjson @@ -1,7 +1,5 @@ { update_check_enabled: true, - log_level: 1, - logging_enabled: true, threat_intel: { // Configuration for custom threat intel feeds // Allowed format for the contents of both online feeds and custom file feeds is one IP or domain per line diff --git a/importer/importer.go b/importer/importer.go index 16f9ee8..fec0a2e 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -28,7 +28,7 @@ import ( "golang.org/x/time/rate" ) -var errAllFilesPreviouslyImported = errors.New("all files were previously imported") +var ErrAllFilesPreviouslyImported = errors.New("all files were previously imported") type zeekRecord interface { zeektypes.Conn | zeektypes.DNS | zeektypes.HTTP | zeektypes.SSL @@ -221,7 +221,7 @@ func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error // verify that there are still files left to import and set file count if totalFileCount < 1 { - return errAllFilesPreviouslyImported + return ErrAllFilesPreviouslyImported } importer.TotalFileCount = totalFileCount diff --git a/integration/analysis_view_test.go b/integration/analysis_view_test.go index b955c5b..86ecd7c 100644 --- a/integration/analysis_view_test.go +++ b/integration/analysis_view_test.go @@ -1053,6 +1053,26 @@ func (it *ValidDatasetTestSuite) TestThreatMixtape() { require.NoError(t, err) require.EqualValues(t, 4668, count, "threat mixtape should have one non-modifier row per unique hash, got: %d", count) + err = it.db.Conn.QueryRow(it.db.GetContext(), ` + SELECT count() FROM ( + SELECT hash, count() as c FROM threat_mixtape + WHERE modifier_name = 'rare_signature' + GROUP BY hash + ) WHERE c > 1 + `).Scan(&count) + require.NoError(t, err) + require.EqualValues(t, 0, count, "threat mixtape should have at most one rare_signature modifier row per unique hash, got: %d", count) + + err = it.db.Conn.QueryRow(it.db.GetContext(), ` + SELECT count() FROM ( + SELECT hash, count() as c FROM threat_mixtape + WHERE modifier_name = 'mime_type_mismatch' + GROUP BY hash + ) WHERE c > 1 + `).Scan(&count) + require.NoError(t, err) + require.EqualValues(t, 0, count, "threat mixtape should have at most one mime_type_mismatch modifier row per unique hash, got: %d", count) + err = it.db.Conn.QueryRow(it.db.GetContext(), ` SELECT count() FROM threat_mixtape WHERE beacon_type != 'dns' AND count != open_count @@ -1086,10 +1106,10 @@ func (it *ValidDatasetTestSuite) TestThreatMixtape() { err = it.db.Conn.QueryRow(chCtx, ` SELECT count() FROM threat_mixtape - WHERE modifier_name = '' AND first_seen_score != {first_seen_increase_score:Float32} + WHERE first_seen_score != 0 `).Scan(&count) require.NoError(t, err) - require.EqualValues(t, 0, count, "no non-modifier entries should have a historical first seen score other than the increase score") + require.EqualValues(t, 0, count, "no entries should have a historical first seen score for a non-rolling dataset") err = it.db.Conn.QueryRow(chCtx, ` SELECT count(DISTINCT import_id) FROM threat_mixtape diff --git a/integration/db_test.go b/integration/db_test.go index aa21112..f97b824 100644 --- a/integration/db_test.go +++ b/integration/db_test.go @@ -49,7 +49,7 @@ func (it *ValidDatasetTestSuite) TestPrimaryKeySize() { "openconnhash_tmp": {NumParts: 2, TotalMarks: 40, AvgMarks: 40, TotalPrimaryKeySize: 2000, CompressionRatio: 0.4}, "ssl": {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 1500, CompressionRatio: 0.7}, "ssl_tmp": {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 1500, CompressionRatio: 0.7}, - "threat_mixtape": {NumParts: 4, TotalMarks: 10, AvgMarks: 5, TotalPrimaryKeySize: 700, CompressionRatio: 0.75}, + "threat_mixtape": {NumParts: 4, TotalMarks: 10, AvgMarks: 5, TotalPrimaryKeySize: 700, CompressionRatio: 0.7}, "tls_proto": {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 600, CompressionRatio: 0.7}, "uconn": {NumParts: 2, TotalMarks: 15, AvgMarks: 15, TotalPrimaryKeySize: 800, CompressionRatio: 0.5}, "uconn_tmp": {NumParts: 2, TotalMarks: 60, AvgMarks: 60, TotalPrimaryKeySize: 2000, CompressionRatio: 0.35}, diff --git a/integration/missing_host_test.go b/integration/missing_host_test.go index ff62774..69b2347 100644 --- a/integration/missing_host_test.go +++ b/integration/missing_host_test.go @@ -107,7 +107,7 @@ func (it *MissingHostSuite) TestThreat() { } expected := expectedResults{ src: "73.54.23.243", dst: "64.225.56.201", - finalScore: 1.05, beaconScore: 0.963, longConnScore: config.HIGH_CATEGORY_SCORE, + finalScore: 0.9000, beaconScore: 0.963, longConnScore: config.HIGH_CATEGORY_SCORE, totalDuration: 270036.631115, count: 331, missingHostCount: 330, missingHostScore: 0.1, totalBytes: 1679442, portProtoService: []string{"80:tcp:http", "22:tcp:ssh"}, prevalence: 1, firstSeen: time.Unix(1713470872, 0).UTC(), @@ -162,7 +162,7 @@ func (it *MissingHostSuite) TestThreat() { require.InDelta(t, expected.missingHostScore, res.MissingHostHeaderScore, 0.001, "missing host score should match") require.InDelta(t, expected.prevalence, res.Prevalence, 0.001, "prevalence should match") require.InDelta(t, -it.cfg.Modifiers.PrevalenceScoreDecrease, res.PrevalenceScore, 0.001, "prevalence score should equal the prevalence decrease config value") - require.InDelta(t, it.cfg.Modifiers.FirstSeenScoreIncrease, res.FirstSeenScore, 0.001, "first seen score should equal the first seen score increase config value") + require.EqualValues(t, 0, res.FirstSeenScore, 0.001, "first seen score should equal 0 for a non-rolling dataset") require.EqualValues(t, expected.firstSeen.UTC(), res.FirstSeen, "first seen date should match") require.InDelta(t, it.cfg.Modifiers.MissingHostCountScoreIncrease, res.MissingHostHeaderScore, 0.001, "missing host header score should equal the missing host header increase score config value") require.InDelta(t, it.cfg.Modifiers.RareSignatureScoreIncrease, res.TotalModifierScore, 0.001, "total modifier score should equal the rare signature increase score config value") diff --git a/integration/modifiers_test.go b/integration/modifiers_test.go new file mode 100644 index 0000000..c05817a --- /dev/null +++ b/integration/modifiers_test.go @@ -0,0 +1,26 @@ +package integration_test + +import "github.com/stretchr/testify/require" + +func (it *ValidDatasetTestSuite) TestRareSignaturesModifier() { + t := it.T() + var count uint64 + err := it.db.Conn.QueryRow(it.db.GetContext(), ` + WITH mixtape AS ( + SELECT DISTINCT src, src_nuid, dst, dst_nuid, fqdn, modifier_value + FROM threat_mixtape + WHERE modifier_name = 'rare_signature' + ), rare_sigs AS ( + SELECT src, src_nuid, signature, uniqExactMerge(times_used_dst) as times_used_dst, uniqExactMerge(times_used_fqdn) as times_used_fqdn + FROM rare_signatures + GROUP BY src, src_nuid, signature + HAVING times_used_dst = 1 OR times_used_fqdn = 1 + ) + SELECT count() FROM mixtape m + LEFT JOIN rare_sigs r ON r.src = m.src AND m.src_nuid = r.src_nuid AND m.modifier_value = r.signature + WHERE (fqdn != '' AND times_used_fqdn != 1) OR (fqdn = '' AND times_used_dst != 1) + `).Scan(&count) + require.NoError(t, err) + require.Zero(t, count, "all rare signature entries in the mixtape should actually be used only once according to rare_signatures table") + +} diff --git a/integration/open_sni_test.go b/integration/open_sni_test.go index 1916e2f..d9b7322 100644 --- a/integration/open_sni_test.go +++ b/integration/open_sni_test.go @@ -75,10 +75,11 @@ func (it *OpenSNITestSuite) TestThreats() { prevalenceTotal int64 portProtoService []string }{ - {src: "10.0.0.238", dst: "::", fqdn: "ce7.stearns.org", finalScore: .40468138, totalDuration: 14737.061150000001, count: 0, proxyCount: 0, openCount: 2, totalBytes: 24106, serverIPs: []net.IP{sslServerIP}, portProtoService: []string{"8443:tcp:ssl"}}, - {src: "10.0.0.238", dst: "::", fqdn: "ce7.stearns.org:8000", finalScore: 0.26993924, totalDuration: 7376.718848, count: 0, proxyCount: 0, openCount: 1, totalBytes: 8144, serverIPs: []net.IP{sslServerIP}, portProtoService: []string{"8000:tcp:http"}}, - {src: "10.0.0.238", dst: "34.222.122.143", finalScore: 0.26667413, totalDuration: 7200.403186, count: 0, proxyCount: 0, openCount: 1, totalBytes: 2715618, portProtoService: []string{"64590:tcp:"}}, - {src: "10.0.0.238", dst: "52.33.59.39", finalScore: 0.26667413, totalDuration: 7200.169165, count: 0, proxyCount: 0, openCount: 1, totalBytes: 4593763, portProtoService: []string{"64004:tcp:"}}, + + {src: "10.0.0.238", dst: "::", fqdn: "ce7.stearns.org", finalScore: 0.25468, totalDuration: 14737.061150000001, count: 0, proxyCount: 0, openCount: 2, totalBytes: 24106, serverIPs: []net.IP{sslServerIP}, portProtoService: []string{"8443:tcp:ssl"}}, + {src: "10.0.0.238", dst: "::", fqdn: "ce7.stearns.org:8000", finalScore: 0.11993, totalDuration: 7376.718848, count: 0, proxyCount: 0, openCount: 1, totalBytes: 8144, serverIPs: []net.IP{sslServerIP}, portProtoService: []string{"8000:tcp:http"}}, + {src: "10.0.0.238", dst: "34.222.122.143", finalScore: 0.11667, totalDuration: 7200.403186, count: 0, proxyCount: 0, openCount: 1, totalBytes: 2715618, portProtoService: []string{"64590:tcp:"}}, + {src: "10.0.0.238", dst: "52.33.59.39", finalScore: 0.11667, totalDuration: 7200.169165, count: 0, proxyCount: 0, openCount: 1, totalBytes: 4593763, portProtoService: []string{"64004:tcp:"}}, } min, _, _, _, err := it.db.GetTrueMinMaxTimestamps() @@ -115,7 +116,7 @@ func (it *OpenSNITestSuite) TestThreats() { require.EqualValues(t, 2024, year, "first seen year should match") require.EqualValues(t, 01, month, "first seen month should match") require.EqualValues(t, 31, day, "first seen day should match") - require.InDelta(t, it.cfg.Modifiers.FirstSeenScoreIncrease, res.FirstSeenScore, 0.001, "first seen score should equal config increase value") + require.InDelta(t, 0, res.FirstSeenScore, 0.001, "first seen score should equal 0 for a non-rolling dataset") i++ } rows.Close() diff --git a/integration/proxy_rolling_test.go b/integration/proxy_rolling_test.go index ac15da6..9028f8a 100644 --- a/integration/proxy_rolling_test.go +++ b/integration/proxy_rolling_test.go @@ -81,7 +81,7 @@ func (it *ProxyRollingTestSuite) TestRollingThreats() { } var results []res - expectedCounts := []uint64{4, 62} + expectedCounts := []uint64{3, 41} err := it.db.Conn.Select(it.db.GetContext(), &results, ` SELECT analyzed_at, count() as c FROM threat_mixtape @@ -110,10 +110,8 @@ func (it *ProxyRollingTestSuite) TestRollingThreats() { proxyCount uint64 proxyIPs []net.IP }{ - // finalScore: 0.4492 - // beaconScore: 0.624 - {src: "10.0.0.111", dst: "::", fqdn: "safebrowsing.googleapis.com:443", finalScore: 0.4492, beaconScore: 0.624, totalDuration: 6.569, count: 46, proxyCount: 46, totalBytes: 308421, proxyIPs: []net.IP{proxyIP}}, - {src: "10.0.0.238", dst: "75.75.75.75", finalScore: 0.3383, beaconScore: 0.673, totalDuration: 595.72157, count: 1160, totalBytes: 319107}, + {src: "10.0.0.238", dst: "75.75.75.75", finalScore: 0.18839, beaconScore: 0.673, totalDuration: 595.72157, count: 1160, totalBytes: 319107}, + {src: "10.0.0.111", dst: "::", fqdn: "safebrowsing.googleapis.com:443", finalScore: 0.1492, beaconScore: 0.624, totalDuration: 6.569, count: 46, proxyCount: 46, totalBytes: 308421, proxyIPs: []net.IP{proxyIP}}, } min, _, _, err := it.db.GetBeaconMinMaxTimestamps() diff --git a/logger/logger.go b/logger/logger.go index 5899abd..1ad9799 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -2,16 +2,15 @@ package logger import ( "io" + "log" "log/syslog" "os" + "strconv" "sync" "time" - "github.com/activecm/rita/v5/config" - "github.com/rs/zerolog" "github.com/rs/zerolog/pkgerrors" - "github.com/spf13/afero" ) var once sync.Once @@ -25,14 +24,16 @@ type LevelWriterAdapter struct { Level zerolog.Level } -// zerolog allows for logging at the following levels (from highest to lowest): -// panic (zerolog.PanicLevel, 5) -// fatal (zerolog.FatalLevel, 4) -// error (zerolog.ErrorLevel, 3) -// warn (zerolog.WarnLevel, 2) -// info (zerolog.InfoLevel, 1) -// debug (zerolog.DebugLevel, 0) -// trace (zerolog.TraceLevel, -1) +/* +zerolog allows for logging at the following levels (from highest to lowest): + panic (zerolog.PanicLevel, 5) + fatal (zerolog.FatalLevel, 4) + error (zerolog.ErrorLevel, 3) + warn (zerolog.WarnLevel, 2) + info (zerolog.InfoLevel, 1) + debug (zerolog.DebugLevel, 0) + trace (zerolog.TraceLevel, -1) +*/ // GetLogger returns a logger instance, initializing it if necessary func GetLogger() zerolog.Logger { @@ -48,27 +49,37 @@ func GetLogger() zerolog.Logger { } tmpLogger := zerolog.New(output).With().Timestamp().Logger() - // get logging settings from config - cfg, err := config.GetConfig() + // get logging configuration from environment variables + // check if logging is enabled + loggingEnabledEnv := os.Getenv("LOGGING_ENABLED") + if loggingEnabledEnv == "" { + log.Fatal("Environment variable LOGGING_ENABLED is not set") + } + loggingEnabled, err := strconv.ParseBool(loggingEnabledEnv) if err != nil { - cfg, err = config.LoadConfig(afero.NewOsFs(), config.DefaultConfigPath) - if err != nil { - tmpLogger.Err(err).Msg("unable to read logging settings from config, reverting to basic logging settings... ") - cfg.LoggingEnabled = false - cfg.LogLevel = 1 - } + log.Fatal(err) } - logLevel := zerolog.Level(1) // cfg.LogLevel) - - var writers []io.Writer + // get log level + logLevelEnv := os.Getenv("LOG_LEVEL") + if logLevelEnv == "" { + log.Fatal("Environment variable LOG_LEVEL is not set") + } + logLevelInt, err := strconv.Atoi(logLevelEnv) + if err != nil { + log.Fatal(err) + } + logLevel := zerolog.Level(logLevelInt) // set both file writer and stdout logging level to debug if DebugMode is set if DebugMode { logLevel = zerolog.DebugLevel } - if cfg.LoggingEnabled { + var writers []io.Writer + + // if logging is enabled, set up writer for syslog + if loggingEnabled { // set up syslog syslogAddress := os.Getenv("SYSLOG_ADDRESS") if syslogAddress == "" { @@ -104,6 +115,7 @@ func GetLogger() zerolog.Logger { return zLogger } +// WriteLevel writes the given bytes to the writer if the level is greater than or equal to the LevelWriterAdapter's Level func (lw LevelWriterAdapter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) { if l >= lw.Level { return lw.Write(p) diff --git a/modifier/modifier.go b/modifier/modifier.go index 4b38e38..3896821 100644 --- a/modifier/modifier.go +++ b/modifier/modifier.go @@ -110,21 +110,22 @@ func (modifier *Modifier) detectRareSignature(ctx context.Context) error { rows, err := modifier.Database.Conn.Query(chCtx, `--sql WITH rare_sig_modifiers AS ( - SELECT src, src_nuid, any(signature) as modifier_value, min(times_used_dst) as times_used_dst, min(times_used_fqdn) as times_used_fqdn FROM ( + SELECT src, src_nuid, dst, dst_nuid, fqdn, signature as modifier_value, x.times_used_dst as times_used_dst, x.times_used_fqdn as times_used_fqdn + FROM rare_signatures rs + SEMI JOIN ( SELECT src, src_nuid, signature, uniqExactMerge(times_used_dst) as times_used_dst, uniqExactMerge(times_used_fqdn) as times_used_fqdn FROM rare_signatures WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64})) AND signature != '' GROUP BY src, src_nuid, signature - HAVING times_used_dst == 1 OR times_used_fqdn == 1 - ) - GROUP BY src, src_nuid + HAVING times_used_fqdn = 1 OR times_used_dst = 1 + ) x ON rs.src = x.src AND rs.src_nuid = x.src_nuid AND rs.signature = x.signature + WHERE if(fqdn != '', times_used_fqdn = 1, times_used_dst = 1) ) SELECT hash, src, src_nuid, dst, dst_nuid, fqdn, r.modifier_value as modifier_value, last_seen, toFloat32(if(length(fqdn) > 0, times_used_fqdn, times_used_dst)) as modifier_score FROM threat_mixtape t - -- WHERE modifier_score == 1 - INNER JOIN rare_sig_modifiers r USING src, src_nuid - WHERE t.import_id = unhex({import_id:String}) - + SEMI JOIN rare_sig_modifiers r USING src, src_nuid, dst, dst_nuid, fqdn + WHERE modifier_name = '' -- join only on non-modifier rows to avoid duplicating results + AND t.import_id = unhex({import_id:String}) -- join only on the results for this import `) if err != nil { diff --git a/viewer/csv.go b/viewer/csv.go index 115d60a..649d9ed 100644 --- a/viewer/csv.go +++ b/viewer/csv.go @@ -61,15 +61,19 @@ func FormatToCSV(items []list.Item, relativeTimestamp time.Time) (string, error) "Connection Count", "Total Bytes", "Port:Proto:Service", + "Modifiers", } // loop over the results and format into rows and columns var data []string for _, row := range items { + // get current row item, ok := row.(Item) if !ok { return "", fmt.Errorf("error casting item to Item") } + + // create a slice to hold the fields for this row fields := []string{ item.GetSeverity(false), item.Src.String(), item.Dst.String(), item.FQDN, fmt.Sprint(item.BeaconScore), strconv.FormatBool(item.StrobeScore > 0), @@ -78,6 +82,16 @@ func FormatToCSV(items []list.Item, relativeTimestamp time.Time) (string, error) fmt.Sprint(item.Prevalence), item.GetFirstSeen(relativeTimestamp), strconv.FormatBool(item.MissingHostCount > 0), fmt.Sprint(item.Count), fmt.Sprint(item.TotalBytes), fmt.Sprintf("\"%s\"", strings.Join(item.PortProtoService, ",")), } + + // create a slice to hold the modifiers + modifierList := make([]string, 0, len(item.Modifiers)) + for _, mod := range item.Modifiers { + modifierList = append(modifierList, fmt.Sprintf("%s:%s", mod["modifier_name"], mod["modifier_value"])) + } + + // add the modifiers to the fields + fields = append(fields, fmt.Sprintf("\"%s\"", strings.Join(modifierList, ","))) + // create comma-delimited string from each field in this row formattedRow := strings.Join(fields, ",") data = append(data, formattedRow) diff --git a/viewer/results.go b/viewer/results.go index 2ae4b81..9bded8a 100644 --- a/viewer/results.go +++ b/viewer/results.go @@ -18,34 +18,34 @@ import ( ) type MixtapeResult struct { - Src net.IP `ch:"src" json:"src"` - Dst net.IP `ch:"dst" json:"dst"` - FQDN string `ch:"fqdn"` - FinalScore float32 `ch:"final_score"` - Count uint64 `ch:"count"` - ProxyCount uint64 `ch:"proxy_count"` - BeaconScore float32 `ch:"beacon_score"` - StrobeScore float32 `ch:"strobe_score"` - BeaconThreatScore float32 `ch:"beacon_threat_score"` - TotalDuration float32 `ch:"total_duration"` - LongConnScore float32 `ch:"long_conn_score"` - FirstSeen time.Time `ch:"first_seen_historical"` - FirstSeenScore float32 `ch:"first_seen_score"` - Prevalence float32 `ch:"prevalence"` - PrevalenceScore float32 `ch:"prevalence_score"` - Subdomains uint64 `ch:"subdomains"` - PortProtoService []string `ch:"port_proto_service"` - C2OverDNSScore float32 `ch:"c2_over_dns_score"` - C2OverDNSDirectConnScore float32 `ch:"c2_over_dns_direct_conn_score"` - ThreatIntelScore float32 `ch:"threat_intel_score"` - ThreatIntelDataSizeScore float32 `ch:"threat_intel_data_size_score"` - TotalBytes uint64 `ch:"total_bytes"` - TotalBytesFormatted string `ch:"total_bytes_formatted"` - MissingHostHeaderScore float32 `ch:"missing_host_header_score"` - MissingHostCount uint64 `ch:"missing_host_count"` - ProxyIPs []net.IP `ch:"proxy_ips"` - - TotalModifierScore float32 `ch:"total_modifier_score"` + Src net.IP `ch:"src" json:"src"` + Dst net.IP `ch:"dst" json:"dst"` + FQDN string `ch:"fqdn"` + FinalScore float32 `ch:"final_score"` + Count uint64 `ch:"count"` + ProxyCount uint64 `ch:"proxy_count"` + BeaconScore float32 `ch:"beacon_score"` + StrobeScore float32 `ch:"strobe_score"` + BeaconThreatScore float32 `ch:"beacon_threat_score"` + TotalDuration float32 `ch:"total_duration"` + LongConnScore float32 `ch:"long_conn_score"` + FirstSeen time.Time `ch:"first_seen_historical"` + FirstSeenScore float32 `ch:"first_seen_score"` + Prevalence float32 `ch:"prevalence"` + PrevalenceScore float32 `ch:"prevalence_score"` + Subdomains uint64 `ch:"subdomains"` + PortProtoService []string `ch:"port_proto_service"` + C2OverDNSScore float32 `ch:"c2_over_dns_score"` + C2OverDNSDirectConnScore float32 `ch:"c2_over_dns_direct_conn_score"` + ThreatIntelScore float32 `ch:"threat_intel_score"` + ThreatIntelDataSizeScore float32 `ch:"threat_intel_data_size_score"` + TotalBytes uint64 `ch:"total_bytes"` + TotalBytesFormatted string `ch:"total_bytes_formatted"` + MissingHostHeaderScore float32 `ch:"missing_host_header_score"` + MissingHostCount uint64 `ch:"missing_host_count"` + ProxyIPs []net.IP `ch:"proxy_ips"` + Modifiers []map[string]string `ch:"modifiers"` + TotalModifierScore float32 `ch:"total_modifier_score"` } type Item MixtapeResult @@ -210,6 +210,7 @@ func BuildResultsQuery(filter Filter, currentPage, pageSize int, minTimestamp ti missing_host_count, missing_host_header_score, c2_over_dns_direct_conn_score, + modifiers, total_modifier_score, toFloat32(base_score + total_modifier_score + prevalence_score + first_seen_score + missing_host_header_score + threat_intel_data_size_score + c2_over_dns_direct_conn_score) as final_score -- base_score @@ -239,6 +240,7 @@ func BuildResultsQuery(filter Filter, currentPage, pageSize int, minTimestamp ti sum(missing_host_count) as missing_host_count, toFloat32(sum(missing_host_header_score)) as missing_host_header_score, toFloat32(sum(c2_over_dns_direct_conn_score)) as c2_over_dns_direct_conn_score, + arraySort(groupUniqArrayIf(map('modifier_name', modifier_name, 'modifier_value', modifier_value), modifier_name != '')) as modifiers, toFloat32(sum(modifier_score)) as total_modifier_score, greatest(beacon_threat_score, long_conn_score, strobe_score, c2_over_dns_score, threat_intel_score) as base_score FROM threat_mixtape t diff --git a/viewer/sidebar.go b/viewer/sidebar.go index 7c2f071..d38ad16 100644 --- a/viewer/sidebar.go +++ b/viewer/sidebar.go @@ -240,6 +240,19 @@ func (m *sidebarModel) getModifiers() []modifier { modifiers = append(modifiers, modifier{label: "Threat Intel " + label, value: m.Data.TotalBytesFormatted, delta: m.Data.ThreatIntelDataSizeScore}) } + if m.Data.C2OverDNSDirectConnScore != 0 { + modifiers = append(modifiers, modifier{label: "No Direct Connections", value: "", delta: 10}) + } + + for _, mod := range m.Data.Modifiers { + switch mod["modifier_name"] { + case "rare_signature": + modifiers = append(modifiers, modifier{label: "Rare Signature", value: mod["modifier_value"], delta: 10}) + case "mime_type_mismatch": + modifiers = append(modifiers, modifier{label: "MIME Type Mismatch", value: "", delta: 10}) + } + } + return modifiers }