package integration_test import ( "context" "net" "testing" "time" "github.com/activecm/rita/v5/cmd" "github.com/activecm/rita/v5/config" "github.com/activecm/rita/v5/database" "github.com/activecm/rita/v5/viewer" "github.com/spf13/afero" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) type ProxyRollingTestSuite ValidDatasetTestSuite func TestProxyRolling(t *testing.T) { proxyRollingSuite := new(ProxyRollingTestSuite) // set up file system interface afs := afero.NewOsFs() cfg, err := config.ReadFileConfig(afs, ConfigPath) require.NoError(t, err) cfg.Env.DBConnection = dockerInfo.clickhouseConnection require.True(t, cfg.Filtering.FilterExternalToInternal) // // import data results, err := cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/proxy_rolling", "proxy_rolling", false, true) require.NoError(t, err) proxyRollingSuite.importResults = results // connect to database db, err := database.ConnectToDB(context.Background(), "proxy_rolling", cfg, nil) require.NoError(t, err) // determine which max timestamp to use for relative time calculations _, maxTimestamp, _, err := db.GetBeaconMinMaxTimestamps() require.NoError(t, err) proxyRollingSuite.maxTimestamp = maxTimestamp // proxyRollingSuite.useCurrentTime = useCurrentTime proxyRollingSuite.db = db proxyRollingSuite.cfg = cfg suite.Run(t, proxyRollingSuite) } func (it *ProxyRollingTestSuite) TestRollingThreats() { t := it.T() // hour 15 // 5 threats // (beacon) // 20% 50% 10.0.0.111 -> img-getpocket.cdn.mozilla.net:443 // 10.0.0.238 -> 75.75.75.75 // 10.0.0.238 -> 185.125.190.56 // 10.0.0.111 -> www.whitehouse.gov:443 // 10.0.0.111 -> aus5.mozilla.org:443 // hour 16 // 2379 threats // 33.84% 67.3% 10.0.0.238 -> 75.75.75.75 // 29.92% 62.4% 10.0.0.111 -> safebrowsing.googleapis.com:443 // 21.66% 25.8% 10.0.0.111 -> www.google.com 1h14m56s // verify that the dataset had multiple imports by checking that there was more than one unique analyzed_at timestamp type res struct { AnalyzedAt time.Time `ch:"analyzed_at"` Count uint64 `ch:"c"` } var results []res expectedCounts := []uint64{3, 41} err := it.db.Conn.Select(it.db.GetContext(), &results, ` SELECT analyzed_at, count() as c FROM threat_mixtape GROUP BY analyzed_at ORDER BY analyzed_at `) require.NoError(t, err) require.Len(t, results, len(expectedCounts)) require.EqualValues(t, expectedCounts[0], results[0].Count, "first import threat count should match") require.EqualValues(t, expectedCounts[1], results[1].Count, "second import threat count should match") proxyIP := net.ParseIP("10.0.0.238") require.NotNil(t, proxyIP) // verify the results of the analysis expectedResults := []struct { src string dst string fqdn string finalScore float64 beaconScore float64 totalDuration float64 totalBytes float64 count uint64 proxyCount uint64 proxyIPs []net.IP }{ {src: "10.0.0.238", dst: "75.75.75.75", finalScore: 0.18839 + 0.15, beaconScore: 0.673, totalDuration: 595.72157, count: 1160, totalBytes: 319107}, {src: "10.0.0.111", dst: "::", fqdn: "safebrowsing.googleapis.com:443", finalScore: 0.1492 + 0.15, beaconScore: 0.624, totalDuration: 6.569, count: 46, proxyCount: 46, totalBytes: 308421, proxyIPs: []net.IP{proxyIP}}, } min, _, _, err := it.db.GetBeaconMinMaxTimestamps() require.NoError(t, err) query, params, _ := viewer.BuildResultsQuery(&viewer.Filter{}, 0, 10, min) ctx := it.db.QueryParameters(params) rows, err := it.db.Conn.Query(ctx, query) require.NoError(t, err) i := 0 for rows.Next() { var res viewer.MixtapeResult err := rows.ScanStruct(&res) require.NoError(t, err) if i < len(expectedResults) { require.Equal(t, expectedResults[i].src, res.Src.String(), "source IP should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.Equal(t, expectedResults[i].dst, res.Dst.String(), "destination IP should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.Equal(t, expectedResults[i].fqdn, res.FQDN, "destination FQDN should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.InDelta(t, expectedResults[i].finalScore, res.FinalScore, 0.001, "final score should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.InDelta(t, expectedResults[i].beaconScore, res.BeaconScore, 0.001, "beacon score should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.InDelta(t, expectedResults[i].totalDuration, res.TotalDuration, 0.001, "total duration should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.InDelta(t, expectedResults[i].totalBytes, res.TotalBytes, 0.001, "total bytes should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.EqualValues(t, expectedResults[i].count, res.Count, "count should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.EqualValues(t, expectedResults[i].proxyCount, res.ProxyCount, "proxy count should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) require.ElementsMatch(t, expectedResults[i].proxyIPs, res.ProxyIPs, "proxy ips should match, src: %s, dst: %s, fqdn: %s", expectedResults[i].src, expectedResults[i].dst, expectedResults[i].fqdn) } i++ } rows.Close() } func (it *ProxyRollingTestSuite) TestProxy() { t := it.T() var httpProxy, usniProxy, missingMixtapeProxy uint64 // get number of unique proxy connections from http logs err := it.db.Conn.QueryRow(it.db.GetContext(), ` SELECT count(DISTINCT hash) FROM http WHERE method = 'CONNECT' `).Scan(&httpProxy) require.NoError(t, err) // get number of unique connections marked as a proxy conn in usni table err = it.db.Conn.QueryRow(it.db.GetContext(), ` SELECT count(DISTINCT hash) FROM usni WHERE proxy = true `).Scan(&usniProxy) require.NoError(t, err) require.EqualValues(t, httpProxy, usniProxy, "number of proxy connections from http and usni should match") // make sure that there are no connections in the mixtape with no proxy count that actually do have proxy connections err = it.db.Conn.QueryRow(it.db.GetContext(), ` SELECT count() FROM ( SELECT src, host FROM http h INNER JOIN threat_mixtape t ON h.src = t.src AND h.host = t.fqdn AND t.proxy_count < 0 WHERE host != '' GROUP BY src, host HAVING countIf(method = 'CONNECT') > 0 ) `).Scan(&missingMixtapeProxy) require.NoError(t, err) require.EqualValues(t, 0, missingMixtapeProxy, "there should be no connections in threat_mixtape with missing proxy_counts that are proxy connections") var numProxyDst uint64 // make sure the proxy IP doesn't appear as a destination in the mixtape // it should get filtered out by either the conn log filtering or by the SNI conn filtering in the spagooper err = it.db.Conn.QueryRow(it.db.GetContext(), ` SELECT count(DISTINCT hash) FROM threat_mixtape WHERE dst = '10.0.0.238' `).Scan(&numProxyDst) require.NoError(t, err) require.EqualValues(t, 0, numProxyDst, "the proxy IP shouldn't appear as a destination") }