package analysis import ( "testing" "github.com/stretchr/testify/require" ) func TestGetBeaconScore(t *testing.T) { // Define test cases tests := []struct { name string tsScore float64 tsWeight float64 dsScore float64 dsWeight float64 durScore float64 durWeight float64 histScore float64 histWeight float64 expectedScore float64 expectedError bool }{ { name: "Valid Scores and Weights", tsScore: 0.8, tsWeight: 0.2, dsScore: 0.7, dsWeight: 0.2, durScore: 0.6, durWeight: 0.3, histScore: 0.5, histWeight: 0.3, expectedScore: 0.63, expectedError: false, }, { name: "All Scores 1, All Weights Equal", tsScore: 1, tsWeight: 0.25, dsScore: 1, dsWeight: 0.25, durScore: 1, durWeight: 0.25, histScore: 1, histWeight: 0.25, expectedScore: 1, expectedError: false, }, { name: "All Ccores 1, All Weights Different", tsScore: 1, tsWeight: 0.1, dsScore: 1, dsWeight: 0.2, durScore: 1, durWeight: 0.3, histScore: 1, histWeight: 0.4, expectedScore: 1, expectedError: false, }, { name: "All Scores Different, All Weights Equal", tsScore: 0.1, tsWeight: 0.25, dsScore: 0.2, dsWeight: 0.25, durScore: 0.3, durWeight: 0.25, histScore: 0.4, histWeight: 0.25, expectedScore: 0.25, expectedError: false, }, { name: "All Scores Different, All Weights Different", tsScore: 0.1, tsWeight: 0.1, dsScore: 0.2, dsWeight: 0.2, durScore: 0.3, durWeight: 0.3, histScore: 0.4, histWeight: 0.4, // 0.1*0.1 + 0.2*0.2 + 0.3*0.3 + 0.4*0.4 = 0.01 + 0.04 + 0.09 + 0.16 = 0.30 // (0.30*1000)/1000 = 0.30 expectedScore: 0.30, expectedError: false, }, { name: "High precision scores and weights", tsScore: 0.111, tsWeight: 0.173, dsScore: 0.222, dsWeight: 0.325, durScore: 0.333, durWeight: 0.299, histScore: 0.444, histWeight: 0.203, // histWeight = 1 - (0.173 + 0.325 + 0.299) = 0.203 // 0.111*0.173 + 0.222*0.325 + 0.333*0.299 + 0.444*0.203 = 0.019263 + 0.07215 + 0.099567 + 0.090132 = 0.281112 // (0.281112*1000)/1000 = 0.281 expectedScore: 0.281, expectedError: false, }, { // to check for rounding errors name: "Sensitivity test for closely-valued scores and weights", tsScore: 0.1001, tsWeight: 0.249, dsScore: 0.1002, dsWeight: 0.251, durScore: 0.1003, durWeight: 0.250, histScore: 0.1004, histWeight: 0.250, expectedScore: 0.10025, expectedError: false, }, { name: "Only one active weight", tsScore: 0.25, tsWeight: 1, dsScore: 1, dsWeight: 0, durScore: 1, durWeight: 0, histScore: 1, histWeight: 0, expectedScore: 0.25, expectedError: false, }, { name: "Negative score input", tsScore: -0.1, tsWeight: 0.25, dsScore: 0.5, dsWeight: 0.25, durScore: 0.5, durWeight: 0.25, histScore: 0.5, histWeight: 0.25, expectedError: true, }, { name: "Score greater than 1", tsScore: 1.1, tsWeight: 0.25, dsScore: 0.5, dsWeight: 0.25, durScore: 0.5, durWeight: 0.25, histScore: 0.5, histWeight: 0.25, expectedError: true, }, { name: "Negative weight input", tsScore: 0.5, tsWeight: -0.1, dsScore: 0.5, dsWeight: 0.25, durScore: 0.5, durWeight: 0.25, histScore: 0.5, histWeight: 0.25, expectedError: true, }, { name: "Weight greater than 1", tsScore: 0.5, tsWeight: 1.1, dsScore: 0.5, dsWeight: 0.25, durScore: 0.5, durWeight: 0.25, histScore: 0.5, histWeight: 0.25, expectedError: true, }, { name: "Weights sum more than 1", tsScore: 0.5, tsWeight: 0.3, dsScore: 0.5, dsWeight: 0.3, durScore: 0.5, durWeight: 0.3, histScore: 0.5, histWeight: 0.3, expectedError: true, }, { name: "Valid weight but sum less than 1", tsScore: 0.5, tsWeight: 0.3, dsScore: 0.5, dsWeight: 0.3, durScore: 0.5, durWeight: 0.2, histScore: 0.5, histWeight: 0.1, expectedError: true, }, { name: "Weights sum to more than 1", tsScore: 0.5, tsWeight: 0.3, dsScore: 0.5, dsWeight: 0.3, durScore: 0.5, durWeight: 0.3, histScore: 0.5, histWeight: 0.2, expectedError: true, }, } // Run test cases for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function score, err := getBeaconScore(test.tsScore, test.tsWeight, test.dsScore, test.dsWeight, test.durScore, test.durWeight, test.histScore, test.histWeight) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated score require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestGetTimestampScore(t *testing.T) { tests := []struct { name string tsList []uint32 expectedScore float64 expectedSkew float64 expectedMAD float64 expectedUniqueIntervals []int64 expectedUniqueIntervalCounts []int64 expectedTSMode int64 expectedTSModeCount int64 expectedError bool }{ { name: "Simple Number List", tsList: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // intervals between timestamps: 1, 1, 1, 1, 1, 1, 1, 1, 1 expectedUniqueIntervals: []int64{1}, expectedUniqueIntervalCounts: []int64{9}, expectedTSMode: 1, expectedTSModeCount: 9, // q1 = 1, q2 = 1, q3 = 1 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 1)) = median(0, 1, 0, 1, 0, 1, 0, 1, 0) = 0 expectedMAD: 0, // MAD score = 1 - 0 = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Connection with Perfect Intervals", // timestamps : 1517338924, 1517338924 + 60, 1517338924 + 120, 1517338924 + 180, 1517338924 + 240, 1517338924 + 300, 1517338924 + 360, 1517338924 + 420, 1517338924 + 480, 1517338924 + 540, tsList: []uint32{1517338924, 1517338984, 1517339044, 1517339104, 1517339164, 1517339224, 1517339284, 1517339344, 1517339404, 1517339464}, // intervals between timestamps: 60, 60, 60, 60, 60, 60, 60, 60, 60 expectedUniqueIntervals: []int64{60}, expectedUniqueIntervalCounts: []int64{9}, expectedTSMode: 60, expectedTSModeCount: 9, // q1 = 60, q2 = 60, q3 = 60 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(60 - 60)) = 0 expectedMAD: 0, // MAD score = 1 - 0 = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Connection with Closely-Valued Intervals", // timestamps : 1517338924, 1517338924 + 98, 1517338924 + 98 + 99, 1517338924 + 98 + 99 + 99, 1517338924 + 98 + 99 + 99 + 100, 1517338924 + 98 + 99 + 99 + 100 + 100, 1517338924 + 98 + 99 + 99 + 100 + 100 + 100, 1517338924 + 98 + 99 + 99 + 100 + 100 + 100 + 101, 1517338924 + 98 + 99 + 99 + 100 + 100 + 100 + 101 + 101, 1517338924 + 98 + 99 + 99 + 100 + 100 + 100 + 101 + 101 + 102, tsList: []uint32{1517338924, 1517339022, 1517339121, 1517339220, 1517339320, 1517339420, 1517339520, 1517339621, 1517339722, 1517339824}, // intervals between timestamps: 98, 99, 99, 100, 100, 100, 101, 101, 102 expectedUniqueIntervals: []int64{98, 99, 100, 101, 102}, expectedUniqueIntervalCounts: []int64{1, 2, 3, 2, 1}, expectedTSMode: 100, expectedTSModeCount: 3, // q1 = 99, q2 = 100, q3 = 101 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 100)) = median(1, 0, 0, 1, 1, 1, 2, 2, 3) = 1 expectedMAD: 1, // MAD score = 1 - (mad/median) = 1 - 1/100 = 0.99 expectedScore: 0.995, // score = round(((1 + 0.99)/2)*1000)/1000 = 0.995 expectedError: false, }, { name: "Connection with Bi-Modal Intervals", // timestamps : 1517338924, 1517338924 + 98, 1517338924 + 98 + 300, 1517338924 + 98 + 300 + 98, 1517338924 + 98 + 300 + 98 + 300, 1517338924 + 98 + 300 + 98 + 300 + 98, 1517338924 + 98 + 300 + 98 + 300 + 98 + 300, 1517338924 + 98 + 300 + 98 + 300 + 98 + 300 + 98, 1517338924 + 98 + 300 + 98 + 300 + 98 + 300 + 98 + 300, 1517338924 + 98 + 300 + 98 + 300 + 98 + 300 + 98 + 300 + 98, tsList: []uint32{1517338924, 1517339022, 1517339322, 1517339420, 1517339720, 1517339818, 1517340118, 1517340216, 1517340516, 1517340614, 1517340914}, // intervals between timestamps: 98, 300, 98, 300, 98, 300, 98, 300, 98 expectedUniqueIntervals: []int64{98, 300}, expectedUniqueIntervalCounts: []int64{5, 5}, expectedTSMode: 98, expectedTSModeCount: 5, // Quartiles: {98 199 300} // q1 = 98, q2 = 199, q3 = 300 // numerator = q3 + q1 - 2*q2 = 300 + 98 - 2*199 = 300 + 98 - 398 = 0 // IQR = q3 - q1 = 300 - 98 = 202 // Bowley Skewness = (Q3+Q1 – 2Q2) / (Q3 – Q1) = numerator / IQR // but if the denominator less than 10 or the median is equal to the lower or upper quartile, the skewness is zero // skewness = 0 // skewness score = 1 - skewness = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 199)) = median(101, 100, 101, 100, 101, 100, 101, 100, 101) = 101 // MAD score = 1 - (mad/median) = 1 - 101/199 = 0.492 // score = round(((1 + 0.492)/2)*1000)/1000 = 0.746 expectedSkew: 0, expectedMAD: 101, expectedScore: 0.746, expectedError: false, }, { name: "Connection with Random Intervals", tsList: []uint32{1517338924, 1517338925, 1517339224, 1517339249, 1517344224, 1517344314, 1517344316, 1517344358, 1517344858, 1517346358}, // intervals between timestamps: 1, 299, 25, 4975, 90, 2, 42, 500, 1500 expectedUniqueIntervals: []int64{1, 2, 25, 42, 90, 299, 500, 1500, 4975}, expectedUniqueIntervalCounts: []int64{1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedTSMode: 1, expectedTSModeCount: 1, // q1 = 13.5, q2 = 90, q3 = 1000 // numerator = q3 + q1 - 2*q2 = 1000 + 13.5 - 2*90 = 1000 + 13.5 - 180 = 833.5 // IQR = q3 - q1 = 1000 - 13.5 = 986.5 // skewness = numerator / IQR = 833.5 / 986.5 = 0.845 // skewness score = 1 - skewness = 1 - 0.845 = 0.155 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 90)) = median(89, 209, 65, 4885, 0, 8, 48, 410, 1410) = 89 // score = round(((0.155 + 0.01111)/2)*1000)/1000 = 0.083 expectedSkew: 0.845, // skewness score = 1 - 0.845 = 0.155 expectedMAD: 89, // MAD score = 1 - (mad/median) = 1 - 89/90 = 0.01111 expectedScore: 0.083, expectedError: false, }, { // should not happen in practice, since we query for connections with > 3 unique timestamps name: "Connection with < 3 Non-Zero Intervals", tsList: []uint32{60, 60, 60, 60, 60, 60, 60, 60, 60}, // intervals between timestamps: 0, 0, 0, 0, 0, 0, 0, 0 expectedError: true, }, { name: "Length of Timestamp List < 4", tsList: []uint32{1517338924, 1517338925}, expectedError: true, }, { name: "Empty Input Slice", tsList: []uint32{}, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function score, skew, mad, intervals, intervalCounts, mode, modeCount, err := getTimestampScore(test.tsList) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated debug values require.EqualValues(test.expectedUniqueIntervals, intervals, "Expected unique intervals to be %v, got %v", test.expectedUniqueIntervals, intervals) require.EqualValues(test.expectedUniqueIntervalCounts, intervalCounts, "Expected unique interval counts to be %v, got %v", test.expectedUniqueIntervalCounts, intervalCounts) require.EqualValues(test.expectedTSMode, mode, "Expected mode to be %v, got %v", test.expectedTSMode, mode) require.EqualValues(test.expectedTSModeCount, modeCount, "Expected mode count to be %v, got %v", test.expectedTSModeCount, modeCount) // check the calculated score values require.InDelta(test.expectedSkew, skew, 0.001, "Expected skew to be %v, got %v", test.expectedSkew, skew) require.InDelta(test.expectedMAD, mad, 0.001, "Expected MAD to be %v, got %v", test.expectedMAD, mad) require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestGetDataSizeScore(t *testing.T) { tests := []struct { name string bytesList []float64 expectedScore float64 expectedSkew float64 expectedMAD float64 expectedUniqueSizes []int64 expectedUniqueSizeCounts []int64 expectedDSMode int64 expectedDSModeCount int64 expectedRange int64 expectedError bool }{ { name: "Simple Number List", bytesList: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedUniqueSizes: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedUniqueSizeCounts: []int64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedDSMode: 1, expectedDSModeCount: 1, expectedRange: 9, // 10 - 1 = 9 // q1 = 3, q2 = 5.5, q3 = 8 // numerator = q3 + q1 - 2*q2 = 8 + 3 - 2*5.5 = 8 + 3 - 11 = 0 // IQR = q3 - q1 = 8 - 3 = 5 // skewness = numerator / IQR = 0 / 5 = 0 expectedSkew: 0, // skewness score = 1 - skewness = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 5.5)) = median(4.5, 3.5, 2.5, 1.5, 0.5, 0.5, 1.5, 2.5, 3.5, 4.5) = 2.5 expectedMAD: 2.5, // MAD score = 1 - (mad/median) = 1 - 2.5/5.5 = 0.545454 expectedScore: 0.773, expectedError: false, }, { name: "Connection with Identical Sizes", bytesList: []float64{60, 60, 60, 60, 60, 60, 60, 60, 60}, expectedUniqueSizes: []int64{60}, expectedUniqueSizeCounts: []int64{9}, expectedDSMode: 60, expectedDSModeCount: 9, expectedRange: 0, // 60 - 60 = 0 expectedSkew: 0, // skewness score = 1 - 0 = 1 expectedMAD: 0, // MAD score = 1 - 0 = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Connection with Closely-Valued Sizes", bytesList: []float64{98, 99, 99, 100, 100, 100, 101, 101, 102}, expectedUniqueSizes: []int64{98, 99, 100, 101, 102}, expectedUniqueSizeCounts: []int64{1, 2, 3, 2, 1}, expectedDSMode: 100, expectedDSModeCount: 3, expectedSkew: 0, // skewness score = 1 - 0 = 1 expectedMAD: 1, // MAD score = 1 - (mad/median) = 1 - 1/100 = 0.99 expectedScore: 0.995, expectedError: false, }, { name: "Connection with Random Sizes", bytesList: []float64{524885, 1, 5000, 98654, 50, 41, 965842, 3}, expectedUniqueSizes: []int64{1, 3, 41, 50, 5000, 98654, 524885, 965842}, expectedUniqueSizeCounts: []int64{1, 1, 1, 1, 1, 1, 1, 1}, expectedDSMode: 1, // if there are multiple modes, the first one in list is chosen expectedDSModeCount: 1, // q1 = 22, q2 = 2525, q3 = 311769.5 // numerator = q3 + q1 - 2*q2 = 311769.5 + 22 - 2*2525 = 311769.5 + 22 - 5050 = 306741.5 // IQR = q3 - q1 = 311769.5 - 22 = 311747.5 // skewness = numerator / IQR = 306741.5 / 311747.5 = 0.984 // skewness score = 1 - skewness = 1 - 0.984 = 0.016 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 2525)) = median(2475, 2475, 2484, 2522, 2524, 96129, 522360, 963317) = 2522+2524/2 = 2523 expectedSkew: 0.984, // skewness score = 1 - 0.984 = 0.016 expectedMAD: 2523, // MAD score = 1 - (mad/median) = 1 - (2523/2525) = 0.000792079 expectedScore: 0.008, // score = round(((0.016+0.000792079)/2)*1000)/1000 = 0.008 expectedError: false, }, { name: "Bytes List is Comprised of Only 0s", bytesList: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, expectedUniqueSizes: []int64{0}, expectedUniqueSizeCounts: []int64{10}, expectedDSMode: 0, expectedDSModeCount: 10, expectedSkew: 0, // skewness score = 1 - 0 = 1 expectedMAD: 0, // MAD score = 0, since median < 1 and defaultMADScore for datasize scoring is 0 expectedScore: 0.5, // score = round(((1 + 0)/2)*1000)/1000 = 0.5 expectedError: false, }, // in practice we should have at least 4, since we require at least 4 connections to calculate the score { name: "Length of Bytes List < 3", bytesList: []float64{100, 200}, expectedError: true, }, { name: "Empty Input Slice", bytesList: []float64{}, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function score, skew, mad, sizes, sizeCounts, mode, modeCount, err := getDataSizeScore(test.bytesList) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated debug values require.EqualValues(test.expectedUniqueSizes, sizes, "Expected unique sizes to be %v, got %v", test.expectedUniqueSizes, sizes) require.EqualValues(test.expectedUniqueSizeCounts, sizeCounts, "Expected unique size counts to be %v, got %v", test.expectedUniqueSizeCounts, sizeCounts) require.InDelta(test.expectedDSMode, mode, 0.001, "Expected mode to be %v, got %v", test.expectedDSMode, mode) require.EqualValues(test.expectedDSModeCount, modeCount, "Expected mode count to be %v, got %v", test.expectedDSModeCount, modeCount) // check the calculated score values require.InDelta(test.expectedSkew, skew, 0.001, "Expected skew to be %v, got %v", test.expectedSkew, skew) require.InDelta(test.expectedMAD, mad, 0.001, "Expected MAD to be %v, got %v", test.expectedMAD, mad) require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestCalculateStatisticalScore(t *testing.T) { tests := []struct { name string values []float64 defaultMadScore float64 expectedScore float64 expectedSkew float64 expectedMAD float64 expectedError bool }{ { name: "Simple Number List", values: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, defaultMadScore: 0, // q1 = 3, q2 = 5.5, q3 = 8 // numerator = q3 + q1 - 2*q2 = 8 + 3 - 2*5.5 = 8 + 3 - 11 = 0 // IQR = q3 - q1 = 8 - 3 = 5 // skewness = numerator / IQR = 0 / 5 = 0 // skewness score = 1 - skewness = 1 - 0 = 1 expectedSkew: 0, // skewness score = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 5.5)) = median(4.5, 3.5, 2.5, 1.5, 0.5, 0.5, 1.5, 2.5, 3.5, 4.5) = 2.5 expectedMAD: 2.5, // MAD score = 1 - (mad/median) = 1 - 2.5/5.5 = 0.545454 expectedScore: 0.773, expectedError: false, }, { name: "Connection with Identical Intervals/Sizes", values: []float64{60, 60, 60, 60, 60, 60, 60, 60, 60}, defaultMadScore: 0, // q1 = 60, q2 = 60, q3 = 60 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(60 - 60)) = 0 expectedMAD: 0, // MAD score = 1 - 0 = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 }, { name: "Connection with Closely-Valued Intervals/Sizes", values: []float64{98, 99, 99, 100, 100, 100, 101, 101, 102}, // q1 = 99, q2 = 100, q3 = 101 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 100)) = median(1, 0, 0, 1, 1, 1, 2, 2, 3) = 1 expectedMAD: 1, // MAD score = 1 - (mad/median) = 1 - 1/100 = 0.99 // score = round(((1 + 0.99)/2)*1000)/1000 = 0.995 expectedScore: 0.995, }, { name: "Connection with Random Intervals/Sizes", // timestamps : 1517338924, 1517338924 + 1, 1517338924 + 300, 1517338924 + 25, 1517338924 + 5000, 1517338924 + 100, 1517338924 + 1000, 1517338924 + 200, 1517338924 + 1500, 1517338924 + 3000, // intervals between timestamps: 1, 299, 25, 4975, 90, 900, 800, 500, 1500 values: []float64{1, 299, 25, 4975, 90, 2, 42, 500, 1500}, defaultMadScore: 1, // q1 = 13.5, q2 = 90, q3 = 1000 // numerator = q3 + q1 - 2*q2 = 1000 + 13.5 - 2*90 = 1000 + 13.5 - 180 = 833.5 // IQR = q3 - q1 = 1000 - 13.5 = 986.5 // skewness = numerator / IQR = 833.5 / 986.5 = 0.845 // skewness score = 1 - skewness = 1 - 0.845 = 0.155 expectedSkew: 0.845, // skewness score = 1 - 0.845 = 0.155 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 90)) = median(89, 209, 65, 4885, 0, 8, 48, 410, 1410) = 89 expectedMAD: 89, // MAD score = 1 - (mad/median) = 1 - 89/90 = 0.01111 expectedScore: 0.083, // score = round(((0.155 + 0.01111)/2)*1000)/1000 = 0.083 expectedError: false, }, { name: "Connection with Random Intervals/Sizes, defaultMadScore = 0", values: []float64{524885, 1, 5000, 98654, 50, 41, 965842, 3}, defaultMadScore: 0, // q1 = 22, q2 = 2525, q3 = 311769.5 // numerator = q3 + q1 - 2*q2 = 311769.5 + 22 - 2*2525 = 311769.5 + 22 - 5050 = 306741.5 // IQR = q3 - q1 = 311769.5 - 22 = 311747.5 // skewness = numerator / IQR = 306741.5 / 311747.5 = 0.983 expectedSkew: 0.983, // skewness score = 1 - skewness = 1 - 0.983 = 0.017 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 2525)) = median(252363, 2524, 2475, 2509, 2475, 2484, 96317, 2522) = 2523 expectedMAD: 2523, // MAD score = 1 - (mad/median) = 1 - (2523/2525) = 0.00079 expectedScore: 0.009, // score = round(((0.017 + 0.00079)/2)*1000)/1000 = 0.009 expectedError: false, }, { // this will not happen in practice for the timestamps, since we use a list of non-zero intervals between timestamps name: "Median == 0, defaultMadScore = 0", values: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, defaultMadScore: 0, expectedSkew: 0, // skewness score = 1 - 0 = 1 expectedMAD: 0, // MAD score = 0, since median < 1 and defaultMADScore = 0 expectedScore: 0.5, // score = round(((1 + 0)/2)*1000)/1000 = 0.5 expectedError: false, }, { // this will not happen in practice for the timestamps, since we use a list of non-zero intervals between timestamps name: "Median == 0, defaultMadScore = 1", values: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, defaultMadScore: 1, expectedSkew: 0, // skewness score = 1 - 0 = 1 expectedMAD: 0, // MAD score = 1, since median < 1 and defaultMADScore = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Median < 1, defaultMadScore = 0", values: []float64{-1, -2, -3, -4, -5, -6, -7, -8, -9, -10}, defaultMadScore: 0, // default score for bytes is zero // q1 = -3, q2 = -5.5, q3 = -8 // numerator = q3 + q1 - 2*q2 = -8 + -3 - 2*-5.5 = -8 + -3 + 11 = 0 // IQR = q3 - q1 = -8 - -3 = -5 // skewness = numerator / IQR = 0 / -5 = 0 expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - -5.5)) = median(4.5, 3.5, 2.5, 1.5, 0.5, 0.5, 1.5, 2.5, 3.5, 4.5) = 2.5 expectedMAD: 2.5, // MAD score = 0 since median < 1 and defaultMADScore = 0 expectedScore: 0.5, // score = round(((1 + 0)/2)*1000)/1000 = 0.5 expectedError: false, }, { name: "Bowley's Skew Unreliable: Q1 == Q2", values: []float64{7, 8, 7, 8, 7, 8, 7, 8, 7}, defaultMadScore: 1, // q1 = 7, q2 = 7, q3 = 8 // skewness will not be calculated and stay at zero expectedSkew: 0, // skewness score = 1 - 0 = 1 // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 7)) = median(0, 1, 0, 1, 0, 1, 0, 1, 0) = 0 expectedMAD: 0, // MAD score = 1 since defaultMADScore = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Bowley's Skew Unreliable: Q2 == Q3", values: []float64{1, 1, 2, 2, 2, 2}, defaultMadScore: 1, // q1 = 1, q2 = 2, q3 = 2 // skewness will not be calculated and stay at zero expectedSkew: 0, // median absolute deviation = median(abs(x - median(x))) = median(abs(x - 2)) = median(1, 1, 0, 0, 0, 0) = 0 expectedMAD: 0, // MAD score = 1 since defaultMADScore = 1 expectedScore: 1, // score = round(((1 + 1)/2)*1000)/1000 = 1 expectedError: false, }, { name: "Empty Input Slice", values: []float64{}, defaultMadScore: 0, expectedSkew: 0, expectedMAD: 0, expectedScore: 0, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function score, skew, mad, err := calculateStatisticalScore(test.values, test.defaultMadScore) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated values require.InDelta(test.expectedSkew, skew, 0.001, "Expected skew to be %v, got %v", test.expectedSkew, skew) require.InDelta(test.expectedMAD, mad, 0.001, "Expected MAD to be %v, got %v", test.expectedMAD, mad) require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestGetDurationScore(t *testing.T) { tests := []struct { name string datasetMin int64 datasetMax int64 histMin int64 histMax int64 totalBars int32 longestConsecutiveRun int32 minHoursThreshold int32 idealConsistencyHours int32 expectedCoverage float64 expectedConsistency float64 expectedScore float64 expectedError bool }{ { name: "Full Dataset Coverage, Full Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924, histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 24, longestConsecutiveRun: 24, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 1, expectedConsistency: 1, expectedScore: 1, expectedError: false, }, { name: "Full Dataset Coverage, Min Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924, histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 6, longestConsecutiveRun: 0, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 1, expectedConsistency: 0, expectedScore: 1, expectedError: false, }, { name: "First Half Dataset Coverage, Min Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 12*3600, // 12 hours later histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 6, longestConsecutiveRun: 0, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.5, expectedConsistency: 0, expectedScore: 0.5, expectedError: false, }, { name: "Last Half Dataset Coverage, Min Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924, histMax: 1517338924 + 12*3600, // 12 hours later totalBars: 6, longestConsecutiveRun: 0, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.5, expectedConsistency: 0, expectedScore: 0.5, expectedError: false, }, { name: "3/4 Dataset Coverage, Min Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 6*3600, // 6 hours later histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 6, longestConsecutiveRun: 0, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.75, expectedConsistency: 0, expectedScore: 0.75, expectedError: false, }, { name: "Max Consistency", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 12*3600, // 12 hours later histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 12, longestConsecutiveRun: 12, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.5, expectedConsistency: 1, expectedScore: 1, expectedError: false, }, { name: "Max Consistency with min TotalBars (6)", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 12*3600, // 12 hours later histMax: 1517338924 + 18*3600, // 18 hours later totalBars: 6, longestConsecutiveRun: 6, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.25, expectedConsistency: 0.5, expectedScore: 0.5, expectedError: false, }, { name: "Average Consistency with Average Coverage", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 6*3600, // 6 hours later histMax: 1517338924 + 18*3600, // 18 hours later totalBars: 12, longestConsecutiveRun: 6, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0.5, expectedConsistency: 0.5, expectedScore: 0.5, expectedError: false, }, { name: "TotalBars < MinHoursThreshold", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924 + 6*3600, // 6 hours later histMax: 1517338924 + 18*3600, // 18 hours later totalBars: 3, longestConsecutiveRun: 1, minHoursThreshold: 6, idealConsistencyHours: 12, expectedCoverage: 0, expectedConsistency: 0, expectedScore: 0, expectedError: false, }, { name: "Ideal Consistency Hours < 1", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924, histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 12, longestConsecutiveRun: 6, minHoursThreshold: 6, idealConsistencyHours: 0, expectedCoverage: 0, expectedConsistency: 0, expectedScore: 0, expectedError: true, }, { name: "Min Hours Threshold < 1", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later histMin: 1517338924, histMax: 1517338924 + 24*3600, // 24 hours later totalBars: 12, longestConsecutiveRun: 6, minHoursThreshold: 0, idealConsistencyHours: 12, expectedCoverage: 0, expectedConsistency: 0, expectedScore: 0, expectedError: true, }, { name: "Dataset Min > Dataset Max", datasetMin: 1, datasetMax: 0, histMin: 0, histMax: 1, totalBars: 6, longestConsecutiveRun: 12, expectedCoverage: 0, expectedConsistency: 0, expectedScore: 0, expectedError: true, }, { name: "Hist Min > Hist Max", datasetMin: 0, datasetMax: 1, histMin: 1, histMax: 0, totalBars: 6, longestConsecutiveRun: 12, expectedCoverage: 0, expectedConsistency: 0, expectedScore: 0, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function coverage, consistency, score, err := getDurationScore(test.datasetMin, test.datasetMax, test.histMin, test.histMax, test.totalBars, test.longestConsecutiveRun, test.minHoursThreshold, test.idealConsistencyHours) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", false, err) // check the calculated values require.InDelta(test.expectedConsistency, consistency, 0.001, "Expected consistency to be %v, got %v", test.expectedConsistency, consistency) require.InDelta(test.expectedCoverage, coverage, 0.001, "Expected coverage to be %v, got %v", test.expectedCoverage, coverage) require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestGetHistogramScore(t *testing.T) { tests := []struct { name string datasetMin int64 datasetMax int64 tsList []uint32 modalSensitivity float64 bimodalOutlierRemoval int32 minHoursForBimodalAnalysis int32 beaconTimeSpan int32 expectedBinEdges []float64 expectedHistogram []int expectedFreqCount map[int32]int32 expectedTotalBars int32 expectedLongestRun int32 expectedScore float64 expectedError bool }{ { name: "Simple Number List", datasetMin: 1, datasetMax: 11, tsList: []uint32{ 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, }, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 6, beaconTimeSpan: 10, expectedBinEdges: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, expectedHistogram: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedFreqCount: map[int32]int32{1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1, 10: 1}, expectedTotalBars: 10, expectedLongestRun: 10, expectedScore: 0.478, expectedError: false, }, { name: "Connection with Regular Intervals", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later tsList: []uint32{1517338924, 1517338924 + 1*3600, 1517338924 + 2*3600, 1517338924 + 3*3600, 1517338924 + 4*3600, 1517338924 + 5*3600, 1517338924 + 6*3600, 1517338924 + 7*3600, 1517338924 + 8*3600, 1517338924 + 9*3600, 1517338924 + 10*3600, 1517338924 + 11*3600, 1517338924 + 12*3600, 1517338924 + 13*3600, 1517338924 + 14*3600, 1517338924 + 15*3600, 1517338924 + 16*3600, 1517338924 + 17*3600, 1517338924 + 18*3600, 1517338924 + 19*3600, 1517338924 + 20*3600, 1517338924 + 21*3600, 1517338924 + 22*3600, 1517338924 + 23*3600}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 11, beaconTimeSpan: 24, // total edges: 24 + 1 = 25 // step: (maxTS - minTS) / total edges - 1 = 24*3600 / 24 = 3600 // first edge: 1517338924, last edge: 1517338924 + 24*3600 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge expectedBinEdges: []float64{1517338924, 1517342524, 1517346124, 1517349724, 1517353324, 1517356924, 1517360524, 1517364124, 1517367724, 1517371324, 1517374924, 1517378524, 1517382124, 1517385724, 1517389324, 1517392924, 1517396524, 1517400124, 1517403724, 1517407324, 1517410924, 1517414524, 1517418124, 1517421724, 1517425324}, expectedHistogram: []int{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedFreqCount: map[int32]int32{1: 24}, expectedTotalBars: 24, expectedLongestRun: 24, expectedScore: 1, expectedError: false, }, { name: "Connection with Closely-Valued Timestamps", datasetMin: 98, datasetMax: 102, tsList: []uint32{98, 99, 99, 100, 100, 100, 101, 101, 102}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 6, beaconTimeSpan: 8, // total edges: 8 + 1 = 9 // step: (maxTS - minTS) / total edges - 1 = 102 - 98 / 8 = 4/8 = 0.5 // first edge: 98, last edge: 102 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge expectedBinEdges: []float64{98, 98.5, 99, 99.5, 100, 100.5, 101, 101.5, 102}, expectedHistogram: []int{1, 0, 2, 0, 3, 0, 2, 1}, expectedFreqCount: map[int32]int32{1: 2, 2: 2, 3: 1}, expectedTotalBars: 5, expectedLongestRun: 3, // MEAN = 1.125, SD = 1.05327, CV = 0.936, cvScore = 1 - 0.936 = 0.064 expectedScore: 0.064, // score = max(cv score, bimodal fit score) = max (0.064, 0) = 0.064 expectedError: false, }, { name: "Connection with Random Intervals, CV > 1", datasetMin: 0, datasetMax: 1000000, tsList: []uint32{524885, 1, 5000, 98654, 50, 41, 965842, 3, 12001, 200400, 104001, 199999}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 6, beaconTimeSpan: 10, // total edges: 10 + 1 = 11 // step: (maxTS - minTS) / total edges - 1 = 1000000 / 10 = 100000 // first edge: 0, last edge: 1000000 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge expectedBinEdges: []float64{0, 100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000, 1000000}, expectedHistogram: []int{7, 2, 1, 0, 0, 1, 0, 0, 0, 1}, expectedFreqCount: map[int32]int32{1: 3, 2: 1, 7: 1}, expectedTotalBars: 5, expectedLongestRun: 4, // mean = 1.2, sd = 2.0396, cv = sd / abs(mean) = 2.0396 / 1.2 = 1.6997, cvScore = 0 (since cv > 1) expectedScore: 0, // score = max(cv score, bimodal fit score) = max (0, 0) = 0 expectedError: false, }, { name: "Connection with Bimodal Histogram", datasetMin: 0, datasetMax: 100, tsList: []uint32{1, 2, 3, 4, 15, 21, 22, 23, 24, 35, 41, 42, 43, 44, 55, 61, 62, 63, 64, 75, 81, 82, 83, 84, 95}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 6, beaconTimeSpan: 10, expectedBinEdges: []float64{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, expectedHistogram: []int{4, 1, 4, 1, 4, 1, 4, 1, 4, 1}, expectedFreqCount: map[int32]int32{1: 5, 4: 5}, expectedTotalBars: 10, expectedLongestRun: 10, // mean = 2.5, sd = 1.5, cv = sd / abs(mean) = 1.5 / 2.5 = 0.6, cvScore = 1 - 0.6 = 0.4 expectedScore: 1, // score = max(cv score, bimodal fit score) = max (0.4, 1) = 1 expectedError: false, }, { name: "Connection with Histogram that has Gaps and a Wraparound Timestamp Run", datasetMin: 0, datasetMax: 250, tsList: []uint32{10, 20, 100, 110, 110, 220}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 6, beaconTimeSpan: 5, expectedBinEdges: []float64{0, 50, 100, 150, 200, 250}, expectedHistogram: []int{2, 0, 3, 0, 1}, expectedFreqCount: map[int32]int32{1: 1, 2: 1, 3: 1}, expectedTotalBars: 3, expectedLongestRun: 2, // mean = 1.2, sd = 1.1662, cv = sd / abs(mean) = 1.1662 / 1.2 = 0.9718, cvScore = 1 - 0.9718 = 0.0282 expectedScore: 0.028, // score = max(cv score, bimodal fit score) = max (0.028, 0) = 0.028 expectedError: false, }, { name: "Connection with Single Bar Histogram", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later tsList: []uint32{1517338924, 1517338924 + 60, 1517338924 + 120, 1517338924 + 180, 1517338924 + 240, 1517338924 + 300, 1517338924 + 360, 1517338924 + 420, 1517338924 + 480, 1517338924 + 540}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 11, beaconTimeSpan: 24, expectedBinEdges: []float64{1517338924, 1517342524, 1517346124, 1517349724, 1517353324, 1517356924, 1517360524, 1517364124, 1517367724, 1517371324, 1517374924, 1517378524, 1517382124, 1517385724, 1517389324, 1517392924, 1517396524, 1517400124, 1517403724, 1517407324, 1517410924, 1517414524, 1517418124, 1517421724, 1517425324}, expectedHistogram: []int{10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, expectedFreqCount: map[int32]int32{10: 1}, expectedTotalBars: 1, expectedLongestRun: 1, expectedScore: 0, expectedError: false, }, { // this should not happen in practice since min hours for bimodal analysis is vetted when the config is loaded. // this is to ensure that the bimodal fit score is not calculated for histograms with too few bars, as in that case // a histogram with 1-2 bars will always be given a high bimoal fit score as it technically has 1-2 modes name: "Connection with Single Bar Histogram and MinHoursForBimodal Analysis Set to < 3", datasetMin: 1517338924, datasetMax: 1517338924 + 24*3600, // 24 hours later tsList: []uint32{1517338924, 1517338924 + 60, 1517338924 + 120, 1517338924 + 180, 1517338924 + 240, 1517338924 + 300, 1517338924 + 360, 1517338924 + 420, 1517338924 + 480, 1517338924 + 540}, modalSensitivity: 0.05, bimodalOutlierRemoval: 1, minHoursForBimodalAnalysis: 1, // < 3 beaconTimeSpan: 24, expectedBinEdges: []float64{1517338924, 1517342524, 1517346124, 1517349724, 1517353324, 1517356924, 1517360524, 1517364124, 1517367724, 1517371324, 1517374924, 1517378524, 1517382124, 1517385724, 1517389324, 1517392924, 1517396524, 1517400124, 1517403724, 1517407324, 1517410924, 1517414524, 1517418124, 1517421724, 1517425324}, expectedHistogram: []int{10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, expectedFreqCount: map[int32]int32{10: 1}, expectedTotalBars: 1, expectedLongestRun: 1, expectedScore: 0, // this would be 1 if minHoursForBimodalAnalysis was not overridden expectedError: false, }, { name: "Empty Timestamp List", datasetMin: 0, datasetMax: 10, tsList: []uint32{}, expectedError: true, }, { name: "Dataset Min > Dataset Max", datasetMin: 1, datasetMax: 0, tsList: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedError: true, }, { name: "Dataset Min == Dataset Max", datasetMin: 1, datasetMax: 1, tsList: []uint32{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function freqList, freqCount, totalBars, longestRun, score, err := getHistogramScore(test.datasetMin, test.datasetMax, test.tsList, test.modalSensitivity, test.bimodalOutlierRemoval, test.minHoursForBimodalAnalysis, test.beaconTimeSpan) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", false, err) // check the calculated values require.EqualValues(test.expectedHistogram, freqList, "Expected frequency list to be %v, got %v", test.expectedHistogram, freqList) require.EqualValues(test.expectedFreqCount, freqCount, "Expected frequency count to be %v, got %v", test.expectedFreqCount, freqCount) require.EqualValues(test.expectedTotalBars, totalBars, "Expected total bars to be %v, got %v", test.expectedTotalBars, totalBars) require.EqualValues(test.expectedLongestRun, longestRun, "Expected longest run to be %v, got %v", test.expectedLongestRun, longestRun) require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestCalculateBowleySkewness(t *testing.T) { testCases := []struct { name string intervalsBetweenTimestamps []float64 expectedSkewness float64 expectedSkewnessScore float64 expectedError bool }{ { name: "Simple Number List", intervalsBetweenTimestamps: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedSkewness: 0, expectedSkewnessScore: 1, expectedError: false, }, { name: "Connection with Perfect Intervals", // in this case bowley's skew would actually not be calculated since q1 == q2 == q3 and the score would default to 100% // timestamps : 1517338924, 1517338924 + 60, 1517338924 + 120, 1517338924 + 180, 1517338924 + 240, 1517338924 + 300, 1517338924 + 360, 1517338924 + 420, 1517338924 + 480, 1517338924 + 540, // intervals between timestamps: 60, 60, 60, 60, 60, 60, 60, 60, 60 intervalsBetweenTimestamps: []float64{60, 60, 60, 60, 60, 60, 60, 60, 60}, expectedSkewness: 0, expectedSkewnessScore: 1, expectedError: false, }, { name: "Connection with Regular Intervals", intervalsBetweenTimestamps: []float64{98, 99, 99, 100, 100, 100, 101, 101, 102}, // q1 = 99, q2 = 100, q3 = 101 // numerator = q3 + q1 - 2*q2 = 101 + 99 - 2*100 = 101 + 99 - 200 = 0 // IQR = q3 - q1 = 101 - 99 = 2 // skewness = numerator / IQR = 0 / 2 = 0 // skewness score = 1 - skewness = 1 - 0 = 1 expectedSkewness: 0, expectedSkewnessScore: 1, expectedError: false, }, { name: "Connection with Random Intervals Sorted", intervalsBetweenTimestamps: []float64{1, 2, 25, 42, 90, 299, 500, 1500, 4975}, // q1 = 13.5, q2 = 90, q3 = 1000 // numerator = q3 + q1 - 2*q2 = 1000 + 13.5 - 2*90 = 1000 + 13.5 - 180 = 833.5 // IQR = q3 - q1 = 1000 - 13.5 = 986.5 // skewness = numerator / IQR = 833.5 / 986.5 = 0.845 // skewness score = 1 - skewness = 1 - 0.845 = 0.155 expectedSkewness: 0.845, expectedSkewnessScore: 0.155, expectedError: false, }, { name: "Connection with Random Intervals Unsorted", // timestamps : 1517338924, 1517338924 + 1, 1517338924 + 300, 1517338924 + 25, 1517338924 + 5000, 1517338924 + 100, 1517338924 + 1000, 1517338924 + 200, 1517338924 + 1500, 1517338924 + 3000, // intervals between timestamps: 1, 299, 25, 4975, 90, 900, 800, 500, 1500 intervalsBetweenTimestamps: []float64{1, 299, 25, 4975, 90, 2, 42, 500, 1500}, // q1 = 13.5, q2 = 90, q3 = 1000 // numerator = q3 + q1 - 2*q2 = 1000 + 13.5 - 2*90 = 1000 + 13.5 - 180 = 833.5 // IQR = q3 - q1 = 1000 - 13.5 = 986.5 // skewness = numerator / IQR = 833.5 / 986.5 = 0.845 // skewness score = 1 - skewness = 1 - 0.845 = 0.155 expectedSkewness: 0.845, expectedSkewnessScore: 0.155, expectedError: false, }, { name: "Connection with Random Intervals Unsorted 2", intervalsBetweenTimestamps: []float64{524885, 1, 5000, 98654, 50, 41, 965842, 3}, // q1 = 22, q2 = 2525, q3 = 311769.5 // numerator = q3 + q1 - 2*q2 = 311769.5 + 22 - 2*2525 = 311769.5 + 22 - 5050 = 306741.5 // IQR = q3 - q1 = 311769.5 - 22 = 311747.5 // skewness = numerator / IQR = 306741.5 / 311747.5 = 0.983 // skewness score = 1 - skewness = 1 - 0.983 = 0.017 expectedSkewness: 0.983, expectedSkewnessScore: 0.017, expectedError: false, }, { name: "Bowley's Skew Unreliable: Q1 == Q2", intervalsBetweenTimestamps: []float64{7, 8, 7, 8, 7, 8, 7, 8, 7}, // q1 = 7, q2 = 7, q3 = 8 expectedSkewness: 0, // skewness will not be calculated and stay at zero expectedSkewnessScore: 1, // score will stay at the deafault 100% expectedError: false, }, { name: "Bowley's Skew Unreliable: Q2 == Q3", intervalsBetweenTimestamps: []float64{1, 1, 2, 2, 2, 2}, // q1 = 1, q2 = 2, q3 = 2 expectedSkewness: 0, // skewness will not be calculated and stay at zero expectedSkewnessScore: 1, // score will stay at the deafault 100% expectedError: false, }, { name: "Unsorted Slice", intervalsBetweenTimestamps: []float64{7.7, 6.6, 4.4, 1.1, 5.5, 3.3, 2.2}, expectedSkewness: 0, expectedSkewnessScore: 1, expectedError: false, }, { name: "Empty", intervalsBetweenTimestamps: []float64{}, expectedSkewness: 0, expectedSkewnessScore: 0, expectedError: true, }, { name: "Less than 3 elements", // can't calculate skewness with fewer than 3 elements intervalsBetweenTimestamps: []float64{1, 2}, expectedSkewness: 0, expectedSkewnessScore: 0, expectedError: true, }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function skew, score, err := calculateBowleySkewness(test.intervalsBetweenTimestamps) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated values require.InDelta(test.expectedSkewness, skew, 0.001, "Expected skewness to be %v, got %v", test.expectedSkewness, skew) require.InDelta(test.expectedSkewnessScore, score, 0.001, "Expected score to be %v, got %v", test.expectedSkewnessScore, score) }) } } func TestCalculateDistinctCounts(t *testing.T) { tests := []struct { name string sortedInput []float64 expectedDistinct []int64 expectedCounts []int64 expectedMode int64 expectedMaxCount int64 expectError bool }{ { name: "Simple List", sortedInput: []float64{1, 2, 2, 3, 3, 3, 4, 5, 5}, expectedDistinct: []int64{1, 2, 3, 4, 5}, expectedCounts: []int64{1, 2, 3, 1, 2}, expectedMode: 3, expectedMaxCount: 3, expectError: false, }, { name: "Simple List 2", sortedInput: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedDistinct: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, expectedCounts: []int64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedMode: 1, expectedMaxCount: 1, expectError: false, }, { name: "Timestamp Slice", sortedInput: []float64{1517338924, 1517338924, 1517338924, 1609459200, 1609459200, 1612137600, 1612137600}, expectedDistinct: []int64{1517338924, 1609459200, 1612137600}, expectedCounts: []int64{3, 2, 2}, expectedMode: 1517338924, expectedMaxCount: 3, expectError: false, }, { name: "Unsorted Slice", sortedInput: []float64{3.3, 2.2, 4.4, 1.1, 5.5, 3.3, 2.2}, expectedDistinct: []int64{1, 2, 3, 4, 5}, expectedCounts: []int64{1, 2, 2, 1, 1}, expectedMode: 2, expectedMaxCount: 2, expectError: false, }, { name: "Empty Slice", sortedInput: []float64{}, expectedDistinct: nil, expectedCounts: nil, expectedMode: 0, expectedMaxCount: 0, expectError: true, }, { name: "Slice with Only One Element", sortedInput: []float64{1}, expectedDistinct: nil, expectedCounts: nil, expectedMode: 0, expectedMaxCount: 0, expectError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function distinctNumbers, countsArray, mode, maxCount, err := calculateDistinctCounts(test.sortedInput) // check if an error was expected require.EqualValues(test.expectError, err != nil, "Expected error to be %v, got %v", test.expectError, err) // check the calculated values require.EqualValues(test.expectedDistinct, distinctNumbers, "Expected distinctNumbers to be %v, got %v", test.expectedDistinct, distinctNumbers) require.EqualValues(test.expectedCounts, countsArray, "Expected countsArray to be %v, got %v", test.expectedCounts, countsArray) require.EqualValues(test.expectedMode, mode, "Expected mode to be %v, got %v", test.expectedMode, mode) require.EqualValues(test.expectedMaxCount, maxCount, "Expected maxCount to be %v, got %v", test.expectedMaxCount, maxCount) }) } } func TestCalculateMedianAbsoluteDeviation(t *testing.T) { tests := []struct { name string inputData []float64 defaultScore float64 expectedMAD float64 expectedScore float64 expectError bool }{ { name: "Simple List", inputData: []float64{1, 2, 2, 3, 3, 3, 4, 5, 5}, defaultScore: 1, expectedMAD: 1, expectedScore: 0.6667, expectError: false, }, { name: "Simple List 2", inputData: []float64{11, 12, 12, 14, 15, 16}, defaultScore: 1, expectedMAD: 1.5, expectedScore: 0.8846, expectError: false, }, { name: "Bigger Numbers", inputData: []float64{1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500}, defaultScore: 1, expectedMAD: 1250, expectedScore: 0.6154, expectError: false, }, { name: "Unsorted Slice", inputData: []float64{4000, 2500, 2000, 1500, 5500, 3500, 1000, 4500, 5000, 3000}, defaultScore: 1, expectedMAD: 1250, expectedScore: 0.6154, expectError: false, }, { name: "Empty Slice", inputData: []float64{}, defaultScore: 1, expectedMAD: 0, expectedScore: 0, expectError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function mad, score, err := calculateMedianAbsoluteDeviation(test.inputData, test.defaultScore) // check if an error was expected require.EqualValues(test.expectError, err != nil, "error should match expected value") // check the calculated MAD require.InDelta(test.expectedMAD, mad, 0.001, "Expected MAD to be %v, got %v", test.expectedMAD, mad) // check the calculated score require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestComputeHistogramBins(t *testing.T) { tests := []struct { name string startTime int64 endTime int64 numBins int32 expectedBinEdges []float64 expectedError bool }{ { name: "Simple List", startTime: 0, endTime: 10, numBins: 2, expectedBinEdges: []float64{0, 5, 10}, expectedError: false, }, { name: "Total Bins Less Than Time Range", startTime: 0, endTime: 100, numBins: 5, expectedBinEdges: []float64{0, 20, 40, 60, 80, 100}, expectedError: false, }, { name: "Total Bins Equal to Time Range", startTime: 0, endTime: 24, numBins: 24, // total edges: 24 + 1 = 25 // step: (maxTS - minTS) / total edges - 1 = 24 / 24 = 1 // first edge: 1, last edge: 24 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge // expectedBinEdges: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}, expectedBinEdges: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}, expectedError: false, }, { name: "Total Bins Exceed Time Range", startTime: 0, endTime: 10, numBins: 24, // total edges: 10 + 1 = 11 // step: (maxTS - minTS) / numBins - 1 = 10 / 24 = 0.4167 // first edge: 0, last edge: 10 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge expectedBinEdges: []float64{0, 0.4167, 0.8333, 1.25, 1.6667, 2.0833, 2.5, 2.9167, 3.3333, 3.75, 4.1667, 4.5833, 5, 5.4167, 5.8333, 6.25, 6.6667, 7.0833, 7.5, 7.9167, 8.3333, 8.75, 9.1667, 9.5833, 10}, expectedError: false, }, { name: "Timestamp List", startTime: 1517336042, endTime: 1517422440, numBins: 24, // total edges: 24 + 1 = 25 // step: (maxTS - minTS) / total edges - 1 = 1517422440 - 1517336042 / 24 = 86398 / 24 = 3599.9167 // first edge: 1517336042, last edge: 1517422440 // bin edges: first edge, first edge + step, first edge + 2*step, ... , last edge expectedBinEdges: []float64{ 1517336042, 1517336042 + 3599.9167, 1517336042 + 2*3599.9167, 1517336042 + 3*3599.9167, 1517336042 + 4*3599.9167, 1517336042 + 5*3599.9167, 1517336042 + 6*3599.9167, 1517336042 + 7*3599.9167, 1517336042 + 8*3599.9167, 1517336042 + 9*3599.9167, 1517336042 + 10*3599.9167, 1517336042 + 11*3599.9167, 1517336042 + 12*3599.9167, 1517336042 + 13*3599.9167, 1517336042 + 14*3599.9167, 1517336042 + 15*3599.9167, 1517336042 + 16*3599.9167, 1517336042 + 17*3599.9167, 1517336042 + 18*3599.9167, 1517336042 + 19*3599.9167, 1517336042 + 20*3599.9167, 1517336042 + 21*3599.9167, 1517336042 + 22*3599.9167, 1517336042 + 23*3599.9167, 1517422440, }, expectedError: false, }, { name: "Invalid Number of Bins", startTime: 0, endTime: 100, numBins: 0, expectedError: true, }, { name: "Invalid Time Range - End Time < Start Time", startTime: 10, endTime: 5, numBins: 5, expectedError: true, }, { name: "Invalid Time Range - End Time == Start Time", startTime: 0, endTime: 0, numBins: 5, expectedBinEdges: nil, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function divs, err := computeHistogramBins(test.startTime, test.endTime, test.numBins) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", false, err) // check the calculated values // require.EqualValues(test.expectedBinEdges, divs, "Expected bin edges to be %v, got %v", test.expectedBinEdges, divs) require.InDeltaSlice(test.expectedBinEdges, divs, 0.01, "Expected bin edges to be %v, got %v", test.expectedBinEdges, divs) }) } } func TestCalculateCoefficientOfVariationScore(t *testing.T) { tests := []struct { name string freqList []int total int expectedScore float64 expectedError bool }{ { name: "Uniform Distribution", freqList: []int{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, expectedScore: 1, expectedError: false, }, { name: "Normal Distribution", freqList: []int{950, 970, 990, 1010, 1030, 1050, 1070}, total: 7150, // sd = 40, mean = 1010, CV = 0.0396 expectedScore: 0.96, // score = round((1-0.0396)*1000) = 960, 960/1000 = 0.96 expectedError: false, }, { name: "CV less than 1", freqList: []int{1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009}, // CV = 0.0028594 expectedScore: 0.9971406, // score = 1-0.0028594 = 0.9971406 expectedError: false, }, { name: "CV greater than 1", freqList: []int{1, 5, 10, 50, 100, 500, 1000, 5000, 10000}, expectedScore: 0.0, // score should be 0 for CV > 1 expectedError: false, }, { name: "Empty List", freqList: []int{}, expectedScore: 0, expectedError: true, }, { name: "Negative Slice Values", freqList: []int{-5, -10, -7, -3, -8}, expectedScore: 0, expectedError: true, }, { name: "Total is zero", freqList: []int{0, 0, 0, 0, 0}, expectedScore: 0, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) score, err := calculateCoefficientOfVariationScore(test.freqList) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // check the calculated values require.InDelta(test.expectedScore, score, 0.001, "Expected score to be %v, got %v", test.expectedScore, score) }) } } func TestCalculateBimodalFitScore(t *testing.T) { tests := []struct { name string freqCount map[int32]int32 totalBars int32 modalOutlierRemoval int32 minHoursForBimodalAnalysis int32 expectedScore float64 expectedError bool }{ { name: "Perfect Single Modal", freqCount: map[int32]int32{ 1: 10, }, totalBars: 10, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, expectedScore: 1, expectedError: false, }, { name: "Imperfect Single Modal", freqCount: map[int32]int32{ 1: 50, 2: 5, }, totalBars: 60, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, // (50+5)/max(60-1, 1) = 55/59 = 0.932 expectedScore: 0.932, expectedError: false, }, { name: "Perfect Single Modal with Outlier Removal", freqCount: map[int32]int32{ 1: 10, 2: 1, }, totalBars: 11, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, expectedScore: 1, expectedError: false, }, { name: "Imperfect Single Modal with Outlier Removal", freqCount: map[int32]int32{ 1: 50, 2: 1, 3: 1, }, totalBars: 56, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, // (50+1/56-1) = 51/55 = 0.927 expectedScore: 0.927, expectedError: false, }, { name: "Perfect Bimodal", freqCount: map[int32]int32{ 1: 10, 2: 10, }, totalBars: 20, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, expectedScore: 1, expectedError: false, }, { name: "Imperfect Bimodal", freqCount: map[int32]int32{ 1: 50, 2: 30, 3: 1, 4: 2, }, totalBars: 83, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, // (50+30)/(83-1) = 80/82 = 0.976 expectedScore: 0.976, expectedError: false, }, { name: "Perfect Bimodal with Outlier Removal", freqCount: map[int32]int32{ 1: 10, 2: 10, 3: 1, }, totalBars: 21, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, expectedScore: 1, expectedError: false, }, { name: "Imperfect Bimodal with Outlier Removal", freqCount: map[int32]int32{ 1: 50, 2: 30, 3: 1, 4: 2, 5: 1, }, totalBars: 84, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 10, // (50+30)/(84-1) = 80/83 = 0.964 expectedScore: 0.964, expectedError: false, }, { name: "Number of Bars < Minimum Hours For Bimodal Analysis", freqCount: map[int32]int32{ 1: 2, 2: 3, }, totalBars: 5, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 11, expectedScore: 0.0, expectedError: false, }, { // this should not happen in practice, as this value will get vetted when the config is loaded name: "Minimum Hours For Bimodal Analysis < 3", freqCount: map[int32]int32{ 1: 1, 2: 1, }, totalBars: 2, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 2, expectedScore: 0.0, // score would be 100% if the setting was not overridden expectedError: false, }, { name: "Number of Bars <= 0", freqCount: map[int32]int32{1: 2, 2: 3}, totalBars: 0, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 11, expectedScore: 0.0, expectedError: true, }, { name: "Empty Frequency Count", freqCount: map[int32]int32{}, totalBars: 20, modalOutlierRemoval: 1, minHoursForBimodalAnalysis: 11, expectedScore: 0.0, expectedError: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { require := require.New(t) // run the function score, err := calculateBimodalFitScore(tc.freqCount, tc.totalBars, tc.modalOutlierRemoval, tc.minHoursForBimodalAnalysis) // check if an error was expected require.EqualValues(tc.expectedError, err != nil, "Expected error to be %v, got %v", tc.expectedError, err) // check the calculated values require.InDelta(tc.expectedScore, score, 0.001, "score should match expected value") }) } } func TestCreateHistogram(t *testing.T) { tests := []struct { name string binEdges []float64 tsList []uint32 modalSensitivity float64 expectedHistogram []int expectedFreqCount map[int32]int32 expectedTotalBars int32 expectedLongestRun int32 expectedError bool errorContains string }{ { name: "Simple Flat Histogram", binEdges: []float64{0, 10, 20, 30}, tsList: []uint32{1, 5, 11, 15, 21, 25}, modalSensitivity: 0.05, expectedHistogram: []int{2, 2, 2}, expectedFreqCount: map[int32]int32{2: 3}, expectedTotalBars: 3, expectedLongestRun: 3, expectedError: false, }, { name: "Multiple Bins, but All Timestamps in One", binEdges: []float64{0, 100, 200}, tsList: []uint32{10, 20, 30, 40}, modalSensitivity: 0.05, expectedHistogram: []int{4, 0}, expectedFreqCount: map[int32]int32{4: 1}, expectedTotalBars: 1, expectedLongestRun: 1, expectedError: false, }, { name: "Single Bin", binEdges: []float64{0, 100}, tsList: []uint32{10, 20, 30, 40, 50}, modalSensitivity: 0.05, expectedHistogram: []int{5}, expectedFreqCount: map[int32]int32{5: 1}, expectedTotalBars: 1, expectedLongestRun: 1, expectedError: false, }, { name: "Histogram with Gaps and Wraparound Timestamp Run", binEdges: []float64{0, 50, 100, 150, 200, 250}, tsList: []uint32{10, 20, 100, 110, 110, 220}, modalSensitivity: 0.05, expectedHistogram: []int{2, 0, 3, 0, 1}, expectedFreqCount: map[int32]int32{1: 1, 2: 1, 3: 1}, expectedTotalBars: 3, expectedLongestRun: 2, expectedError: false, }, { name: "Bimodal Histogram", binEdges: []float64{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, tsList: []uint32{1, 2, 3, 4, 15, 21, 22, 23, 24, 35, 41, 42, 43, 44, 55, 61, 62, 63, 64, 75, 81, 82, 83, 84, 95}, modalSensitivity: 0.05, expectedHistogram: []int{4, 1, 4, 1, 4, 1, 4, 1, 4, 1}, expectedFreqCount: map[int32]int32{1: 5, 4: 5}, expectedTotalBars: 10, expectedLongestRun: 10, expectedError: false, }, { name: "Last Value in Value List Equal to Last Bin Edge", tsList: []uint32{98, 99, 99, 100, 100, 100, 101, 101, 102}, modalSensitivity: 0.05, binEdges: []float64{98, 98.5, 99, 99.5, 100, 100.5, 101, 101.5, 102}, expectedHistogram: []int{1, 0, 2, 0, 3, 0, 2, 1}, expectedFreqCount: map[int32]int32{1: 2, 2: 2, 3: 1}, expectedTotalBars: 5, expectedLongestRun: 3, expectedError: false, }, { name: "High Modal Sensitivity", binEdges: []float64{0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}, tsList: []uint32{ 10, 15, 20, 25, 30, 35, 40, 45, 50, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, 300, 305, 330, 335, 337, 339, 343, 349, 380, 399, 400, 404, 424, 434, 437, 439, 443, 449, 480, 495, 499, 510, 515, 520, 525, 530, 535, 540, 545, 550, 610, 615, 620, 625, 630, 635, 640, 645, 650, 655, 710, 715, 720, 725, 730, 735, 740, 745, 750, 755, 760, 810, 815, 820, 825, 830, 835, 840, 845, 850, 855, 900, 910, 920, 930, 940, 960, 970, 980, 990, }, modalSensitivity: 0.2, expectedHistogram: []int{9, 10, 11, 10, 11, 9, 10, 11, 10, 9}, expectedFreqCount: map[int32]int32{9: 10}, expectedTotalBars: 10, expectedLongestRun: 10, expectedError: false, }, { name: "Low Modal Sensitivity", binEdges: []float64{0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}, tsList: []uint32{ 10, 15, 20, 25, 30, 35, 40, 45, 50, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, 300, 305, 330, 335, 337, 339, 343, 349, 380, 399, 400, 404, 424, 434, 437, 439, 443, 449, 480, 495, 499, 510, 515, 520, 525, 530, 535, 540, 545, 550, 610, 615, 620, 625, 630, 635, 640, 645, 650, 655, 710, 715, 720, 725, 730, 735, 740, 745, 750, 755, 760, 810, 815, 820, 825, 830, 835, 840, 845, 850, 855, 900, 910, 920, 930, 940, 960, 970, 980, 990, }, modalSensitivity: 0.05, expectedHistogram: []int{9, 10, 11, 10, 11, 9, 10, 11, 10, 9}, expectedFreqCount: map[int32]int32{9: 3, 10: 4, 11: 3}, expectedTotalBars: 10, expectedLongestRun: 10, expectedError: false, }, { name: "Bimodal - High Sensitivity", binEdges: []float64{0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000}, tsList: []uint32{ 10, 15, 20, 25, 30, 35, 40, 45, 50, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, 300, 305, 330, 335, 337, 339, 343, 349, 380, 399, 400, 404, 424, 434, 437, 439, 443, 449, 480, 495, 499, 510, 515, 520, 525, 530, 535, 540, 545, 550, 610, 615, 620, 625, 630, 635, 640, 645, 650, 655, 710, 715, 720, 725, 730, 735, 740, 745, 750, 755, 760, 810, 815, 820, 825, 830, 835, 840, 845, 850, 855, 900, 910, 920, 930, 940, 960, 970, 980, 990, 1010, 1011, 1055, 1110, 1155, 1210, 1260, 1270, 1300, 1330, 1404, 1440, 1445, 1550, 1560, 1600, 1650, 1660, 1770, 1780, 1880, 1890, 1899, 1999, 2000, }, modalSensitivity: 0.3, expectedHistogram: []int{9, 10, 11, 10, 11, 9, 10, 11, 10, 9, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2}, expectedFreqCount: map[int32]int32{0: 10, 8: 10}, expectedTotalBars: 20, expectedLongestRun: 20, }, { name: "Bimodal - Low Sensitivity", binEdges: []float64{0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000}, tsList: []uint32{ 10, 15, 20, 25, 30, 35, 40, 45, 50, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, 300, 305, 330, 335, 337, 339, 343, 349, 380, 399, 400, 404, 424, 434, 437, 439, 443, 449, 480, 495, 499, 510, 515, 520, 525, 530, 535, 540, 545, 550, 610, 615, 620, 625, 630, 635, 640, 645, 650, 655, 710, 715, 720, 725, 730, 735, 740, 745, 750, 755, 760, 810, 815, 820, 825, 830, 835, 840, 845, 850, 855, 900, 910, 920, 930, 940, 960, 970, 980, 990, 1010, 1011, 1055, 1110, 1155, 1210, 1260, 1270, 1300, 1330, 1404, 1440, 1445, 1550, 1560, 1600, 1650, 1660, 1770, 1780, 1880, 1890, 1899, 1999, 2000, }, modalSensitivity: 0.05, expectedHistogram: []int{9, 10, 11, 10, 11, 9, 10, 11, 10, 9, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2}, expectedFreqCount: map[int32]int32{9: 3, 10: 4, 11: 3, 2: 5, 3: 5}, expectedTotalBars: 20, expectedLongestRun: 20, }, { name: "Unsorted Slice", binEdges: []float64{0, 100, 200, 300, 400, 500}, tsList: []uint32{450, 10, 30, 205, 299}, modalSensitivity: 0.05, expectedHistogram: []int{2, 0, 2, 0, 1}, expectedFreqCount: map[int32]int32{1: 1, 2: 2}, expectedTotalBars: 3, expectedLongestRun: 2, expectedError: false, }, { name: "Invalid Bin Edges", binEdges: []float64{10}, tsList: []uint32{15, 22, 35}, modalSensitivity: 0.05, expectedHistogram: []int(nil), expectedFreqCount: map[int32]int32(nil), expectedTotalBars: 0, expectedLongestRun: 0, expectedError: true, errorContains: "binEdges must contain at least 2 elements", }, { name: "Empty Timestamps Slice", binEdges: []float64{10, 20, 30, 40}, tsList: []uint32{}, modalSensitivity: 0.05, expectedHistogram: []int(nil), expectedFreqCount: map[int32]int32(nil), expectedTotalBars: 0, expectedLongestRun: 0, expectedError: true, errorContains: "timestamp slice must not be empty", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function frequencies, freqCount, totalBars, longestRun, err := createHistogram(test.binEdges, test.tsList, test.modalSensitivity) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err) // if test.errorContains != "" { // require.Contains(err.Error(), test.errorContains) // } // check the calculated values require.EqualValues(test.expectedHistogram, frequencies, "Expected frequencies to be %v, got %v", test.expectedHistogram, frequencies) require.EqualValues(test.expectedFreqCount, freqCount, "Expected freqCount to be %v, got %v", test.expectedFreqCount, freqCount) require.EqualValues(test.expectedTotalBars, totalBars, "Expected totalBars to be %v, got %v", test.expectedTotalBars, totalBars) require.EqualValues(test.expectedLongestRun, longestRun, "Expected longestRun to be %v, got %v", test.expectedLongestRun, longestRun) }) } } func TestGetFrequencyCounts(t *testing.T) { tests := []struct { name string histogram []int bimodalSensitivity float64 expectedCounts map[int32]int32 totalBars int32 longestRun int32 expectedError bool }{ { name: "Simple Flat Histogram", histogram: []int{2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, bimodalSensitivity: 0.05, expectedCounts: map[int32]int32{2: 10}, totalBars: 10, longestRun: 10, expectedError: false, }, { name: "Simple Bimodal Histogram", histogram: []int{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}, bimodalSensitivity: 0.05, expectedCounts: map[int32]int32{2: 10, 4: 10}, totalBars: 20, longestRun: 20, expectedError: false, }, { name: "Flat Histogram with Gaps", histogram: []int{2, 0, 2, 0, 2, 0, 2, 0, 2, 0}, bimodalSensitivity: 0.05, expectedCounts: map[int32]int32{2: 5}, totalBars: 5, longestRun: 1, expectedError: false, }, { name: "Bimodal Histogram with Gaps", histogram: []int{2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 0}, bimodalSensitivity: 0.05, expectedCounts: map[int32]int32{2: 5, 4: 4}, totalBars: 9, longestRun: 5, expectedError: false, }, // histogram: []int{9999, 9950, 10000, 10042, 10001, 9960}, { name: "Flat Histogram with Closely-Valued Bars", histogram: []int{900, 901, 899, 900, 899, 902, 900, 901}, bimodalSensitivity: 0.05, // bin size = ceiling(largestnum * 0.05) = 902 * 0.05 = 45.1 = 46 // first bin: // math.Floor(float64(frequency)/binSize) * binSize = 900/46 = 19.5652 = 19 * 46 = 874 // all values within 46 of 874 will be grouped together expectedCounts: map[int32]int32{874: 8}, totalBars: 8, longestRun: 8, expectedError: false, }, { name: "Bimodal Histogram with Closely-Valued Bars", histogram: []int{100, 105, 100, 300, 305, 300}, bimodalSensitivity: 0.1, // bin size = ceiling(largestnum * 0.1) = 305 * 0.1 = 30.5 = 31 // first bin: // math.Floor(float64(frequency)/binSize) * binSize = 100/31 = 3.2258 = 3 * 31 = 93 // second bin: // math.Floor(float64(frequency)/binSize) * binSize = 300/31 = 9.6774 = 9 * 31 = 279 expectedCounts: map[int32]int32{93: 3, 279: 3}, totalBars: 6, longestRun: 6, expectedError: false, }, { name: "Multimodal Histogram", histogram: []int{100, 105, 100, 300, 305, 300, 500, 505, 500, 700, 705, 700}, bimodalSensitivity: 0.1, // bin size = ceiling(largestnum * 0.1) = 705 * 0.1 = 70.5 = 71 // first bin: // math.Floor(float64(frequency)/binSize) * binSize = 100/71 = 1.4085 = 1 * 71 = 71 // second bin: // math.Floor(float64(frequency)/binSize) * binSize = 300/71 = 4.2254 = 4 * 71 = 284 // third bin: // math.Floor(float64(frequency)/binSize) * binSize = 500/71 = 7.0423 = 7 * 71 = 497 // fourth bin: // math.Floor(float64(frequency)/binSize) * binSize = 700/71 = 9.8592 = 9 * 71 = 639 expectedCounts: map[int32]int32{71: 3, 284: 3, 497: 3, 639: 3}, totalBars: 12, longestRun: 12, expectedError: false, }, { name: "High Sensitivity - Fine Grained", histogram: []int{100, 100, 101, 101, 102, 200, 200, 201, 201, 202, 300, 300, 301, 301, 302}, bimodalSensitivity: 0.001, // High sensitivity to minor variations expectedCounts: map[int32]int32{100: 2, 101: 2, 102: 1, 200: 2, 201: 2, 202: 1, 300: 2, 301: 2, 302: 1}, totalBars: 15, longestRun: 15, // Maximum consecutive run is 2 }, { name: "Low Sensitivity - Coarse Grained", histogram: []int{100, 100, 101, 101, 102, 200, 200, 201, 201, 202, 300, 300, 301, 301, 302}, bimodalSensitivity: 0.1, // Lower sensitivity, more forgiving to variations // bin size = ceiling(largestnum * 0.1) = 302 * 0.1 = 30.2 = 31 // first bin: // math.Floor(float64(frequency)/binSize) * binSize = 100/31 = 3.2258 = 3 * 31 = 93 // second bin: // math.Floor(float64(frequency)/binSize) * binSize = 200/31 = 6.4516 = 6 * 31 = 186 // third bin: // math.Floor(float64(frequency)/binSize) * binSize = 300/31 = 9.6774 = 9 * 31 = 279 expectedCounts: map[int32]int32{93: 5, 186: 5, 279: 5}, totalBars: 15, longestRun: 15, }, { name: "Wraparound Longest Run with Gap", histogram: []int{2002, 2000, 1999, 2005, 1990, 0, 0, 0, 0, 0, 0, 2000, 2001, 2002, 2003, 2004}, bimodalSensitivity: 0.05, // bin size = ceiling(largestnum * 0.05) = 2005 * 0.05 = 100.25 = 101 // first bin: // math.Floor(float64(frequency)/binSize) * binSize = 2000/101 = 19.8019 = 19 * 101 = 1919 expectedCounts: map[int32]int32{1919: 10}, totalBars: 10, longestRun: 10, expectedError: false, }, { name: "Empty Histogram", histogram: []int{}, bimodalSensitivity: 0.05, expectedCounts: nil, totalBars: 0, longestRun: 0, expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { require := require.New(t) // run the function freqCounts, totalBars, longestRun, err := getFrequencyCounts(test.histogram, test.bimodalSensitivity) // check if an error was expected require.EqualValues(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, false) // check the calculated values require.EqualValues(test.expectedCounts, freqCounts, "Expected freqCounts to be %v, got %v", test.expectedCounts, freqCounts) require.EqualValues(test.totalBars, totalBars, "Expected totalBars to be %v, got %v", test.totalBars, totalBars) require.EqualValues(test.longestRun, longestRun, "Expected longestRun to be %v, got %v", test.longestRun, longestRun) }) } }