Add ability to perform zone transfers (#48)

* Store zone transfer records Co-Authored-By: moth <25512187+0x6d6f7468@users.noreply.github.com> * Update config * Add tests * Tests, connectivity test * Update tests --------- Co-authored-by: moth <25512187+0x6d6f7468@users.noreply.github.com>
2026-06-08 13:02:45 +00:00 · 2025-05-13 18:31:40 -04:00
parent 3555d9b095
commit b23387c2e2
12 changed files with 688 additions and 27 deletions
@@ -26,6 +26,7 @@ func Commands() []*cli.Command {
 		DeleteCommand,
 		ListCommand,
 		ValidateConfigCommand,
+		ZoneTransferCommand,
 	}
 }

@@ -0,0 +1,74 @@
+package cmd
+
+import (
+	"context"
+
+	"github.com/activecm/rita/v5/config"
+	"github.com/activecm/rita/v5/database"
+	"github.com/activecm/rita/v5/zonetransfer"
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ZoneTransferCommand = &cli.Command{
+	Name:        "zone-transfer",
+	Usage:       "perform a zone transfer on the configured domain",
+	UsageText:   "zone-transfer",
+	Description: "performs a zone transfer on the configured domain",
+	Args:        false,
+	Flags: []cli.Flag{
+		ConfigFlag(false),
+	},
+	Action: func(cCtx *cli.Context) error {
+
+		// check if too many arguments were provided
+		if cCtx.NArg() > 0 {
+			return ErrTooManyArguments
+		}
+
+		// set up file system interface
+		afs := afero.NewOsFs()
+
+		// load config file
+		cfg, err := config.ReadFileConfig(afs, cCtx.String("config"))
+		if err != nil {
+			return err
+		}
+
+		// run the delete command
+		if err := runZoneTransferCmd(cfg); err != nil {
+			return err
+		}
+
+		// check for updates after running the command
+		if err := CheckForUpdate(cfg); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+func runZoneTransferCmd(cfg *config.Config) error {
+
+	// connect to server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	if err != nil {
+		return err
+	}
+
+	// create tables if they don't already exist
+	if err := server.CreateServerDBTables(); err != nil {
+		return err
+	}
+
+	zt, err := zonetransfer.NewZoneTransfer(server, cfg)
+	if err != nil {
+		return err
+	}
+
+	if err := zt.PerformZoneTransfer(); err != nil {
+		return err
+	}
+	return nil
+}
@@ -0,0 +1,38 @@
+package cmd_test
+
+import (
+	"github.com/activecm/rita/v5/cmd"
+	"github.com/urfave/cli/v2"
+
+	"github.com/stretchr/testify/require"
+)
+
+func (c *CmdTestSuite) TestZoneTransfer() {
+	commands := []*cli.Command{cmd.ZoneTransferCommand}
+	flags := []cli.Flag{}
+
+	// create a new app and context
+	app, ctx := setupTestApp(commands, flags)
+
+	tests := []struct {
+		name          string
+		args          []string
+		expectedError error
+	}{
+		{name: "Too Many Arguments", args: []string{"app", "zone-transfer", "bingbong"}, expectedError: cmd.ErrTooManyArguments},
+	}
+	for _, test := range tests {
+		c.Run(test.name, func() {
+			require := require.New(c.T())
+			// run app with test.args
+			err := app.RunContext(ctx, test.args)
+			if test.expectedError != nil {
+				require.Error(err, "error should not be nil")
+				require.Contains(err.Error(), test.expectedError.Error(), "error should contain expected value")
+			} else {
+				require.NoError(err, "error should be nil")
+			}
+		})
+	}
+
+}
@@ -39,11 +39,12 @@ const (

 type (
 	Config struct {
-		Env       Env `json:"env" validate:"required"`
-		RITA      `validate:"required"`
-		Filtering Filtering `json:"filtering" validate:"required"`
-		Scoring   Scoring   `json:"scoring" validate:"required"`
-		Modifiers Modifiers `json:"modifiers" validate:"required"`
+		Env          Env `json:"env" validate:"required"`
+		RITA         `validate:"required"`
+		Filtering    Filtering    `json:"filtering" validate:"required"`
+		Scoring      Scoring      `json:"scoring" validate:"required"`
+		Modifiers    Modifiers    `json:"modifiers" validate:"required"`
+		ZoneTransfer ZoneTransfer `json:"zone_transfer"`
 	}

 	Env struct { // set by .env file
@@ -82,6 +83,12 @@ type (
 		ThreatScoring `validate:"required"`
 	}

+	ZoneTransfer struct {
+		Enabled    bool   `ch:"enabled" json:"enabled"`
+		DomainName string `ch:"domain_name" json:"domain_name" validate:"required_if=Enabled true,omitempty,fqdn"`
+		NameServer string `ch:"name_server" json:"name_server" validate:"required_if=Enabled true,omitempty,hostname_port"`
+	}
+
 	BeaconScoring struct {
 		UniqueConnectionThreshold         int64           `ch:"unique_connection_threshold" json:"unique_connection_threshold" validate:"gte=4"`
 		TimestampScoreWeight              float64         `ch:"timestamp_score_weight" json:"timestamp_score_weight" validate:"gte=0,lte=1"`
@@ -592,6 +599,11 @@ func defaultConfig() Config {

 			MIMETypeMismatchScoreIncrease: 0.15, // +15% score for connections with mismatched MIME type/URI
 		},
+		ZoneTransfer: ZoneTransfer{
+			Enabled:    false,
+			DomainName: "",
+			NameServer: "",
+		},
 	}
 }

@@ -470,6 +470,26 @@ func TestConfig_Validate(t *testing.T) {
 			{name: "MIMETypeMismatchScoreIncrease > Range", config: func(cfg *Config) { cfg.Modifiers.MIMETypeMismatchScoreIncrease = 1.1 }, expectedErrs: []string{"'MIMETypeMismatchScoreIncrease' failed on the 'lte' tag"}},
 			{name: "Empty Struct", config: func(cfg *Config) { cfg.Modifiers = Modifiers{} }, expectedErrs: []string{"'Modifiers' failed on the 'required' tag"}},
 		}},
+		{"Zone Transfer", []testCase{
+			{name: "Disabled, Empty Domain Name", config: func(cfg *Config) { cfg.ZoneTransfer.DomainName = "" }},
+			{name: "Disabled, Empty Name Server", config: func(cfg *Config) { cfg.ZoneTransfer.NameServer = "" }},
+			{name: "Enabled, Empty Domain Name", config: func(cfg *Config) { cfg.ZoneTransfer.DomainName = ""; cfg.ZoneTransfer.Enabled = true }, expectedErrs: []string{"'DomainName' failed on the 'required_if' tag", "'NameServer' failed on the 'required_if' tag"}},
+			{name: "Enabled, Empty Name Server", config: func(cfg *Config) { cfg.ZoneTransfer.NameServer = ""; cfg.ZoneTransfer.Enabled = true }, expectedErrs: []string{"'DomainName' failed on the 'required_if' tag", "'NameServer' failed on the 'required_if' tag"}},
+			{name: "Invalid Domain Name", config: func(cfg *Config) {
+				cfg.ZoneTransfer.DomainName = "192.168.0.1"
+				cfg.ZoneTransfer.NameServer = "dc1.bug.corp:53"
+				cfg.ZoneTransfer.Enabled = true
+			}, expectedErrs: []string{"'DomainName' failed on the 'fqdn' tag"}},
+			{name: "Invalid Name Server", config: func(cfg *Config) {
+				cfg.ZoneTransfer.DomainName = "bug.corp."
+				cfg.ZoneTransfer.NameServer = "bug.corp"
+				cfg.ZoneTransfer.Enabled = true
+			}, expectedErrs: []string{"'NameServer' failed on the 'hostname_port' tag"}},
+			{name: "Disabled but Valid", config: func(cfg *Config) {
+				cfg.ZoneTransfer.DomainName = "bug.corp."
+				cfg.ZoneTransfer.NameServer = "dc1.bug.corp:53"
+			}},
+		}},
 	}
 	for _, test := range tests {
 		t.Run(test.group, func(t *testing.T) {
@@ -61,6 +61,14 @@ func (server *ServerConn) createMetaDatabase() error {
 		return err
 	}

+	if err := server.createMetaDatabasePerformedZoneTransfersTable(); err != nil {
+		return err
+	}
+
+	if err := server.createMetaDatabaseZoneTransferTable(); err != nil {
+		return err
+	}
+
 	if err := server.createThreatIntelTables(); err != nil {
 		return err
 	}
@@ -166,6 +174,40 @@ func (server *ServerConn) createMetaDatabaseMinMaxTable() error {
 	return nil
 }

+func (server *ServerConn) createMetaDatabasePerformedZoneTransfersTable() error {
+	err := server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.performed_zone_transfers (
+			domain_name String,
+			name_server String,
+			serial_soa UInt32,
+			mbox String,
+			is_ixfr Bool, -- for debugging
+			performed_at DateTime()
+		)
+		ENGINE = ReplacingMergeTree(performed_at)
+		PRIMARY KEY (domain_name, name_server)
+	`)
+
+	return err
+}
+
+func (server *ServerConn) createMetaDatabaseZoneTransferTable() error {
+	err := server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.zone_transfer (
+		performed_at DateTime(),
+		domain_name String,
+		name_server String,
+		hostname String,
+		ip IPv6,
+		ttl Int32, -- RFC2181 defines TTLs as int32
+	) ENGINE = MergeTree()
+	PRIMARY KEY (domain_name, name_server, hostname, ip) 
+	ORDER BY (domain_name, name_server, hostname, ip, performed_at)
+	`)
+
+	return err
+}
+
 // createMetaDatabaseFilesTable creates the metadatabase.files table
 func (server *ServerConn) createMetaDatabaseSampleDBsTable() error {
 	err := server.Conn.Exec(server.ctx, `
@@ -1447,6 +1447,23 @@ func (db *DB) createExplodedDNSTable(ctx context.Context) error {
 	return err
 }

+func (db *DB) createIPToHostnameTable(ctx context.Context) error {
+	if err := db.Conn.Exec(ctx, `--sql
+	
+	CREATE TABLE IF NOT EXISTS {database:Identifier}.ip_to_hostname (
+		import_time DateTime(),
+		import_id FixedString(16),
+		hostname String,
+		ip IPv6,
+		ttl Int32, -- RFC2181 defines TTLs as int32
+	) ENGINE = MergeTree()
+	PRIMARY KEY (import_id, hostname, ip) 
+	`); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (db *DB) createSensorDBTables() error {
 	ctx := db.QueryParameters(clickhouse.Parameters{
 		"database": db.selected,
@@ -1569,6 +1586,11 @@ func (db *DB) createSensorDBTables() error {
 		return err
 	}

+	err = db.createIPToHostnameTable(ctx)
+	if err != nil {
+		return err
+	}
+
 	if err := db.createMinMaxMaterializedView(); err != nil {
 		return err
 	}
@@ -21,6 +21,7 @@ var AnalysisSnapshotHourTTLs = []string{"big_ol_histogram", "tls_proto", "http_p
 var AnalysisSnapshotAnalyzedAtTTLs = []string{"threat_mixtape"}
 var MetaDatabaseTTLs = []string{"historical_first_seen", "files"}
 var MetaDatabaseYearTTLS = []string{"imports"}
+var ZoneTransferTTLs = []string{"performed_zone_transfers", "zone_transfer"}

 func (db *DB) createLogTableTTLs() error {
 	if !db.Rolling {
@@ -171,5 +172,17 @@ func (server *ServerConn) createMetaDatabaseTTLs(monthsToKeepHistoricalFirstSeen
 		return err
 	}

+	err = server.Conn.Exec(ctx, `--sql
+		ALTER TABLE metadatabase.performed_zone_transfers MODIFY TTL performed_at + INTERVAL 90 DAYS`)
+	if err != nil {
+		return err
+	}
+
+	err = server.Conn.Exec(ctx, `--sql
+		ALTER TABLE metadatabase.zone_transfer MODIFY TTL performed_at + INTERVAL 90 DAYS`)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
@@ -12,6 +12,7 @@ require (
 	github.com/hjson/hjson-go/v4 v4.4.0
 	github.com/joho/godotenv v1.5.1
 	github.com/json-iterator/go v1.1.12
+	github.com/miekg/dns v1.1.65
 	github.com/montanaflynn/stats v0.7.1
 	github.com/muesli/reflow v0.3.0
 	github.com/rs/zerolog v1.33.0
@@ -22,8 +23,8 @@ require (
 	github.com/testcontainers/testcontainers-go/modules/compose v0.31.0
 	github.com/urfave/cli/v2 v2.27.2
 	github.com/vbauerster/mpb/v8 v8.7.3
-	golang.org/x/sync v0.7.0
-	golang.org/x/text v0.16.0
+	golang.org/x/sync v0.11.0
+	golang.org/x/text v0.22.0
 	golang.org/x/time v0.5.0
 )

@@ -33,6 +34,8 @@ require (
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
+	golang.org/x/mod v0.23.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
 )

 require (
@@ -225,12 +228,12 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/mock v0.4.0 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
-	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/term v0.29.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
@@ -408,6 +408,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
+github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
 github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
@@ -695,8 +697,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
@@ -706,8 +708,8 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -724,8 +726,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
 golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
@@ -737,8 +739,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -772,21 +774,21 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -798,8 +800,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -0,0 +1,139 @@
+package integration_test
+
+import (
+	"context"
+	"testing"
+
+	"time"
+
+	"github.com/activecm/rita/v5/config"
+	"github.com/activecm/rita/v5/database"
+	"github.com/activecm/rita/v5/zonetransfer"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+type ZoneTransferSuite FilterTestSuite
+
+func TestZoneTransfer(t *testing.T) {
+	suite.Run(t, new(ZoneTransferSuite))
+
+}
+
+// Reset config after each test since these tests load the config from a file
+func (it *ZoneTransferSuite) SetupSuite() {
+	t := it.T()
+	afs := afero.NewOsFs()
+	cfg, err := config.ReadFileConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	cfg.Env.DBConnection = dockerInfo.clickhouseConnection
+	it.cfg = cfg
+
+	db, err := database.ConnectToServer(context.Background(), it.cfg)
+	require.NoError(t, err)
+	require.NoError(t, db.Conn.Exec(db.GetContext(), "TRUNCATE TABLE metadatabase.performed_zone_transfers"))
+
+}
+
+func (it *ZoneTransferSuite) SetupTest() {
+	err := it.cfg.Reset()
+	it.Require().NoError(err)
+}
+
+func (it *ZoneTransferSuite) TearDownSuite() {
+	err := it.cfg.Reset()
+	it.Require().NoError(err)
+}
+
+func (it *ZoneTransferSuite) TestRecordZoneTransferPerformed() {
+	t := it.T()
+
+	it.cfg.ZoneTransfer.Enabled = true
+	it.cfg.ZoneTransfer.DomainName = "bug.corp."
+	it.cfg.ZoneTransfer.NameServer = "dc1.bug.corp:53"
+
+	// connect to database
+	db, err := database.ConnectToServer(context.Background(), it.cfg)
+	require.NoError(t, err)
+
+	tests := []struct {
+		label         string
+		toCreate      zonetransfer.PerformedZoneTransfer
+		changeConfig  bool
+		expectedIndex int
+	}{
+		{
+			label: "First entry should return itself",
+			toCreate: zonetransfer.PerformedZoneTransfer{
+				PerformedAt: time.Unix(1515531081, 0).UTC(),
+				DomainName:  "bug.corp.",
+				NameServer:  "dc1.bug.corp:53",
+				Serial:      100,
+				MBox:        "example@bug.corp",
+			},
+			expectedIndex: 0,
+		},
+
+		{
+			label: "Newer timestamp should return itself",
+			toCreate: zonetransfer.PerformedZoneTransfer{
+				PerformedAt: time.Unix(1615531081, 0).UTC(),
+				DomainName:  "bug.corp.",
+				NameServer:  "dc1.bug.corp:53",
+				Serial:      100,
+				MBox:        "example@bug.corp",
+			},
+			expectedIndex: 1,
+		},
+		{
+			label: "Older timestamp shouldn't return itself",
+			toCreate: zonetransfer.PerformedZoneTransfer{
+				PerformedAt: time.Unix(1415531081, 0).UTC(),
+				DomainName:  "bug.corp.",
+				NameServer:  "dc1.bug.corp:53",
+				Serial:      100,
+				MBox:        "example@bug.corp",
+			},
+			expectedIndex: 1,
+		},
+		{
+			label:        "It should filter by domain name and name server",
+			changeConfig: true,
+			toCreate: zonetransfer.PerformedZoneTransfer{
+				PerformedAt: time.Unix(1415531081, 0).UTC(),
+				DomainName:  "bug2.corp2.",
+				NameServer:  "dc1.bug2.corp2:53",
+				Serial:      100,
+				MBox:        "example@bug.corp",
+			},
+			expectedIndex: 3,
+		},
+	}
+
+	zt, err := zonetransfer.NewZoneTransfer(db, it.cfg)
+	require.NoError(t, err)
+	// there should be no error if no zone transfer was performed yet, but result should be nil
+	latestZT, err := zt.FindLastZoneTransfer()
+	require.NoError(t, err)
+	require.Nil(t, latestZT)
+
+	for _, tc := range tests {
+		// switch to a different domain/name server for this
+		if tc.changeConfig {
+			it.cfg.ZoneTransfer.DomainName = tc.toCreate.DomainName
+			it.cfg.ZoneTransfer.NameServer = tc.toCreate.NameServer
+		}
+		zt, err := zonetransfer.NewZoneTransfer(db, it.cfg)
+		require.NoError(t, err)
+
+		zt.SetTransferInfo(tc.toCreate)
+		require.NoError(t, zt.RecordZoneTransferPerformed(), "recording zone transfer shouldn't fail: %s", tc.label)
+
+		latestZT, err := zt.FindLastZoneTransfer()
+		require.NoError(t, err)
+		require.NotNil(t, latestZT)
+
+		require.Equal(t, tests[tc.expectedIndex].toCreate, *latestZT, "latest zone transfer found should match expected performed zone transfer: %s", tc.label)
+	}
+}
@@ -0,0 +1,295 @@
+package zonetransfer
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/activecm/rita/v5/config"
+	"github.com/activecm/rita/v5/database"
+	"github.com/activecm/rita/v5/logger"
+	"github.com/miekg/dns"
+	"golang.org/x/time/rate"
+)
+
+var ErrDomainNotConfigured = errors.New("domain name or name server has not been configured for zone transfer")
+var ErrZoneTransferNotEnabled = errors.New("zone transfers are not enabled for RITA")
+
+type Record struct {
+	PerformedAt time.Time `ch:"performed_at"`
+	Hostname    string    `ch:"hostname"`
+	IP          net.IP    `ch:"ip"`
+	TTL         uint32    `ch:"ttl"`
+	// domain forest info
+	DomainName string `ch:"domain_name"`
+	NameServer string `ch:"name_server"`
+}
+
+type PerformedZoneTransfer struct {
+	PerformedAt time.Time `ch:"performed_at"`
+	DomainName  string    `ch:"domain_name"`
+	NameServer  string    `ch:"name_server"`
+	Serial      uint32    `ch:"serial_soa"`
+	MBox        string    `ch:"mbox"`
+	IsIXFR      bool      `ch:"is_ixfr"`
+}
+
+type ZoneTransferConnectivityErrors struct {
+	NameServerUnreachableUDPError error `json:"name_server_unreachable_udp"`
+	NameServerUnreachableTCPError error `json:"name_server_unreachable_tcp"`
+	UDPQueryFailedError           error `json:"udp_query_failed"`
+	AXFRFailedError               error `json:"axfr_failed"`
+}
+
+type ZoneTransfer struct {
+	domainName  string
+	nameServer  string
+	latestSOA   dns.SOA
+	db          *database.ServerConn
+	cfg         *config.Config
+	performedAt time.Time
+}
+
+// NewZoneTransfer creates the struct needed for handling zone transfers
+func NewZoneTransfer(db *database.ServerConn, cfg *config.Config) (*ZoneTransfer, error) {
+	if len(cfg.ZoneTransfer.DomainName) == 0 || len(cfg.ZoneTransfer.NameServer) == 0 {
+		return nil, ErrDomainNotConfigured
+	}
+	if db == nil {
+		return nil, database.ErrInvalidDatabaseConnection
+	}
+	if err := db.CreateServerDBTables(); err != nil {
+		return nil, fmt.Errorf("unable to setup system for zone transfers, err: %w", err)
+	}
+	return &ZoneTransfer{
+		domainName:  cfg.ZoneTransfer.DomainName,
+		nameServer:  cfg.ZoneTransfer.NameServer,
+		db:          db,
+		cfg:         cfg,
+		performedAt: time.Now().UTC(),
+	}, nil
+}
+
+// DoZT handles a zone transfer
+func (zt *ZoneTransfer) DoZT(axfr bool) error {
+	t := new(dns.Transfer)
+	m := new(dns.Msg)
+	if axfr {
+		// Set up an AXFR if requested
+		m.SetAxfr(zt.domainName)
+	} else {
+		// Otherwise, set up an IXFR, using the domain name and latest SOA Serial and Mbox values
+		m.SetIxfr(zt.domainName, zt.latestSOA.Serial, zt.domainName, zt.latestSOA.Mbox)
+	}
+
+	// create a channel for the dns "envelope"
+	ch, err := t.In(m, zt.nameServer)
+
+	if err != nil {
+		return err
+	}
+
+	// create a rate limiter to control the rate of writing to the database
+	limiter := rate.NewLimiter(5, 5)
+
+	// create a writer for the zone_transfer table
+	writer := database.NewBulkWriter(zt.db, zt.cfg, 1, "metadatabase.zone_transfer", "zone_transfer", "INSERT INTO metadatabase.zone_transfer", limiter, false)
+	writer.Start(0)
+
+	// since there could be a large volume of records coming in for a large domain, these records should be streamed via the dns envelope
+	// and then written into batches as the results come in
+	for env := range ch {
+		if env.Error != nil {
+			err = env.Error
+			break
+		}
+
+		// Type switch to handle the DNS envelope types we care about (SOA for IXFR info, A and AAAA for mapping)
+		for _, rr := range env.RR {
+			// create record with standard metadata
+			record := Record{PerformedAt: zt.performedAt, DomainName: zt.domainName, NameServer: zt.nameServer}
+
+			switch rec := rr.(type) {
+			case *dns.SOA:
+				// contains the serial and mbox info to store
+				zt.latestSOA = *rec
+			case *dns.A:
+				record.Hostname = strings.TrimSuffix(rec.Header().Name, ".")
+				record.IP = rec.A
+				record.TTL = rec.Header().Ttl
+				writer.WriteChannel <- &record
+			case *dns.AAAA:
+				record.Hostname = strings.TrimSuffix(rec.Header().Name, ".")
+				record.IP = rec.AAAA
+				record.TTL = rec.Header().Ttl
+				writer.WriteChannel <- &record
+			}
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	writer.Close()
+
+	// record that a zone transfer occurred
+	zt.RecordZoneTransferPerformed()
+
+	return nil
+}
+
+// RecordZoneTransferPerformed marks a completed zone transfer in the metadatabase and stores the most recent serial (SOA) found in the dns query
+func (zt *ZoneTransfer) RecordZoneTransferPerformed() error {
+	chCtx := zt.db.QueryParameters(clickhouse.Parameters{
+		"performed_at": fmt.Sprintf("%d", zt.performedAt.UTC().Unix()),
+		"domain_name":  zt.domainName,
+		"name_server":  zt.nameServer,
+		"serial_soa":   strconv.FormatUint(uint64(zt.latestSOA.Serial), 10),
+		"mbox":         zt.latestSOA.Mbox,
+	})
+	if err := zt.db.Conn.Exec(chCtx, `
+		INSERT INTO metadatabase.performed_zone_transfers (performed_at, domain_name, name_server, serial_soa, mbox) 
+		VALUES ( fromUnixTimestamp({performed_at:Int64}), {domain_name:String}, {name_server:String}, {serial_soa:UInt32}, {mbox:String} )
+	`); err != nil {
+		return err
+	}
+	return nil
+}
+
+// FindLastZoneTransfer finds the last zone transfer that was performed for this zt's domain and name server
+func (zt *ZoneTransfer) FindLastZoneTransfer() (*PerformedZoneTransfer, error) {
+	chCtx := zt.db.QueryParameters(clickhouse.Parameters{
+		"domain_name": zt.domainName,
+		"name_server": zt.nameServer,
+	})
+
+	var lastZoneTransfer PerformedZoneTransfer
+	if err := zt.db.Conn.QueryRow(chCtx, `
+		SELECT max(performed_at) AS performed_at, domain_name, name_server, serial_soa, mbox, is_ixfr
+		FROM metadatabase.performed_zone_transfers
+		WHERE domain_name = {domain_name:String} AND name_server = {name_server:String}
+		GROUP BY domain_name, name_server, serial_soa, mbox, is_ixfr
+		ORDER BY performed_at DESC
+		LIMIT 1
+`).ScanStruct(&lastZoneTransfer); err != nil {
+		// return nil PerformedZoneTransfer with no error if no zone transfer was found (ignore error)
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return &lastZoneTransfer, nil
+}
+
+// PerformZoneTransfer performs either an AXFR or IXFR zone transfer
+func (zt *ZoneTransfer) PerformZoneTransfer() error {
+	// skip if zone transfers are not enabled
+	if !zt.cfg.ZoneTransfer.Enabled {
+		return ErrZoneTransferNotEnabled
+	}
+
+	/*  ==== ZONE TRANSFERS ====
+	AXFR zone transfers do a transfer of the entire domain,
+
+	IXFR zone transfers do incremental zone transfers. They update the portions of the domain that have updated since the
+	Serial (SOA) that was provided with the transfer request. For example, on Monday a zone transfer was performed and we noted that
+	the current Serial is 100. On Tuesday, we perform an IXFR zone transfer and pass the Serial of 100 with the request. We will only
+	get back the results that have changed since Monday instead of the entire domain forest.
+
+	Performed zone transfers are stored in metadatabase.performed_zone_transfers, which tracks when a zone transfer was performed on a domain/name server and
+	what the latest Serial was at the time. If no results match in performed_zone_transfers, then we do an AXFR transfer.
+	*/
+
+	// Try to find last zone transfer that was performed for this domain & name server
+	lastZoneTransfer, err := zt.FindLastZoneTransfer()
+	if err != nil {
+		return fmt.Errorf("unable to find last performed zone transfer on domain '%s' via name server '%s': %w", zt.domainName, zt.nameServer, err)
+	}
+
+	zlog := logger.GetLogger()
+
+	transferType := "AXFR"
+	// There was a match in performed_zone_transfers, do an IXFR transfer
+	if lastZoneTransfer != nil {
+		// Do an IXFR
+		transferType = "IXFR"
+		if err := zt.DoZT(false); err != nil {
+			return fmt.Errorf("unable to perform IXFR zone transfer on domain '%s' via name server '%s': %w", zt.domainName, zt.nameServer, err)
+		}
+	} else {
+		// No match, do an AXFR
+		if err := zt.DoZT(true); err != nil {
+			return fmt.Errorf("unable to perform AXFR zone transfer on domain '%s' via name server '%s': %w", zt.domainName, zt.nameServer, err)
+		}
+	}
+	zlog.Info().Str("domain", zt.domainName).Str("name_server", zt.nameServer).Uint32("lastest_soa", zt.latestSOA.Serial).Msg(fmt.Sprintf("Successfully performed %s zone transfer", transferType))
+
+	return nil
+}
+
+func (zt *ZoneTransfer) TestNetConnectivity(protocol string) error {
+	conn, err := net.DialTimeout(protocol, zt.cfg.ZoneTransfer.NameServer, 5*time.Second)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}
+
+func (zt *ZoneTransfer) TestConnectivity() ZoneTransferConnectivityErrors {
+
+	var result ZoneTransferConnectivityErrors
+
+	zlog := logger.GetLogger()
+
+	if err := zt.TestNetConnectivity("udp"); err != nil {
+		result.NameServerUnreachableUDPError = err
+		zlog.Error().Err(err).Str("name_server", zt.nameServer).Msg("name server unreachable via UDP")
+	}
+
+	if err := zt.TestNetConnectivity("tcp"); err != nil {
+		result.NameServerUnreachableTCPError = err
+		zlog.Error().Err(err).Str("name_server", zt.nameServer).Msg("name server unreachable via TCP")
+	}
+
+	client := new(dns.Client)
+	client.Net = "udp"
+	msg := new(dns.Msg)
+	msg.SetQuestion(zt.domainName, dns.TypeSOA)
+
+	_, _, err := client.Exchange(msg, zt.nameServer)
+	if err != nil {
+		result.UDPQueryFailedError = err
+		zlog.Error().Err(err).Str("name_server", zt.nameServer).Str("domain_name", zt.domainName).Msg("failed to perform SOA query over UDP")
+	}
+
+	t := new(dns.Transfer)
+	m := new(dns.Msg)
+	m.SetAxfr(zt.domainName)
+
+	_, err = t.In(m, zt.nameServer)
+	if err != nil {
+		result.AXFRFailedError = err
+		zlog.Error().Err(err).Str("name_server", zt.nameServer).Str("domain_name", zt.domainName).Msg("failed to perform AXFR zone transfer")
+	}
+
+	zlog.Info().Str("name_server", zt.nameServer).Str("domain_name", zt.domainName).Msg("connectivity check to domain for zone transfers was successful")
+
+	return result
+}
+
+// for testing
+func (zt *ZoneTransfer) SetTransferInfo(transferInfo PerformedZoneTransfer) {
+	zt.performedAt = transferInfo.PerformedAt
+	zt.domainName = transferInfo.DomainName
+	zt.nameServer = transferInfo.NameServer
+	zt.latestSOA.Serial = transferInfo.Serial
+	zt.latestSOA.Mbox = transferInfo.MBox
+}