diff --git a/cmd/cmd_hostkeydb.go b/cmd/cmd_hostkeydb.go new file mode 100644 index 0000000..d0cedb0 --- /dev/null +++ b/cmd/cmd_hostkeydb.go @@ -0,0 +1,179 @@ +package cmd + +import ( + "bufio" + "database/sql" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" + "time" + + "github.com/runZeroInc/sshamble/auth" + "github.com/runZeroInc/sshamble/badkeys" + "github.com/runZeroInc/sshamble/crypto/ssh" + "github.com/spf13/cobra" + _ "modernc.org/sqlite" +) + +// hostkeydbCmd processes a scan output file and buckets results +var hostkeydbCmd = &cobra.Command{ + Use: "hostkeydb -o results-directory scan.json ...", + Short: "Analyzes a scan JSON output file and produces a hostkey DB", + Long: "Analyzes a scan JSON output file and produces a hostkey DB", + Run: runHostkeyDB, +} + +func init() { + hostkeydbCmd.Flags().StringVarP(&gOutput, "output", "o", "", "The output database filename") + hostkeydbCmd.Flags().StringVarP(&gLogfile, "log", "l", "-", "The file to write logs to (default is stderr)") + hostkeydbCmd.Flags().StringVarP(&gLogLevel, "log-level", "L", "debug", "The log level to write (trace,debug,info,warn,error)") +} + +const DBBatchSize = 100000 + +func runHostkeyDB(cmd *cobra.Command, args []string) { + conf := &ScanConfig{} + configureLogging(conf) + + var tx *sql.Tx + var stmt *sql.Stmt + var err error + + if gOutput == "" || gOutput == "-" || gOutput == "stdout" { + conf.Logger.Fatalf("no output directory supplied (-o)") + } + + if len(args) < 1 { + conf.Logger.Fatalf("no input files specified") + } + + db, err := sql.Open("sqlite", gOutput) + if err != nil { + conf.Logger.Fatalf("failed to open db: %v", err) + } + + start := time.Now() + + if _, err = db.Exec(` + pragma synchronous = OFF; + pragma journal_mode = OFF; + drop table if exists hostkeys; + create table hostkeys(hash blob, ts integer, host blob, port integer); + `); err != nil { + conf.Logger.Fatalf("failed to create db schema: %v", err) + } + + openTx := func() { + if tx != nil { + conf.Logger.Fatalf("db tx already open") + } + tx, err = db.Begin() + if err != nil { + conf.Logger.Fatalf("failed to create db tx: %v", err) + } + stmt, err = tx.Prepare("insert into hostkeys(hash, ts, host, port) values (?, ?, ?, ?)") + if err != nil { + conf.Logger.Fatalf("db prepare: %v", err) + } + } + + closeTx := func() { + if tx != nil { + if err = stmt.Close(); err != nil { + conf.Logger.Fatalf("db close: %v", err) + } + + if err = tx.Commit(); err != nil { + conf.Logger.Fatalf("commit: %v", err) + } + tx = nil + } + } + + openTx() + + hcache := make(map[string]struct{}) + + for _, inp := range args { + var readFD io.Reader = os.Stdin + if inp != "-" && inp != "stdin" { + fd, err := os.Open(inp) + if err != nil { + conf.Logger.Fatalf("failed to read input file %s: %v", inp, err) + } + defer fd.Close() + readFD = fd + } + scan := bufio.NewScanner(readFD) + buff := make([]byte, MaxJSONLine) + scan.Buffer(buff, MaxJSONLine) + + conf.Logger.Infof("analyzing results from %s...", inp) + cnt := 0 + for scan.Scan() { + line := strings.TrimSpace(scan.Text()) + res := auth.AuthResult{} + if err := json.Unmarshal([]byte(line), &res); err != nil { + conf.Logger.Errorf("%s: failed to parse line: %s (%v)", inp, line, err) + } + + endp := res.Host + strconv.Itoa(res.Port) + for _, hkv := range res.HostKeys { + raw, err := base64.StdEncoding.DecodeString(hkv) + if err != nil { + continue + } + pk, err := ssh.ParsePublicKey(raw) + if err != nil { + continue + } + hpre, err := badkeys.PrefixFromPublicKey(pk) + if err != nil { + continue + } + + if _, found := hcache[endp+"/"+string(hpre)]; found { + continue + } + + if _, err = stmt.Exec(hpre, res.TS, []byte(net.ParseIP(res.Host)), res.Port); err != nil { + conf.Logger.Fatalf("db insert: %v", err) + } + + cnt++ + if cnt%DBBatchSize == 0 { + conf.Logger.Infof("writing batch at %d results", cnt) + closeTx() + openTx() + } + + hcache[endp+"/"+string(hpre)] = struct{}{} + } + } + } + closeTx() + + fmt.Println("creating the index...") + + if _, err := db.Exec(`create index hostkeys_hash_idx on hostkeys (hash)`); err != nil { + conf.Logger.Fatalf("db index: %v", err) + } + + conf.Logger.Printf("done in %v", time.Since(start)) + + if err = db.Close(); err != nil { + conf.Logger.Fatalf("db close: %v", err) + } + + fi, err := os.Stat(gOutput) + if err != nil { + conf.Logger.Fatalf("db stat: %v", err) + } + + conf.Logger.Printf("%s db size: %v", gOutput, fi.Size()) +} diff --git a/cmd/cmd_root.go b/cmd/cmd_root.go index 127e2c0..d631770 100644 --- a/cmd/cmd_root.go +++ b/cmd/cmd_root.go @@ -74,6 +74,7 @@ func init() { rootCmd.AddCommand(scanCmd) rootCmd.AddCommand(analyzeCmd) rootCmd.AddCommand(badkeysCmd) + rootCmd.AddCommand(hostkeydbCmd) rootCmd.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} } diff --git a/go.mod b/go.mod index 09521c4..694dce6 100644 --- a/go.mod +++ b/go.mod @@ -14,15 +14,22 @@ require ( golang.org/x/sys v0.24.0 golang.org/x/term v0.23.0 gonum.org/v1/gonum v0.15.1 + modernc.org/sqlite v1.32.0 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -35,4 +42,10 @@ require ( golang.org/x/text v0.17.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 6fb74f7..f6f530a 100644 --- a/go.sum +++ b/go.sum @@ -3,12 +3,19 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -21,10 +28,14 @@ github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvz github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3 h1:2YMbJ6WbdQI9K73chxh9OWMDsZ2PNjAIRGTonp3T0l0= github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3/go.mod h1:LQkXsHRSPIEklPCq8OMQAzYNS2NGtYStdNE/ej1oJU8= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= @@ -32,6 +43,8 @@ github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xl github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -76,6 +89,7 @@ golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeId golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= @@ -94,3 +108,17 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= +modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=