shodan-gzip.go

package main

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/buger/jsonparser"
	"github.com/vulncheck-oss/go-exploit/db"
)

// decompress the gzip into a tmp directory: .tmp/shodan.json.
func decompressShodan(shodanGZIP string) bool {
	tmpDir := ".tmp"
	err := os.Mkdir(tmpDir, 0o755)
	if err != nil && !os.IsExist(err) {
		log.Printf("Failed to create directory: %v\n", err)

		return false
	}

	inFile, err := os.Open(shodanGZIP)
	if err != nil {
		log.Printf("Failed to open input file: %v\n", err)

		return false
	}
	defer inFile.Close()

	gzipReader, err := gzip.NewReader(inFile)
	if err != nil {
		log.Printf("Failed to create gzip reader: %v\n", err)

		return false
	}
	defer gzipReader.Close()

	outputFile := tmpDir + "/shodan.json"
	outFile, err := os.Create(outputFile)
	if err != nil {
		log.Printf("Failed to create output file: %v\n", err)

		return false
	}
	defer outFile.Close()

	_, err = io.Copy(outFile, gzipReader)
	if err != nil {
		log.Printf("Failed to copy data to output file: %v\n", err)

		return false
	}

	return true
}

// we need to extract:
// 1. ip_str
// 2. port
// 3. data (http headers)
// 4. http.html
//
// return headers+html, ip, port, ok.
func parseShodanJSON(line string) (string, string, int, bool) {
	// do html first so we can fail early
	html, err := jsonparser.GetString([]byte(line), "http", "html")
	if err != nil {
		return "", "", 0, false
	}

	headers, err := jsonparser.GetString([]byte(line), "data")
	if err != nil {
		return "", "", 0, false
	}
	// Shodan fixes up the payload so remove transfer encoding: chunked, otherwise
	// go deserialization fails
	headers = strings.Replace(headers, "Transfer-Encoding: chunked\r\n", "", 1)

	ipStr, err := jsonparser.GetString([]byte(line), "ip_str")
	if err != nil {
		return "", "", 0, false
	}

	port, err := jsonparser.GetInt([]byte(line), "port")
	if err != nil {
		return "", "", 0, false
	}

	return headers + html, ipStr, int(port), true
}

func readShodanJSON(jsonFile string) bool {
	file, err := os.Open(jsonFile)
	if err != nil {
		fmt.Printf("Failed to open file: %v\n", err)

		return false
	}
	defer file.Close()

	// Create a new buffered reader
	reader := bufio.NewReader(file)

	// Read the file line by line
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if err.Error() == "EOF" {
				// End of file reached
				break
			}
			fmt.Printf("Error reading line: %v\n", err)

			return false
		}

		httpData, ip, port, ok := parseShodanJSON(line)
		if !ok {
			continue
		}
		db.CacheHTTPResponse(ip, port, "/", []byte(httpData))
	}

	return true
}

func DoShodanGZIP(inputFile string) bool {
	defer os.RemoveAll(".tmp/")

	log.Println("Decompressing the Shodan GZIP...")
	if !decompressShodan(inputFile) {
		return false
	}
	log.Println("Decompressed file written to .tmp/shodan.json")

	log.Println("Generating database entries...")

	return readShodanJSON(".tmp/shodan.json")
}