From 0b279a2fdafa95c232a7135e7a4377553b0a7835 Mon Sep 17 00:00:00 2001 From: Rob Sears Date: Wed, 23 Oct 2019 16:02:54 -0500 Subject: [PATCH] Add the golang example code --- golang | 1 - golang/.gitignore | 4 + golang/README.md | 28 +++++ golang/go-bonsai.go | 20 ++++ golang/lib/controllers/http_handlers.go | 73 +++++++++++++ golang/lib/elasticsearch/elasticsearch.go | 97 ++++++++++++++++++ golang/lib/errors/errors.go | 23 +++++ golang/lib/views/templates/container.tmpl | 20 ++++ golang/lib/views/templates/footer.tmpl | 4 + golang/lib/views/templates/header.tmpl | 13 +++ golang/lib/views/templates/index.tmpl | 99 ++++++++++++++++++ golang/lib/views/templates/nav.tmpl | 28 +++++ golang/lib/views/templates/root.tmpl | 42 ++++++++ golang/lib/views/templates/sidebar.tmpl | 15 +++ golang/lib/views/templates/view.tmpl | 13 +++ golang/lib/views/views.go | 119 ++++++++++++++++++++++ 16 files changed, 598 insertions(+), 1 deletion(-) delete mode 160000 golang create mode 100644 golang/.gitignore create mode 100644 golang/README.md create mode 100644 golang/go-bonsai.go create mode 100644 golang/lib/controllers/http_handlers.go create mode 100644 golang/lib/elasticsearch/elasticsearch.go create mode 100644 golang/lib/errors/errors.go create mode 100644 golang/lib/views/templates/container.tmpl create mode 100644 golang/lib/views/templates/footer.tmpl create mode 100644 golang/lib/views/templates/header.tmpl create mode 100644 golang/lib/views/templates/index.tmpl create mode 100644 golang/lib/views/templates/nav.tmpl create mode 100644 golang/lib/views/templates/root.tmpl create mode 100644 golang/lib/views/templates/sidebar.tmpl create mode 100644 golang/lib/views/templates/view.tmpl create mode 100644 golang/lib/views/views.go diff --git a/golang b/golang deleted file mode 160000 index c031432..0000000 --- a/golang +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c03143267eac8fbaad182f002883177fbf0c5549 diff --git a/golang/.gitignore b/golang/.gitignore new file mode 100644 index 0000000..c689e65 --- /dev/null +++ b/golang/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +.idea +go-bonsai +*.iml diff --git a/golang/README.md b/golang/README.md new file mode 100644 index 0000000..17dd8a2 --- /dev/null +++ b/golang/README.md @@ -0,0 +1,28 @@ +## Go Bonsai! A lightweight Elasticsearch cluster exploration tool written in golang. + +To get started, clone the repo: + +``` +git clone git@github.com:omc/dogfood-go.git +``` + +Make sure to export your Bonsai cluster URL to the dev environment: + +``` +export BONSAI_URL="https://user:pass@some-cool-cluster-12345678.us-east-1.bonsai.io" +``` + +Build the tool: + +``` +go build bonsai-example-go.go +``` + +Run it: + +``` +$ chmod +x go-bonsai +$ ./go-bonsai +``` + +Check it out by opening up a browser and navigating to http://localhost:8080 diff --git a/golang/go-bonsai.go b/golang/go-bonsai.go new file mode 100644 index 0000000..b63968b --- /dev/null +++ b/golang/go-bonsai.go @@ -0,0 +1,20 @@ +package main + +import( + "./lib/controllers" + "net/http" +) + +func main() { + http.HandleFunc("/", rootHandler) + http.HandleFunc("/index/", indexHandler) + http.ListenAndServe(":8080", nil) +} + +func indexHandler(w http.ResponseWriter, r *http.Request) { + controllers.IndexBuilder(w, r) +} + +func rootHandler(w http.ResponseWriter, r *http.Request) { + controllers.RootBuilder(w, r) +} diff --git a/golang/lib/controllers/http_handlers.go b/golang/lib/controllers/http_handlers.go new file mode 100644 index 0000000..438afe3 --- /dev/null +++ b/golang/lib/controllers/http_handlers.go @@ -0,0 +1,73 @@ +package controllers + +import( + "../elasticsearch" + "../views" + "../errors" + "gopkg.in/olivere/elastic.v2" + "net/http" + "sort" +) + +func IndexBuilder(w http.ResponseWriter, r *http.Request) { + es,index := elasticsearch.ConfirmIndex(w, r) + if index != "" { + + // Get a list of sorted index names + indices, _ := es.IndexNames() + sort.Strings(indices) + + // Query for some docs + // Use default settings for FROM and SIZE + query := elastic.NewQueryStringQuery("*") + docs,_ := es.Search().From(0).Size(10).Index(index).Query(query).Pretty(true).Do() + + // Get a mapping for the index + mappings := elasticsearch.GetIndexMappings(es, index) + var mappingTypes []string + typeCounts := make(map[string]int64) + if docs.TotalHits() > 0 { + // No docs? Then there ain't no mappin' to get + types := mappings[index].(map[string]interface{})["mappings"].(map[string]interface{}) + mappingTypes = elasticsearch.GetTypesFromMapping(types) + sort.Strings(mappingTypes) + + for _,i:=range mappingTypes { + typeDocs,_ := es.Search().From(0).Size(10).Index(index).Type(i).Query(query).Pretty(true).Do() + typeCounts[i] = typeDocs.TotalHits() + } + } + + //Create a view with the data and render it for the user + p := &views.View{ + Title: "Overview of " + index, + Hits: docs.TotalHits(), + Docs: docs.Hits.Hits, + Name: index, + IndexCount: len(indices), + Indices: indices, + Mappings: mappings, + MappingTypes: mappingTypes, + TypeCounts: typeCounts} + views.Render(p, w, r, "index") + } +} + +func RootBuilder(w http.ResponseWriter, r *http.Request) { + es := elasticsearch.GetClient() + indices,err := es.IndexNames() + + if err != nil { + errors.ExitOnError(err, errors.INDEX_NAME_ERROR) + } + sort.Strings(indices) + health := elasticsearch.GetClusterHealth(es) + stats := elasticsearch.GetClusterStats(es) + p := &views.View{ + Title: "Cluster Overview", + Health: health, + Stats: stats, + IndexCount: len(indices), + Indices: indices} + views.Render(p, w, r, "root") +} diff --git a/golang/lib/elasticsearch/elasticsearch.go b/golang/lib/elasticsearch/elasticsearch.go new file mode 100644 index 0000000..285b1b7 --- /dev/null +++ b/golang/lib/elasticsearch/elasticsearch.go @@ -0,0 +1,97 @@ +package elasticsearch + +import( + "gopkg.in/olivere/elastic.v2" + "os" + "regexp" + "strings" + "net/http" + "fmt" + "../errors" + "../views" +) + +// Instantiate an Elasticsearch client +func GetClient() *elastic.Client { + user,pass,host := parseBonsaiURL() + client, err := elastic.NewSimpleClient( + elastic.SetURL(host), + elastic.SetMaxRetries(10), + elastic.SetBasicAuth(user,pass), + elastic.SetSniff(false)) + if err != nil { + errors.ExitOnError(err, errors.CLIENT_FAILURE) + } + return client +} + +// +func GetClusterHealth(es *elastic.Client) *elastic.ClusterHealthResponse { + health,_ := es.ClusterHealth(). + Do() + return health +} + +// +func GetClusterStats(es *elastic.Client) *elastic.IndicesStatsResponse { + stats,_ := es.IndexStats(). + Index("_all"). + Human(true). + Pretty(true). + Do() + return stats +} + +func SearchDocuments(es *elastic.Client, index string) *elastic.SearchResult { + docs,_ := es.Search(). + Index(index). + Do() + return docs +} + +func GetIndexMappings(es *elastic.Client, index string) map[string]interface{} { + mappings,err := es.GetMapping().Index(index).Do() + if err != nil { + errors.ExitOnError(err, errors.INDEX_MAPPING_ERROR) + } + return mappings +} + +// Make sure that an index exists before we do anything with it. +// If it exists, return the client for reuse +func ConfirmIndex(w http.ResponseWriter, r *http.Request) (*elastic.Client, string) { + rex, err := regexp.Compile("/index/(.*?)") + index := rex.ReplaceAllString(r.URL.Path, "$1") + if err != nil { + errors.ExitOnError(err, errors.REGEX_COMPILE) + } + es := GetClient() + indexCheck, err := es.IndexExists(index).Do() + if indexCheck == false { + indices,_ := es.IndexNames() + fmt.Printf("Aw, brah, I couldn't find an index called '%s'!\n", index) + p := &views.View{IndexCount: len(indices), Indices: indices, ErrorMessage: "That index doesn't exist!"} + views.Render(p, w, r, "root") + return es,"" + } else if err != nil { + errors.ExitOnError(err, errors.INDEX_EXISTS_ERROR) + } + return es, index +} + +func parseBonsaiURL() (string, string, string){ + url := os.Getenv("BONSAI_URL") + rex, _ := regexp.Compile(".*?://([a-z0-9]{1,}):([a-z0-9]{1,})@.*$") + user := rex.ReplaceAllString(url, "$1") + pass := rex.ReplaceAllString(url, "$2") + host := strings.Replace(url, user+":"+pass+"@", "", -1) + return user,pass,host +} + +func GetTypesFromMapping(m map[string]interface{}) (types []string) { + for k := range m { + types = append(types, k) + } + return types +} + diff --git a/golang/lib/errors/errors.go b/golang/lib/errors/errors.go new file mode 100644 index 0000000..9c376fc --- /dev/null +++ b/golang/lib/errors/errors.go @@ -0,0 +1,23 @@ +package errors + +import ( + "fmt" + "os" +) + +const ( + CLIENT_FAILURE = "Could not instantiate a client to the Elasticsearch cluster" + JSON_DECODE_FAILURE = "Could not decode JSON" + INDEX_EXISTS_ERROR = "The Elasticsearch client failed to determine whether the index exists" + INDEX_NAME_ERROR = "An error occurred while retrieving the cluster's index names" + REGEX_COMPILE = "Could not compile the regular expression" + TEMPLATE_PARSE_FAILURE = "The template engine failed to parse the requested view" + TEMPLATE_EXECUTE_FAILURE = "The template engine failed to execute the requested view" + INDEX_MAPPING_ERROR = "The requested index mapping could not be found" +) + +// Quit the program if there is an error +func ExitOnError(err error, errMsg string) { + fmt.Printf("The program exited with error category '%s'. Specific failure: %s\n", errMsg, err) + os.Exit(1) +} \ No newline at end of file diff --git a/golang/lib/views/templates/container.tmpl b/golang/lib/views/templates/container.tmpl new file mode 100644 index 0000000..78d6792 --- /dev/null +++ b/golang/lib/views/templates/container.tmpl @@ -0,0 +1,20 @@ +{{define "container"}} +
+
+
+ {{ template "sidebar" . }} +
+
+ + {{if .ErrorMessage}} +
+ {{ .ErrorMessage }} +
+ {{end}} + + {{ template "content" . }} + +
+
+
+{{end}} diff --git a/golang/lib/views/templates/footer.tmpl b/golang/lib/views/templates/footer.tmpl new file mode 100644 index 0000000..33765bb --- /dev/null +++ b/golang/lib/views/templates/footer.tmpl @@ -0,0 +1,4 @@ +{{define "footer"}} + + +{{end}} diff --git a/golang/lib/views/templates/header.tmpl b/golang/lib/views/templates/header.tmpl new file mode 100644 index 0000000..f09e3ff --- /dev/null +++ b/golang/lib/views/templates/header.tmpl @@ -0,0 +1,13 @@ +{{define "header"}} + + + + + + + + + + + {{ .Title }} +{{end}} diff --git a/golang/lib/views/templates/index.tmpl b/golang/lib/views/templates/index.tmpl new file mode 100644 index 0000000..607d427 --- /dev/null +++ b/golang/lib/views/templates/index.tmpl @@ -0,0 +1,99 @@ +{{ define "content"}} +

Overview of {{ .Name }}

+ + {{if gt .Hits 0}} + +
+ + + + + +
+
+ +
+ +
+
+
+ + + + + + + {{range $type := .MappingTypes}} + + + + + {{ end }} + +
TypeFields
{{ $type }} +
    + {{ range $field := mappingFields $.Mappings $.Name $type}} +
  • {{ $field }}
    • + {{ end }} +
+
+
+
+
+
+ + + + + + + + {{range $doc := .Docs}} + + + + + + {{ end }} + +
IDTypeSource
{{ $doc.Id }}{{ $doc.Type }} + {{ jsonDecode $doc }} +
+
+
+
+ +
+ {{end}} + +{{ end }} \ No newline at end of file diff --git a/golang/lib/views/templates/nav.tmpl b/golang/lib/views/templates/nav.tmpl new file mode 100644 index 0000000..e34af79 --- /dev/null +++ b/golang/lib/views/templates/nav.tmpl @@ -0,0 +1,28 @@ +{{define "nav"}} + +{{end}} diff --git a/golang/lib/views/templates/root.tmpl b/golang/lib/views/templates/root.tmpl new file mode 100644 index 0000000..ccd136b --- /dev/null +++ b/golang/lib/views/templates/root.tmpl @@ -0,0 +1,42 @@ +{{ define "content" }} +

Overview

+
+
+
+
+ +
+

Cluster state:

+
    +
  • Cluster name: {{ .Health.ClusterName }}
    • +
  • Cluster status: {{ .Health.Status }}
    • +
  • Number of nodes: {{ .Health.NumberOfNodes }}
    • +
  • Number of data nodes: {{ .Health.NumberOfDataNodes }}
    • +
+
+
+
+ +
+

Shards:

+
    +
  • Active: {{ .Health.ActiveShards }}
    • +
  • Relocating: {{ .Health.RelocatingShards }}
    • +
  • Initializing: {{ .Health.InitializingShards }}
    • +
  • Unassigned: {{ .Health.UnassignedShards }}
    • +
+
+
+
+ +
+

Stats:

+
    +
  • Total documents (primary): {{ .Stats.All.Primaries.Docs.Count }}
    • +
  • Primary Disk Store: {{ .Stats.All.Primaries.Store.Size }}
    • +
  • Total Disk Usage: {{ .Stats.All.Total.Store.Size }}
    • +
+
+
+
+{{ end }} \ No newline at end of file diff --git a/golang/lib/views/templates/sidebar.tmpl b/golang/lib/views/templates/sidebar.tmpl new file mode 100644 index 0000000..0c5a233 --- /dev/null +++ b/golang/lib/views/templates/sidebar.tmpl @@ -0,0 +1,15 @@ +{{define "sidebar"}} + + +{{end}} \ No newline at end of file diff --git a/golang/lib/views/templates/view.tmpl b/golang/lib/views/templates/view.tmpl new file mode 100644 index 0000000..11fa2ed --- /dev/null +++ b/golang/lib/views/templates/view.tmpl @@ -0,0 +1,13 @@ +{{ define "view" }} + + + + {{ template "header" . }} + + + {{ template "nav" . }} + {{ template "container" . }} + {{ template "footer" . }} + + +{{ end }} \ No newline at end of file diff --git a/golang/lib/views/views.go b/golang/lib/views/views.go new file mode 100644 index 0000000..de7cfad --- /dev/null +++ b/golang/lib/views/views.go @@ -0,0 +1,119 @@ +package views + +import( + "fmt" + "html/template" + "net/http" + "../errors" + "gopkg.in/olivere/elastic.v2" + "os" + "path/filepath" + "sort" +"strings" +) + +// Define a struct for views +type View struct { + + /* Page attributes */ + Title string + + /* Flash messages */ + SuccessMessage string + InfoWarning string + WarningMessage string + ErrorMessage string + + /* Data */ + Hits int64 + IndexCount int + Indices []string + Name string + Health *elastic.ClusterHealthResponse + Stats *elastic.IndicesStatsResponse + Docs []*elastic.SearchHit + Mappings map[string]interface {} + MappingTypes []string + TypeCounts map[string]int64 +} + +// Render the view: +func Render(p *View, w http.ResponseWriter, r *http.Request, name string) { + + funcMap := template.FuncMap { + "jsonDecode": jsonDecode, + "mappingFields": mappingFields, + "d3data": d3data, + } + + cwd, _ := os.Getwd() + templatePath := filepath.Join( cwd, "./lib/views/templates/") + fmt.Printf("Rendering view '%s' for %s. Using template %s\n", name , r.URL.Path, templatePath+"/" + name + ".tmpl") + t, err := template.New("main").Funcs(funcMap).ParseFiles( + templatePath + "/header.tmpl", + templatePath + "/nav.tmpl", + templatePath + "/container.tmpl", + templatePath + "/sidebar.tmpl", + templatePath + "/" + name + ".tmpl", + templatePath + "/footer.tmpl", + templatePath + "/view.tmpl") + if err != nil { + errors.ExitOnError(err, errors.TEMPLATE_PARSE_FAILURE) + } + err = t.ExecuteTemplate(w, "view", p) + if err != nil { + errors.ExitOnError(err, errors.TEMPLATE_EXECUTE_FAILURE) + } + +} + +func jsonDecode(hit elastic.SearchHit) string { + /* + Go uses these six types for all values parsed into interfaces: + + bool, for JSON booleans + float64, for JSON numbers + string, for JSON strings + []interface{}, for JSON arrays + map[string]interface{}, for JSON objects + nil for JSON null + */ + + return string(*hit.Source) +} + +func mappingFields(mapping map[string]interface{}, index string, indexType string) []string { + var fields []string + m := mapping[index]. + (map[string]interface{})["mappings"]. + (map[string]interface{})[indexType]. + (map[string]interface{})["properties"]. + (map[string]interface{}) + for v,q := range m { + fields = append(fields, formatField(v, formatFieldType(q.(map[string]interface{})))) + } + sort.Strings(fields) + return fields +} + +func d3data(typeCounts map[string]int64) template.JS { + d3data := "{" + for i,v := range typeCounts { + d3data = d3data + fmt.Sprintf("'%s': %d,", i, v) + } + d3data = d3data + "}" + d3data = strings.Replace(d3data, ",}", "}", -1) + return template.JS(d3data) +} + +func formatFieldType(fieldType map[string]interface{}) string { + dataType := fmt.Sprintf("%v",fieldType["type"]) + if dataType == "" { + return "json" + } + return dataType +} + +func formatField(name string, fieldType string) string { + return name + ": " + fieldType +} \ No newline at end of file