diff --git a/Dockerfile b/Dockerfile index 3f5d0ec4..da907024 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,16 +16,12 @@ ENV GO111MODULE=on USER root RUN go get -d -v RUN make validate-schema -RUN make publish-search-index-dry-run RUN make parse-services RUN make generate-search-index RUN CGO_ENABLED=1 go build -o /go/bin/chrome-service-backend # Build the migration binary. RUN CGO_ENABLED=1 go build -o /go/bin/chrome-migrate cmd/migrate/migrate.go -# Build the search index binary. -RUN CGO_ENABLED=1 go build -o /go/bin/chrome-search-index cmd/search/publishSearchIndex.go - FROM registry.redhat.io/ubi8-minimal:latest # Setup permissions to allow RDSCA to be written from clowder to container diff --git a/Makefile b/Makefile index a29e42f0..63f8de57 100644 --- a/Makefile +++ b/Makefile @@ -38,15 +38,7 @@ clean: validate-schema: go run cmd/validate/* - -publish-search-index: - go run cmd/search/* - -publish-search-index-dry-run: export SEARCH_INDEX_DRY_RUN = true - -publish-search-index-dry-run: - go run cmd/search/* - + generate-search-index: export SEARCH_INDEX_WRITE = true generate-search-index: diff --git a/cmd/search/publishSearchIndex.go b/cmd/search/publishSearchIndex.go deleted file mode 100644 index 50f74c7f..00000000 --- a/cmd/search/publishSearchIndex.go +++ /dev/null @@ -1,669 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/joho/godotenv" -) - -type SearchEnv string -type Release string - -const ( - Prod SearchEnv = "prod" - Stage SearchEnv = "stage" - Stale Release = "stable" - Beta Release = "beta" - ssoPathname string = "/auth/realms/redhat-external/protocol/openid-connect/token" - hydraPathname string = "/hydra/rest/search/console/index" -) - -func (se SearchEnv) IsValidEnv() error { - switch se { - case Prod, Stage: - return nil - } - - return fmt.Errorf("invalid environment. Expected one of %s, %s, got %s", Prod, Stage, se) -} - -type EnvMap map[SearchEnv]string - -type TokenResponse struct { - AccessToken string `json:"access_token"` -} - -type ModuleIndexEntry struct { - Icon string `json:"icon,omitempty"` - Title []string `json:"title"` - Bundle []string `json:"bundle"` - BundleTitle []string `json:"bundleTitle"` - AltTitle []string `json:"alt_title,omitempty"` - Id string `json:"id"` - Uri string `json:"uri"` - SolrCommand string `json:"solrCommand"` - ContentType string `json:"contentType"` - ViewUri string `json:"view_uri"` - RelativeUri string `json:"relative_uri"` - PocDescriptionT string `json:"poc_description_t"` -} - -type LinkEntry struct { - Id string `json:"id"` - Title string `json:"title"` - Href string `json:"href"` - Description string `json:"description"` - AltTitle []string `json:"alt_title,omitempty"` -} - -func findFirstValidChildLink(routes []interface{}) LinkEntry { - result := LinkEntry{} - for _, r := range routes { - route, ok := r.(map[string]interface{}) - nestedRoutes, nestedOk := route["routes"].([]interface{}) - href, hrefOk := route["href"].(string) - if hrefOk { - result.Href = href - } else if ok && route["expandable"] == true && nestedOk { - // deeply nested item - result = findFirstValidChildLink(nestedRoutes) - } - - // exit if result was found - if len(result.Href) > 0 { - break - } - } - - return result -} - -func convertAltTitles(jsonEntry interface{}) []string { - altTitlesInterface, ok := jsonEntry.([]interface{}) - if !ok { - fmt.Println("Cannot convert all title to array") - return []string{} - } - var altTitles []string - for _, v := range altTitlesInterface { - altTitles = append(altTitles, v.(string)) - } - return altTitles -} - -func parseLinkEntry(item map[string]interface{}) (LinkEntry, bool) { - id, idOk := item["id"].(string) - if !idOk || len(id) == 0 { - return LinkEntry{}, false - } - - title, titleOk := item["title"].(string) - if !titleOk || len(title) == 0 { - return LinkEntry{}, false - } - - href, hrefOk := item["href"].(string) - if !hrefOk || len(href) == 0 { - return LinkEntry{}, false - } - - return LinkEntry{ - Id: id, - Title: title, - Href: href, - }, true -} - -func flattenLinks(data interface{}, locator string) ([]LinkEntry, error) { - flatData := []LinkEntry{} - - topLevel, ok := data.(map[string]interface{}) - // this is top section or a group item of nav file - if ok && topLevel["navItems"] != nil { - data, err := flattenLinks(topLevel["navItems"], fmt.Sprintf("%s.%s", locator, "navItems")) - return append(flatData, data...), err - } - - // argument came in as an array - isArray, ok := data.([]interface{}) - if ok { - for i, item := range isArray { - items, err := flattenLinks(item, fmt.Sprintf("%s[%d]", locator, i)) - if err != nil { - return []LinkEntry{}, err - } - flatData = append(flatData, items...) - } - - return flatData, nil - } - - routes, routesOk := topLevel["routes"].([]interface{}) - id, idOk := topLevel["id"].(string) - // expandable item is a valid indexable item - if topLevel["expandable"] == true && routesOk && idOk { - // need to find a firs valid child route - link := findFirstValidChildLink(routes) - link.Id = id - link.Title = topLevel["title"].(string) - // Alternative titles are optional - if topLevel["alt_title"] != nil { - link.AltTitle = convertAltTitles(topLevel["alt_title"]) - } - description, ok := topLevel["description"].(string) - if ok { - link.Description = description - } - flatData = append(flatData, link) - } - - if topLevel["expandable"] == true && routesOk { - for _, r := range routes { - i, ok := r.(map[string]interface{}) - id, idOk := i["id"].(string) - _, nestedRoutesOk := i["routes"].([]interface{}) - if ok && idOk { - // all of these are required and type assertion can't fail - link, linkOk := parseLinkEntry(i) - if !linkOk { - err := fmt.Errorf("[ERROR] parsing link for href entry at %s", locator) - return []LinkEntry{}, err - } - - // Alternative titles are optional - if i["alt_title"] != nil { - link.AltTitle = convertAltTitles(i["alt_title"]) - } - - // description is optional - description, ok := i["description"].(string) - if ok { - link.Description = description - } - flatData = append(flatData, link) - } else if nestedRoutesOk { - nestedItems, err := flattenLinks(r, fmt.Sprintf("%s.%s", locator, "routes")) - if err != nil { - return []LinkEntry{}, err - } - flatData = append(flatData, nestedItems...) - } else { - fmt.Printf("[WARN] Unable to convert link id %v to string. %v in file %s\n", id, i, locator) - } - } - return flatData, nil - } - - // this is directly a link - item, ok := data.(map[string]interface{}) - if ok { - href, ok := item["href"].(string) - if ok && len(href) > 0 { - link, linkOk := parseLinkEntry(item) - if !linkOk { - err := fmt.Errorf("[ERROR] parsing link for href entry at %s", locator) - return []LinkEntry{}, err - } - - // Alternative titles are optional - if topLevel["alt_title"] != nil { - link.AltTitle = convertAltTitles(topLevel["alt_title"]) - } - - // description is optional - description, ok := item["description"].(string) - if ok { - link.Description = description - } - flatData = append(flatData, link) - } - } - return flatData, nil -} - -type groupLinkTemplate struct { - Id string `json:"id"` - IsGroup bool `json:"isGroup"` - Title string `json:"title"` - Links []interface{} `json:"links"` -} - -type servicesTemplate struct { - Id string `json:"id"` - Description string `json:"description"` - Icon string `json:"icon"` - Title string `json:"title"` - // Links can be []links or groupLinkTemplate - Links []interface{} `json:"links"` -} - -type ServiceLink struct { - LinkEntry - IsGroup bool `json:"-"` - Links []LinkEntry `json:"links,omitempty"` - AltTitle []string `json:"alt_title,omitempty"` -} - -type ServiceEntry struct { - servicesTemplate - Links []ServiceLink `json:"links,omitempty"` -} - -func findLinkById(id string, flatLinks []LinkEntry) (ServiceLink, bool) { - var link ServiceLink - for _, l := range flatLinks { - if l.Id == id { - link.LinkEntry = l - link.AltTitle = l.AltTitle - return link, true - } - } - - return link, false -} - -func injectLinks(templateData []byte, flatLinks []LinkEntry) ([]ServiceEntry, error) { - var templates []servicesTemplate - var services []ServiceEntry - err := json.Unmarshal(templateData, &templates) - if err != nil { - return services, err - } - - for _, v := range templates { - finalLinks := []ServiceLink{} - serviceEntry := ServiceEntry{ - v, - finalLinks, - } - for _, link := range v.Links { - // is not a group links section - stringLink, ok := link.(string) - if ok { - entry, found := findLinkById(stringLink, flatLinks) - if found { - finalLinks = append(finalLinks, entry) - } - } - // is a group link - g, ok := link.(map[string]interface{}) - gStr, err := json.Marshal(g) - if err == nil { - var group groupLinkTemplate - err := json.Unmarshal(gStr, &group) - if err == nil { - if ok { - for _, stringLink := range group.Links { - var castLink string - castLink, ok = stringLink.(string) - var found bool - var entry ServiceLink - if ok { - entry, found = findLinkById(castLink, flatLinks) - } - /** - * Else branch is not handled because the link is "artificial" and does not exist in navigation. - * If a link is not in the navigation, it is not indexed. - */ - if found { - finalLinks = append(finalLinks, entry) - } - } - } - - } - - // custom static entry - custom, customOk := g["custom"].(bool) - if customOk && custom { - var customLink ServiceLink - err := json.Unmarshal(gStr, &customLink) - if err == nil { - finalLinks = append(finalLinks, customLink) - } - } - } - } - serviceEntry.Links = finalLinks - services = append(services, serviceEntry) - } - - return services, nil -} - -func flattenIndexBase(indexBase []ServiceEntry, env SearchEnv) ([]ModuleIndexEntry, error) { - hccOrigins := EnvMap{ - Prod: "https://console.redhat.com", - Stage: "https://console.stage.redhat.com", - } - bundleMapping := map[string]string{ - "application-services": "Application Services", - "openshift": "OpenShift", - "ansible": "Ansible Automation Platform", - "insights": "Red Hat Insights", - "edge": "Edge management", - "settings": "Settings", - "landing": "Home", - "allservices": "Home", - "iam": "Identity & Access Management", - "internal": "Internal", - "containers": "Containers", - "quay": "Quay.io", - } - var flatLinks []ModuleIndexEntry - for _, s := range indexBase { - for _, e := range s.Links { - bundle := strings.Split(e.Href, "/")[1] - newLink := ModuleIndexEntry{ - Icon: s.Icon, - PocDescriptionT: e.Description, - Title: []string{e.Title}, - Bundle: []string{bundle}, - BundleTitle: []string{bundleMapping[bundle]}, - Id: fmt.Sprintf("hcc-module-%s-%s", e.Href, e.Id), - Uri: fmt.Sprintf("%s%s", hccOrigins[env], e.Href), - SolrCommand: "index", - ContentType: "moduleDefinition", - ViewUri: fmt.Sprintf("%s%s", hccOrigins[env], e.Href), - RelativeUri: e.Href, - AltTitle: e.AltTitle, - } - flatLinks = append(flatLinks, newLink) - } - } - return flatLinks, nil -} - -// create search index compatible documents array -func constructIndex(env SearchEnv, release Release) ([]ModuleIndexEntry, error) { - // get services template file - stageContent, err := ioutil.ReadFile(fmt.Sprintf("static/%s/%s/services/services.json", release, env)) - if err != nil { - return []ModuleIndexEntry{}, err - } - - // get static service template only for search index - // TODO: Add releases for static services - staticContent, err := ioutil.ReadFile("cmd/search/static-services-entries.json") - if err != nil { - return []ModuleIndexEntry{}, err - } - - // get all environment navigation files paths request to fill in template file - stageNavFiles, err := filepath.Glob(fmt.Sprintf("static/%s/%s/navigation/*-navigation.json", release, env)) - if err != nil { - return []ModuleIndexEntry{}, err - } - - flatLinks := []LinkEntry{} - for _, file := range stageNavFiles { - var navItemData interface{} - navFile, err := ioutil.ReadFile(file) - if !strings.Contains(file, "landing") { - if err != nil { - return []ModuleIndexEntry{}, err - } - err = json.Unmarshal(navFile, &navItemData) - if err != nil { - return []ModuleIndexEntry{}, err - } - - flatData, err := flattenLinks(navItemData, file) - if err != nil { - return []ModuleIndexEntry{}, err - } - // add group ID to link id - fragments := strings.Split(strings.Split(file, "-navigation.json")[0], "/") - navGroupId := fragments[len(fragments)-1] - for index, link := range flatData { - link.Id = fmt.Sprintf("%s.%s", navGroupId, link.Id) - flatData[index] = link - } - flatLinks = append(flatLinks, flatData...) - } - } - indexBase, err := injectLinks(stageContent, flatLinks) - if err != nil { - return []ModuleIndexEntry{}, err - } - - staticBase, err := injectLinks(staticContent, flatLinks) - if err != nil { - return []ModuleIndexEntry{}, err - } - envIndex, err := flattenIndexBase(append(indexBase, staticBase...), env) - - if err != nil { - return []ModuleIndexEntry{}, err - } - - return envIndex, nil -} - -func getEnvToken(secret string, host string) (string, error) { - data := url.Values{} - // set payload data - data.Set("client_id", "CRC-search-indexing") - data.Set("grant_type", "client_credentials") - data.Set("scope", "email profile openid") - data.Set("client_secret", secret) - - // create request and encode data - req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", host, ssoPathname), strings.NewReader(data.Encode())) - if err != nil { - return "", err - } - - // add request headers - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - client := &http.Client{} - // fire request - res, err := client.Do(req) - if err != nil { - return "", err - } - // parse body - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - bodyString := string(body) - // handle non 200 response - if res.StatusCode >= 400 { - return bodyString, fmt.Errorf(bodyString) - } - defer res.Body.Close() - - // retrieve access token - var respJson TokenResponse - err = json.Unmarshal([]byte(bodyString), &respJson) - if err != nil { - return "", err - } - - return respJson.AccessToken, nil -} - -type UploadPayload struct { - DataSource string `json:"dataSource"` - Documents []ModuleIndexEntry `json:"documents"` -} - -func uploadIndex(token string, index []ModuleIndexEntry, host string) error { - // fmt.Println(index) - // return nil - payload := UploadPayload{ - DataSource: "console", - Documents: index, - } - b, err := json.Marshal(payload) - if err != nil { - return err - } - req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", host, hydraPathname), bytes.NewBuffer(b)) - if err != nil { - return err - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyURL(&url.URL{ - Scheme: "http", - Host: "squid.corp.redhat.com:3128", - }), - }, - } - - res, err := client.Do(req) - if err != nil { - return err - } - // parse body - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - bodyString := string(body) - // handle non 200 response - if res.StatusCode >= 400 { - return fmt.Errorf(bodyString) - } - defer res.Body.Close() - - return nil -} - -func deployIndex(env SearchEnv, envSecret string, ssoHost string, hydraHost string) error { - err := env.IsValidEnv() - if err != nil { - return err - } - token, err := getEnvToken(envSecret, ssoHost) - if err != nil { - return err - } - index, err := constructIndex(env, "stable") - if err != nil { - return err - } - - err = uploadIndex(token, index, hydraHost) - if err != nil { - return err - } - - return nil -} - -func handleErrors(errors []error, dryRun bool) { - if len(errors) == 0 { - fmt.Println("Search index published successfully") - } else { - for _, e := range errors { - fmt.Println(e) - } - fmt.Println("Search index publishing failed. See above errors.") - if dryRun { - os.Exit(1) - } - } -} - -func main() { - // load env variables - godotenv.Load() - fmt.Println("Publishing search index") - secrets := EnvMap{ - Prod: os.Getenv("SEARCH_CLIENT_SECRET_PROD"), - Stage: os.Getenv("SEARCH_CLIENT_SECRET_STAGE"), - } - - ssoHosts := EnvMap{ - Prod: "https://sso.redhat.com", - Stage: "https://sso.stage.redhat.com", - } - - hydraHost := EnvMap{ - Prod: "https://access.redhat.com", - Stage: "https://access.stage.redhat.com", - } - - dryRun, _ := strconv.ParseBool(os.Getenv("SEARCH_INDEX_DRY_RUN")) - writeIndex, _ := strconv.ParseBool(os.Getenv("SEARCH_INDEX_WRITE")) - - fmt.Println("Write index:", writeIndex) - errors := []error{} - - if writeIndex { - cwd, err := filepath.Abs(".") - if err != nil { - fmt.Println("Failed to get current working directory") - errors = append(errors, err) - handleErrors(errors, dryRun) - return - } - writeEnvs := []SearchEnv{Prod, Stage} - writeReleases := []Release{Stale, Beta} - for _, env := range writeEnvs { - for _, release := range writeReleases { - searchIndex, err := constructIndex(env, release) - if err != nil { - fmt.Println("Failed to construct search index for", env, release) - errors = append(errors, err) - } else { - dirname := fmt.Sprintf("%s/static/%s/%s/search", cwd, release, env) - fileName := fmt.Sprintf("%s/search-index.json", dirname) - err := os.MkdirAll(dirname, os.ModePerm) - if err != nil { - fmt.Println("Failed to create directory", dirname) - errors = append(errors, err) - } else { - j, err := json.Marshal(searchIndex) - if err != nil { - fmt.Println("Failed to marshal search index") - errors = append(errors, err) - } - err = os.WriteFile(fileName, j, 0644) - if err != nil { - fmt.Println("Failed to write search index to", fileName) - errors = append(errors, err) - } - } - - } - - } - } - handleErrors(errors, dryRun) - return - } - - for _, env := range []SearchEnv{Stage, Prod} { - var err error - if dryRun { - fmt.Println("Attempt dry run search index for", env, "environment.") - _, err = constructIndex(env, "stable") - } else { - fmt.Println("Attempt to publish search index for", env, "environment.") - err = deployIndex(env, secrets[env], ssoHosts[env], hydraHost[env]) - } - if err != nil { - fmt.Println("[ERROR] Failed to deploy search index for", env, "environment.") - fmt.Println(err) - errors = append(errors, err) - } - } - - handleErrors(errors, dryRun) -} diff --git a/deploy/clowdapp.yml b/deploy/clowdapp.yml index 676afa35..a9ff083c 100644 --- a/deploy/clowdapp.yml +++ b/deploy/clowdapp.yml @@ -36,12 +36,6 @@ objects: - -c - chrome-migrate inheritEnv: true - - name: publish-search-index - command: - - bash - - -c - - chrome-search-index - inheritEnv: true livenessProbe: failureThreshold: 3 httpGet: diff --git a/docs/search-index.md b/docs/search-index.md index 919fcd11..951d67b3 100644 --- a/docs/search-index.md +++ b/docs/search-index.md @@ -75,11 +75,3 @@ To generate the search index follow these steps: 1. Make sure the correct files were changed. If you are changing navigation files, make sure to edit both stage and production files. 2. Run the `make generate-search-index` command from the project root - -## Publishing search index - -To publish the search index follow these steps: - -1. Make sure the correct files were changed. If you are changing navigation files, make sure to edit both stage and production files. -2. Make sure you are on RH wired network or on VPN -3. Run the `make publish-search-index` command from the project root