From 910d18db822b3fe5ceea4e6dc6f8689f666fd80d Mon Sep 17 00:00:00 2001 From: Senthil Murugan Manikantan Date: Mon, 11 Mar 2019 14:38:46 -0400 Subject: [PATCH] Support multiple hubs (#235) Added support for multiple hub values --- api/swagger-spec.json | 48 +++++++--- cmd/mockperceptor/mockperceptor.go | 24 ++--- pkg/api/finishedscanclientjob.go | 4 +- pkg/api/imagespec.go | 22 +++-- pkg/api/mockresponder.go | 16 ++-- pkg/api/model.go | 46 +++++----- pkg/api/responder.go | 4 +- pkg/api/server.go | 9 +- pkg/core/config.go | 86 ++++++++++++------ pkg/core/configmanager.go | 10 +-- pkg/core/executable.go | 15 +--- pkg/core/hubmanager.go | 67 +++++++------- pkg/core/model/actions_test.go | 18 ++-- pkg/core/model/image.go | 17 ++-- pkg/core/perceptor.go | 108 +++++++++++++++-------- pkg/core/perceptor_test.go | 137 ++++++++++++++++++++--------- pkg/core/scanscheduler.go | 13 +-- pkg/hub/hub.go | 70 +++++++++------ pkg/hub/hub_test.go | 2 +- pkg/hub/model.go | 29 +++--- 20 files changed, 456 insertions(+), 289 deletions(-) diff --git a/api/swagger-spec.json b/api/swagger-spec.json index 5ea16fb1..38ddc24f 100644 --- a/api/swagger-spec.json +++ b/api/swagger-spec.json @@ -393,9 +393,14 @@ "Repository", "Tag", "Sha", - "HubProjectName", - "HubProjectVersionName", - "HubScanName", + "Scheme", + "Domain", + "Port", + "User", + "Password", + "BlackDuckProjectName", + "BlackDuckProjectVersionName", + "BlackDuckScanName", "Priority" ], "properties": { @@ -411,16 +416,37 @@ "description": "The SHA of the image", "type": "string" }, - "HubProjectName": { - "description": "The Hub project name", + "Scheme": { + "description": "The Scheme of the Black Duck instance. Currently only https is supported", "type": "string" }, - "HubProjectVersionName": { - "description": "The Hub project version name", + "Domain": { + "description": "The Domain name of the Black Duck instance", "type": "string" }, - "HubScanName": { - "description": "The Hub scan name", + "Port": { + "description": "The Port of the Black Duck instance", + "type": "integer", + "format": "int" + }, + "User": { + "description": "The name of the user that can access the Black Duck instance", + "type": "string" + }, + "Password": { + "description": "The password for the user that can access the Blackduck instance", + "type": "string" + }, + "BlackDuckProjectName": { + "description": "The name of the project to use in the Black Duck instance", + "type": "string" + }, + "BlackDuckProjectVersionName": { + "description": "The name of the project version to use in the Black Duck instance", + "type": "string" + }, + "BlackDuckScanName": { + "description": "The name of the scan to use in the Black Duck instance", "type": "string" }, "Priority": { @@ -580,8 +606,8 @@ "description": "The number of scans that may be run at the same time", "type": "integer" }, - "HubClientTimeoutMilliseconds": { - "description": "The timeout for HTTP requests issued to the Hub", + "ClientTimeoutMilliseconds": { + "description": "The timeout for HTTP requests issued to the Black Duck instance", "type": "integer" }, "LogLevel": { diff --git a/cmd/mockperceptor/mockperceptor.go b/cmd/mockperceptor/mockperceptor.go index 6e404e64..52fa0950 100644 --- a/cmd/mockperceptor/mockperceptor.go +++ b/cmd/mockperceptor/mockperceptor.go @@ -66,8 +66,8 @@ func main() { type MockPerceptorResponder struct{} // GetModel ..... -func (mr *MockPerceptorResponder) GetModel() api.Model { - return api.Model{} +func (mr *MockPerceptorResponder) GetModel() (*api.Model, error) { + return &api.Model{}, nil } // AddPod ..... @@ -119,14 +119,18 @@ func (mr *MockPerceptorResponder) GetNextImage() api.NextImage { log.Info("GetNextImage") start := time.Now().String() imageSpec := api.ImageSpec{ - Repository: "docker.io/alpine", - Tag: "latest", - Sha: "621c2f39f8133acb8e64023a94dbdf0d5ca81896102b9e57c0dc184cadaf5528", - HubURL: "", - HubProjectName: "string", - HubProjectVersionName: "string", - HubScanName: start, - Priority: 1, + Repository: "docker.io/alpine", + Tag: "latest", + Sha: "621c2f39f8133acb8e64023a94dbdf0d5ca81896102b9e57c0dc184cadaf5528", + Scheme: "https", + Domain: "", + Port: 8443, + User: "mock-username", + Password: "mock-password", + BlackDuckProjectName: "string", + BlackDuckProjectVersionName: "string", + BlackDuckScanName: start, + Priority: 1, } return api.NextImage{ImageSpec: &imageSpec} } diff --git a/pkg/api/finishedscanclientjob.go b/pkg/api/finishedscanclientjob.go index f251fde3..fc8c6516 100644 --- a/pkg/api/finishedscanclientjob.go +++ b/pkg/api/finishedscanclientjob.go @@ -21,8 +21,8 @@ under the License. package api -// FinishedScanClientJob ..... +// FinishedScanClientJob stores the scan client job finished status type FinishedScanClientJob struct { - ImageSpec ImageSpec + ImageSpec *ImageSpec Err string } diff --git a/pkg/api/imagespec.go b/pkg/api/imagespec.go index dbbc1dfb..1f65a492 100644 --- a/pkg/api/imagespec.go +++ b/pkg/api/imagespec.go @@ -21,14 +21,18 @@ under the License. package api -// ImageSpec ..... +// ImageSpec stores the Image specification type ImageSpec struct { - Repository string - Tag string - Sha string - HubURL string - HubProjectName string - HubProjectVersionName string - HubScanName string - Priority int + Repository string + Tag string + Sha string + Scheme string + Domain string + Port int + User string + Password string + BlackDuckProjectName string + BlackDuckProjectVersionName string + BlackDuckScanName string + Priority int } diff --git a/pkg/api/mockresponder.go b/pkg/api/mockresponder.go index 3a982c7f..9cdabd68 100644 --- a/pkg/api/mockresponder.go +++ b/pkg/api/mockresponder.go @@ -55,7 +55,7 @@ type ImageInfo struct { } // GetModel ..... -func (mr *MockResponder) GetModel() Model { +func (mr *MockResponder) GetModel() (*Model, error) { // images := map[string]*ModelImageInfo{} // for key, image := range mr.Images { // scanResults := map[string]interface{}{ @@ -81,7 +81,7 @@ func (mr *MockResponder) GetModel() Model { // Images: images, // Pods: mr.Pods, // } - return Model{} + return &Model{}, nil } // perceiver @@ -199,12 +199,12 @@ func (mr *MockResponder) UpdateAllImages(allImages AllImages) error { func (mr *MockResponder) GetNextImage() NextImage { mr.NextImageCounter++ imageSpec := ImageSpec{ - HubProjectName: fmt.Sprintf("mock-perceptor-%d", mr.NextImageCounter), - HubProjectVersionName: fmt.Sprintf("mock-perceptor-project-version-%d", mr.NextImageCounter), - HubScanName: fmt.Sprintf("mock-perceptor-scan-name-%d", mr.NextImageCounter), - Repository: "abc/def/ghi", - Tag: "latest", - Sha: "123abc456def"} + BlackDuckProjectName: fmt.Sprintf("mock-perceptor-%d", mr.NextImageCounter), + BlackDuckProjectVersionName: fmt.Sprintf("mock-perceptor-project-version-%d", mr.NextImageCounter), + BlackDuckScanName: fmt.Sprintf("mock-perceptor-scan-name-%d", mr.NextImageCounter), + Repository: "abc/def/ghi", + Tag: "latest", + Sha: "123abc456def"} return NextImage{ImageSpec: &imageSpec} } diff --git a/pkg/api/model.go b/pkg/api/model.go index a2c0a4ad..7feb7a56 100644 --- a/pkg/api/model.go +++ b/pkg/api/model.go @@ -25,18 +25,16 @@ import ( "time" ) -// Model ... +// Model stores the perceptor model type Model struct { - Hubs map[string]*ModelHub - CoreModel *CoreModel - Config *ModelConfig - Scheduler *ModelScanScheduler + BlackDucks map[string]*ModelBlackDuck + CoreModel *CoreModel + Config *ModelConfig + Scheduler *ModelScanScheduler } // ModelScanScheduler ... type ModelScanScheduler struct { - ConcurrentScanLimit int - TotalScanLimit int } // CoreModel ..... @@ -56,22 +54,28 @@ type ModelImageTransition struct { Time string } -// ModelHubConfig ... -type ModelHubConfig struct { - User string - PasswordEnvVar string - ClientTimeout ModelTime +// ModelHost ... +type ModelHost struct { + Scheme string + Domain string // it can be domain name or ip address Port int + User string ConcurrentScanLimit int - TotalScanLimit int +} + +// ModelBlackDuckConfig ... +type ModelBlackDuckConfig struct { + Hosts []*ModelHost + ClientTimeout ModelTime + TLSVerification bool } // ModelConfig ..... type ModelConfig struct { - Timings *ModelTimings - Hub *ModelHubConfig - Port int - LogLevel string + Timings *ModelTimings + BlackDuck *ModelBlackDuckConfig + Port int + LogLevel string } // ModelTime ... @@ -125,13 +129,13 @@ type ModelCircuitBreaker struct { ConsecutiveFailures int } -// ModelHub describes a hub client model -type ModelHub struct { - // can we log in to the hub? +// ModelBlackDuck describes a Black Duck client model +type ModelBlackDuck struct { + // can we log in to the Black Duck? // IsLoggedIn bool // have all the projects been sucked in? HasLoadedAllCodeLocations bool - // map of project name to ... ? hub URL? + // map of project name to ... ? Black Duck URL? // Projects map[string]string // map of code location name to mapped project version url CodeLocations map[string]*ModelCodeLocation diff --git a/pkg/api/responder.go b/pkg/api/responder.go index 46bc539b..e94b52c9 100644 --- a/pkg/api/responder.go +++ b/pkg/api/responder.go @@ -25,9 +25,9 @@ import ( "net/http" ) -// Responder ..... +// Responder interface stores all the methods corresponding to Perceptor api type Responder interface { - GetModel() Model + GetModel() (*Model, error) // perceiver AddPod(pod Pod) error diff --git a/pkg/api/server.go b/pkg/api/server.go index 6237a6ad..e8855e9a 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -30,12 +30,17 @@ import ( log "github.com/sirupsen/logrus" ) -// SetupHTTPServer ..... +// SetupHTTPServer will setup all api's to be served func SetupHTTPServer(responder Responder) { // state of the program http.HandleFunc("/model", func(w http.ResponseWriter, r *http.Request) { if r.Method == "GET" { - jsonBytes, err := json.MarshalIndent(responder.GetModel(), "", " ") + model, err := responder.GetModel() + if err != nil { + responder.Error(w, r, err, 500) + return + } + jsonBytes, err := json.MarshalIndent(model, "", " ") if err != nil { responder.Error(w, r, err, 500) return diff --git a/pkg/core/config.go b/pkg/core/config.go index f3973088..dc8ce385 100644 --- a/pkg/core/config.go +++ b/pkg/core/config.go @@ -23,79 +23,110 @@ package core import ( "encoding/json" + "fmt" + "os" "time" "github.com/blackducksoftware/perceptor/pkg/api" log "github.com/sirupsen/logrus" ) -// HubConfig handles Hub-specific configuration -type HubConfig struct { - Hosts []string - User string - PasswordEnvVar string +// Host configures the Black Duck hosts +type Host struct { + Scheme string + Domain string // it can be domain name or ip address Port int + User string + Password string ConcurrentScanLimit int - TotalScanLimit int } -// Timings ... +// BlackDuckConfig handles BlackDuck-specific configuration +type BlackDuckConfig struct { + ConnectionsEnvironmentVariableName string + TLSVerification bool +} + +// Timings stores all timings configuration that is used for various operations type Timings struct { CheckForStalledScansPauseHours int StalledScanClientTimeoutHours int ModelMetricsPauseSeconds int UnknownImagePauseMilliseconds int - HubClientTimeoutMilliseconds int + ClientTimeoutMilliseconds int } -// ClientTimeout ... +// ClientTimeout returns the Black Duck client timeout func (t *Timings) ClientTimeout() time.Duration { - return time.Duration(t.HubClientTimeoutMilliseconds) * time.Millisecond + return time.Duration(t.ClientTimeoutMilliseconds) * time.Millisecond } -// CheckForStalledScansPause ... +// CheckForStalledScansPause returns an interval in hours to check the stalled scans func (t *Timings) CheckForStalledScansPause() time.Duration { return time.Duration(t.CheckForStalledScansPauseHours) * time.Hour } -// StalledScanClientTimeout ... +// StalledScanClientTimeout returns client timeout in hours for the stalled scans func (t *Timings) StalledScanClientTimeout() time.Duration { return time.Duration(t.StalledScanClientTimeoutHours) * time.Hour } -// ModelMetricsPause ... +// ModelMetricsPause returns an interval to pause the model metrics func (t *Timings) ModelMetricsPause() time.Duration { return time.Duration(t.ModelMetricsPauseSeconds) * time.Second } -// UnknownImagePause ... +// UnknownImagePause returns an interval in milliseconds to check for unknown images func (t *Timings) UnknownImagePause() time.Duration { return time.Duration(t.UnknownImagePauseMilliseconds) * time.Millisecond } -// PerceptorConfig ... +// PerceptorConfig stores the perceptor configuration type PerceptorConfig struct { Timings *Timings UseMockMode bool Port int } -// Config ... +// Config stores the input perceptor configuration type Config struct { - Hub *HubConfig + BlackDuck *BlackDuckConfig Perceptor *PerceptorConfig LogLevel string } -func (config *Config) model() *api.ModelConfig { +// getModelBlackDuckHosts will get the list of Black Duck hosts +func (config *Config) getModelBlackDuckHosts() ([]*api.ModelHost, error) { + connectionStrings, ok := os.LookupEnv(config.BlackDuck.ConnectionsEnvironmentVariableName) + if !ok { + return nil, fmt.Errorf("cannot find Black Duck hosts: environment variable %s not found", config.BlackDuck.ConnectionsEnvironmentVariableName) + } + + blackduckHosts := map[string]*api.ModelHost{} + err := json.Unmarshal([]byte(connectionStrings), &blackduckHosts) + if err != nil { + return nil, fmt.Errorf("unable to unmarshall Black Duck hosts due to %+v", err) + } + + hosts := []*api.ModelHost{} + for _, host := range blackduckHosts { + hosts = append(hosts, host) + } + + return hosts, nil +} + +// model will return the model configurations +func (config *Config) model() (*api.ModelConfig, error) { + hosts, err := config.getModelBlackDuckHosts() + if err != nil { + return nil, err + } return &api.ModelConfig{ - Hub: &api.ModelHubConfig{ - ClientTimeout: *api.NewModelTime(config.Perceptor.Timings.ClientTimeout()), - ConcurrentScanLimit: config.Hub.ConcurrentScanLimit, - PasswordEnvVar: config.Hub.PasswordEnvVar, - Port: config.Hub.Port, - TotalScanLimit: config.Hub.TotalScanLimit, - User: config.Hub.User, + BlackDuck: &api.ModelBlackDuckConfig{ + Hosts: hosts, + ClientTimeout: *api.NewModelTime(config.Perceptor.Timings.ClientTimeout()), + TLSVerification: config.BlackDuck.TLSVerification, }, LogLevel: config.LogLevel, Port: config.Perceptor.Port, @@ -105,14 +136,15 @@ func (config *Config) model() *api.ModelConfig { StalledScanClientTimeout: *api.NewModelTime(config.Perceptor.Timings.StalledScanClientTimeout()), UnknownImagePause: *api.NewModelTime(config.Perceptor.Timings.UnknownImagePause()), }, - } + }, nil } -// GetLogLevel ..... +// GetLogLevel returns the log level func (config *Config) GetLogLevel() (log.Level, error) { return log.ParseLevel(config.LogLevel) } +// dump will dump the perceptor configuration func (config *Config) dump() (string, error) { bytes, err := json.Marshal(config) if err != nil { diff --git a/pkg/core/configmanager.go b/pkg/core/configmanager.go index 8b348970..70ab7680 100644 --- a/pkg/core/configmanager.go +++ b/pkg/core/configmanager.go @@ -42,7 +42,7 @@ type ConfigManager struct { readConfigTimer *util.Timer } -// NewConfigManager ... +// NewConfigManager returns the configuration manager func NewConfigManager(configPath string, stop <-chan struct{}) *ConfigManager { cm := &ConfigManager{ ConfigPath: configPath, @@ -76,12 +76,8 @@ func (cm *ConfigManager) GetConfig() (*Config, error) { viper.BindEnv("Perceptor.Timings.UnknownImagePauseMilliseconds") viper.BindEnv("Perceptor.Timings.ClientTimeoutMilliseconds") - viper.BindEnv("Hub.Hosts") - viper.BindEnv("Hub.User") - viper.BindEnv("Hub.TotalScanLimit") - viper.BindEnv("Hub.Port") - viper.BindEnv("Hub.PasswordEnvVar") - viper.BindEnv("Hub.ConcurrentScanLimit") + viper.BindEnv("Blackduck.ConnectionsEnvironmentVariableName") + viper.BindEnv("Blackduck.TLSVerification") viper.BindEnv("LogLevel") diff --git a/pkg/core/executable.go b/pkg/core/executable.go index 5a794d1f..2143f5e4 100644 --- a/pkg/core/executable.go +++ b/pkg/core/executable.go @@ -28,15 +28,13 @@ import ( "os" "github.com/blackducksoftware/perceptor/pkg/api" - // import just for the side-effect of changing how logrus works _ "github.com/blackducksoftware/perceptor/pkg/logging" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" ) -// RunPerceptor ..... +// RunPerceptor starts the perceptor func RunPerceptor(configPath string) { log.Info("start") @@ -76,18 +74,11 @@ func RunPerceptor(configPath string) { newHub = createMockHubClient } else { log.Infof("instantiating perceptor in real mode") - password, ok := os.LookupEnv(config.Hub.PasswordEnvVar) - if !ok { - panic(fmt.Errorf("cannot find Hub password: environment variable %s not found", config.Hub.PasswordEnvVar)) - } - newHub = createHubClient(config.Hub.User, password, config.Hub.Port, config.Perceptor.Timings.ClientTimeout()) + newHub = createHubClient(config.Perceptor.Timings.ClientTimeout()) } manager := NewHubManager(newHub, stop) - scanScheduler := &ScanScheduler{ - ConcurrentScanLimit: config.Hub.ConcurrentScanLimit, - TotalScanLimit: config.Hub.TotalScanLimit, - HubManager: manager} + scanScheduler := &ScanScheduler{HubManager: manager} perceptor, err := NewPerceptor(config, config.Perceptor.Timings, scanScheduler, manager) if err != nil { log.Errorf("unable to instantiate percepter: %s", err.Error()) diff --git a/pkg/core/hubmanager.go b/pkg/core/hubmanager.go index a63381bb..42a2d87e 100644 --- a/pkg/core/hubmanager.go +++ b/pkg/core/hubmanager.go @@ -33,25 +33,28 @@ import ( var commonMistakesRegex = regexp.MustCompile("(http|://|:\\d+)") -type hubClientCreator func(host string) (*hub.Hub, error) +type hubClientCreator func(scheme string, host string, port int, username string, password string, concurrentScanLimit int) (*hub.Hub, error) -func createMockHubClient(hubURL string) (*hub.Hub, error) { +// createMockHubClient creates the mock Black Duck client +func createMockHubClient(scheme string, host string, port int, username string, password string, concurrentScanLimit int) (*hub.Hub, error) { mockRawClient := hub.NewMockRawClient(false, []string{}) - return hub.NewHub("mock-username", "mock-password", hubURL, mockRawClient, hub.DefaultTimings), nil + return hub.NewHub(username, password, host, concurrentScanLimit, mockRawClient, hub.DefaultTimings), nil } -func createHubClient(username string, password string, port int, httpTimeout time.Duration) hubClientCreator { - return func(host string) (*hub.Hub, error) { +// createHubClient creates the Black Duck http client +func createHubClient(httpTimeout time.Duration) hubClientCreator { + return func(scheme string, host string, port int, username string, password string, concurrentScanLimit int) (*hub.Hub, error) { potentialProblems := commonMistakesRegex.FindAllString(host, -1) if len(potentialProblems) > 0 { log.Warnf("Hub host %s may be invalid, potential problems are: %s", host, potentialProblems) } - baseURL := fmt.Sprintf("https://%s:%d", host, port) + baseURL := fmt.Sprintf("%s://%s:%d", scheme, host, port) + log.Debugf("creating Black Duck client with base URL: %s", baseURL) rawClient, err := hubclient.NewWithSession(baseURL, hubclient.HubClientDebugTimings, httpTimeout) if err != nil { return nil, err } - return hub.NewHub(username, password, host, rawClient, hub.DefaultTimings), nil + return hub.NewHub(username, password, host, concurrentScanLimit, rawClient, hub.DefaultTimings), nil } } @@ -61,9 +64,9 @@ type Update struct { Update hub.Update } -// HubManagerInterface ... +// HubManagerInterface includes all methods related to setup the Black Duck type HubManagerInterface interface { - SetHubs(hubURLs []string) + SetHubs(hubs map[string]*Host) HubClients() map[string]*hub.Hub StartScanClient(hubURL string, scanName string) error FinishScanClient(hubURL string, scanName string, err error) error @@ -71,7 +74,7 @@ type HubManagerInterface interface { Updates() <-chan *Update } -// HubManager ... +// HubManager stores the Black Duck Manager configuration type HubManager struct { newHub hubClientCreator // @@ -83,7 +86,7 @@ type HubManager struct { didFetchCodeLocations chan []string } -// NewHubManager ... +// NewHubManager returns the new Black Duck Manager configuration func NewHubManager(newHub hubClientCreator, stop <-chan struct{}) *HubManager { // TODO needs to be made concurrent-safe return &HubManager{ @@ -95,14 +98,10 @@ func NewHubManager(newHub hubClientCreator, stop <-chan struct{}) *HubManager { didFetchCodeLocations: make(chan []string)} } -// SetHubs ... -func (hm *HubManager) SetHubs(hubURLs []string) { - newHubURLs := map[string]bool{} - for _, hubURL := range hubURLs { - newHubURLs[hubURL] = true - } +// SetHubs setup the Black Duck +func (hm *HubManager) SetHubs(hubs map[string]*Host) { hubsToCreate := map[string]bool{} - for hubURL := range newHubURLs { + for hubURL := range hubs { if _, ok := hm.hubs[hubURL]; !ok { hubsToCreate[hubURL] = true } @@ -110,31 +109,35 @@ func (hm *HubManager) SetHubs(hubURLs []string) { // 1. create new hubs // TODO handle retries and failures intelligently go func() { - for hubURL := range hubsToCreate { - err := hm.create(hubURL) - if err != nil { - log.Errorf("unable to create Hub client for %s: %s", hubURL, err.Error()) + for host := range hubsToCreate { + if _, ok := hm.hubs[host]; !ok { + hub := hubs[host] + err := hm.create(hub.Scheme, hub.Domain, hub.Port, hub.User, hub.Password, hub.ConcurrentScanLimit) + if err != nil { + log.Errorf("unable to create Hub client for %s: %s", hub.Domain, err.Error()) + } } } }() // 2. delete removed hubs for hubURL, hub := range hm.hubs { - if _, ok := newHubURLs[hubURL]; !ok { + if _, ok := hubs[hubURL]; !ok { hub.Stop() delete(hm.hubs, hubURL) } } } -func (hm *HubManager) create(hubURL string) error { - if _, ok := hm.hubs[hubURL]; ok { - return fmt.Errorf("cannot create hub %s: already exists", hubURL) +// create creates the Black Duck instance +func (hm *HubManager) create(scheme string, host string, port int, username string, password string, concurrentScanLimit int) error { + if _, ok := hm.hubs[host]; ok { + return fmt.Errorf("cannot create hub %s: already exists", host) } - hubClient, err := hm.newHub(hubURL) + hubClient, err := hm.newHub(scheme, host, port, username, password, concurrentScanLimit) if err != nil { return err } - hm.hubs[hubURL] = hubClient + hm.hubs[host] = hubClient go func() { stop := hubClient.StopCh() updates := hubClient.Updates() @@ -143,7 +146,7 @@ func (hm *HubManager) create(hubURL string) error { case <-stop: return case nextUpdate := <-updates: - hm.updates <- &Update{HubURL: hubURL, Update: nextUpdate} + hm.updates <- &Update{HubURL: host, Update: nextUpdate} } } }() @@ -155,12 +158,12 @@ func (hm *HubManager) Updates() <-chan *Update { return hm.updates } -// HubClients ... +// HubClients returns the list of Black Duck instance func (hm *HubManager) HubClients() map[string]*hub.Hub { return hm.hubs } -// StartScanClient ... +// StartScanClient starts the Black Duck client func (hm *HubManager) StartScanClient(hubURL string, scanName string) error { hub, ok := hm.hubs[hubURL] if !ok { @@ -181,7 +184,7 @@ func (hm *HubManager) FinishScanClient(hubURL string, scanName string, scanErr e return nil } -// ScanResults ... +// ScanResults returns the scan results func (hm *HubManager) ScanResults() map[string]map[string]*hub.Scan { allScanResults := map[string]map[string]*hub.Scan{} for hubURL, hub := range hm.hubs { diff --git a/pkg/core/model/actions_test.go b/pkg/core/model/actions_test.go index 9d68a9e8..ed15efaa 100644 --- a/pkg/core/model/actions_test.go +++ b/pkg/core/model/actions_test.go @@ -276,23 +276,23 @@ func RunActionTests() { It("default hub data", func() { sha := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" image := NewImage("abc", "latest", DockerImageSha(sha), 0, "", "") - Expect(image.HubProjectName()).To(Equal("abc")) - Expect(image.HubProjectVersionName()).To(Equal("latest-" + sha[:20])) - Expect(image.HubScanName()).To(Equal(sha)) + Expect(image.GetBlackDuckProjectName()).To(Equal("abc")) + Expect(image.GetBlackDuckProjectVersionName()).To(Equal("latest-" + sha[:20])) + Expect(image.GetBlackDuckScanName()).To(Equal(sha)) }) It("missing tag", func() { sha := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" image := NewImage("abc", "", DockerImageSha(sha), 0, "", "") - Expect(image.HubProjectName()).To(Equal("abc")) - Expect(image.HubProjectVersionName()).To(Equal(sha[:20])) - Expect(image.HubScanName()).To(Equal(sha)) + Expect(image.GetBlackDuckProjectName()).To(Equal("abc")) + Expect(image.GetBlackDuckProjectVersionName()).To(Equal(sha[:20])) + Expect(image.GetBlackDuckScanName()).To(Equal(sha)) }) It("specific hub data", func() { sha := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" image := NewImage("abc", "", DockerImageSha(sha), 0, "def", "ghi") - Expect(image.HubProjectName()).To(Equal("def")) - Expect(image.HubProjectVersionName()).To(Equal("ghi")) - Expect(image.HubScanName()).To(Equal(sha)) + Expect(image.GetBlackDuckProjectName()).To(Equal("def")) + Expect(image.GetBlackDuckProjectVersionName()).To(Equal("ghi")) + Expect(image.GetBlackDuckScanName()).To(Equal(sha)) }) }) Describe("metrics", func() { diff --git a/pkg/core/model/image.go b/pkg/core/model/image.go index 02f303ee..49cc2288 100644 --- a/pkg/core/model/image.go +++ b/pkg/core/model/image.go @@ -25,7 +25,7 @@ import ( "fmt" ) -// Image ..... +// Image stores the Image configuration type Image struct { Repository string Tag string @@ -35,27 +35,28 @@ type Image struct { BlackDuckProjectVersion string } -// NewImage ..... +// NewImage returns the image congifurations func NewImage(repository string, tag string, sha DockerImageSha, priority int, blackDuckProjectName string, blackDuckProjectVersion string) *Image { return &Image{Repository: repository, Tag: tag, Sha: sha, Priority: priority, BlackDuckProjectName: blackDuckProjectName, BlackDuckProjectVersion: blackDuckProjectVersion} } +// shaPrefix returns the sha prefix func (image Image) shaPrefix() string { return string(image.Sha)[:20] } // These strings are for the scanner -// HubProjectName ..... -func (image Image) HubProjectName() string { +// GetBlackDuckProjectName returns the Black Duck project name +func (image Image) GetBlackDuckProjectName() string { if image.BlackDuckProjectName != "" { return image.BlackDuckProjectName } return image.Repository } -// HubProjectVersionName ..... -func (image Image) HubProjectVersionName() string { +// GetBlackDuckProjectVersionName returns the Black Duck project version name +func (image Image) GetBlackDuckProjectVersionName() string { if image.BlackDuckProjectVersion != "" { return image.BlackDuckProjectVersion } @@ -67,8 +68,8 @@ func (image Image) HubProjectVersionName() string { return fmt.Sprintf("%s%s", tag, image.shaPrefix()) } -// HubScanName ..... -func (image Image) HubScanName() string { +// GetBlackDuckScanName returns the Black Duck scan name +func (image Image) GetBlackDuckScanName() string { return string(image.Sha) } diff --git a/pkg/core/perceptor.go b/pkg/core/perceptor.go index 397b4323..26ea193b 100644 --- a/pkg/core/perceptor.go +++ b/pkg/core/perceptor.go @@ -22,8 +22,10 @@ under the License. package core import ( + "encoding/json" "fmt" "net/http" + "os" api "github.com/blackducksoftware/perceptor/pkg/api" m "github.com/blackducksoftware/perceptor/pkg/core/model" @@ -50,6 +52,7 @@ type Perceptor struct { // channels stop <-chan struct{} getNextImageCh chan chan *api.ImageSpec + hosts map[string]*Host } // NewPerceptor creates a Perceptor using a real hub client. @@ -142,6 +145,10 @@ func NewPerceptor(config *Config, timings *Timings, scanScheduler *ScanScheduler }() // 2. perceptor + hosts, err := getBlackDuckHosts(config) + if err != nil { + panic(err) + } perceptor := &Perceptor{ model: model, routineTaskManager: routineTaskManager, @@ -150,6 +157,7 @@ func NewPerceptor(config *Config, timings *Timings, scanScheduler *ScanScheduler config: config, stop: stop, getNextImageCh: make(chan chan *api.ImageSpec), + hosts: hosts, } go func() { @@ -167,6 +175,22 @@ func NewPerceptor(config *Config, timings *Timings, scanScheduler *ScanScheduler return perceptor, nil } +// getBlackDuckHosts will get the list of Black Duck hosts +func getBlackDuckHosts(config *Config) (map[string]*Host, error) { + connectionStrings, ok := os.LookupEnv(config.BlackDuck.ConnectionsEnvironmentVariableName) + if !ok { + return nil, fmt.Errorf("cannot find Black Duck hosts: environment variable %s not found", config.BlackDuck.ConnectionsEnvironmentVariableName) + } + + blackduckHosts := map[string]*Host{} + err := json.Unmarshal([]byte(connectionStrings), &blackduckHosts) + if err != nil { + return nil, fmt.Errorf("unable to unmarshall Black Duck hosts due to %+v", err) + } + + return blackduckHosts, nil +} + // UpdateConfig ... func (pcp *Perceptor) UpdateConfig(config *Config) { configString, err := config.dump() @@ -175,7 +199,8 @@ func (pcp *Perceptor) UpdateConfig(config *Config) { } else { log.Errorf("set config, but unable to dump to string: %s", err.Error()) } - pcp.hubManager.SetHubs(config.Hub.Hosts) + + pcp.hubManager.SetHubs(pcp.hosts) logLevel, err := config.GetLogLevel() if err != nil { log.Errorf("unable to get log level: %s", err.Error()) @@ -187,22 +212,26 @@ func (pcp *Perceptor) UpdateConfig(config *Config) { // Section: api.Responder implementation -// GetModel ..... -func (pcp *Perceptor) GetModel() api.Model { +// GetModel returns the api model +func (pcp *Perceptor) GetModel() (*api.Model, error) { coreModel := pcp.model.GetModel() - hubModels := map[string]*api.ModelHub{} + hubModels := map[string]*api.ModelBlackDuck{} for hubURL, hub := range pcp.hubManager.HubClients() { hubModels[hubURL] = <-hub.Model() } - return api.Model{ - CoreModel: coreModel, - Hubs: hubModels, - Config: pcp.config.model(), - Scheduler: pcp.scanScheduler.model(), + configModel, err := pcp.config.model() + if err != nil { + return nil, err } + return &api.Model{ + CoreModel: coreModel, + BlackDucks: hubModels, + Config: configModel, + Scheduler: pcp.scanScheduler.model(), + }, nil } -// AddPod ..... +// AddPod adds the pod to the model func (pcp *Perceptor) AddPod(apiPod api.Pod) error { recordAddPod() pod, err := APIPodToCorePod(apiPod) @@ -214,14 +243,14 @@ func (pcp *Perceptor) AddPod(apiPod api.Pod) error { return nil } -// DeletePod ..... +// DeletePod deletes the pod from the model func (pcp *Perceptor) DeletePod(qualifiedName string) { recordDeletePod() pcp.model.DeletePod(qualifiedName) log.Debugf("handled delete pod %s", qualifiedName) } -// UpdatePod ..... +// UpdatePod updates the pod in the model func (pcp *Perceptor) UpdatePod(apiPod api.Pod) error { recordUpdatePod() pod, err := APIPodToCorePod(apiPod) @@ -233,7 +262,7 @@ func (pcp *Perceptor) UpdatePod(apiPod api.Pod) error { return nil } -// AddImage ..... +// AddImage adds an image to the model func (pcp *Perceptor) AddImage(apiImage api.Image) error { recordAddImage() image, err := APIImageToCoreImage(apiImage) @@ -245,7 +274,7 @@ func (pcp *Perceptor) AddImage(apiImage api.Image) error { return nil } -// UpdateAllPods ..... +// UpdateAllPods updates all pods in the model func (pcp *Perceptor) UpdateAllPods(allPods api.AllPods) error { recordAllPods() pods := []m.Pod{} @@ -261,7 +290,7 @@ func (pcp *Perceptor) UpdateAllPods(allPods api.AllPods) error { return nil } -// UpdateAllImages ..... +// UpdateAllImages updates all images in the model func (pcp *Perceptor) UpdateAllImages(allImages api.AllImages) error { recordAllImages() images := []m.Image{} @@ -287,6 +316,7 @@ func (pcp *Perceptor) GetScanResults() api.ScanResults { return pcp.model.GetScanResults() } +// getNextImage returns the next image from the queue func (pcp *Perceptor) getNextImage(ch chan<- *api.ImageSpec) { finish := func(spec *api.ImageSpec) { select { @@ -307,21 +337,29 @@ func (pcp *Perceptor) getNextImage(ch chan<- *api.ImageSpec) { return } - finish(&api.ImageSpec{ - Repository: image.Repository, - Tag: image.Tag, - Sha: string(image.Sha), - HubURL: hub.Host(), - HubProjectName: image.HubProjectName(), - HubProjectVersionName: image.HubProjectVersionName(), - HubScanName: image.HubScanName(), - Priority: image.Priority}) - log.Debugf("handle didStartScan") - pcp.model.StartScanClient(image.Sha) - pcp.hubManager.StartScanClient(hub.Host(), string(image.Sha)) + if host, ok := pcp.hosts[hub.Host()]; ok { + finish(&api.ImageSpec{ + Repository: image.Repository, + Tag: image.Tag, + Sha: string(image.Sha), + Scheme: host.Scheme, + Domain: host.Domain, + Port: host.Port, + User: host.User, + Password: host.Password, + BlackDuckProjectName: image.GetBlackDuckProjectName(), + BlackDuckProjectVersionName: image.GetBlackDuckProjectVersionName(), + BlackDuckScanName: image.GetBlackDuckScanName(), + Priority: image.Priority}) + log.Debugf("handle didStartScan") + pcp.model.StartScanClient(image.Sha) + pcp.hubManager.StartScanClient(hub.Host(), string(image.Sha)) + return + } + log.Errorf("unable to find the Black Duck host %s from the secret", hub.Host()) } -// GetNextImage ..... +// GetNextImage returns the next image from the queue func (pcp *Perceptor) GetNextImage() api.NextImage { recordGetNextImage() log.Debugf("handling GET next image") @@ -332,7 +370,7 @@ func (pcp *Perceptor) GetNextImage() api.NextImage { return nextImage } -// PostFinishScan ..... +// PostFinishScan executes the post finished scan job func (pcp *Perceptor) PostFinishScan(job api.FinishedScanClientJob) error { recordPostFinishedScan() go func() { @@ -341,11 +379,11 @@ func (pcp *Perceptor) PostFinishScan(job api.FinishedScanClientJob) error { if job.Err != "" { scanErr = fmt.Errorf(job.Err) } - err := pcp.hubManager.FinishScanClient(job.ImageSpec.HubURL, job.ImageSpec.HubScanName, scanErr) + err := pcp.hubManager.FinishScanClient(job.ImageSpec.Domain, job.ImageSpec.BlackDuckScanName, scanErr) if err != nil { - log.Errorf("unable to record FinishScanClient for hub %s, image %s:", job.ImageSpec.HubURL, job.ImageSpec.HubScanName) + log.Errorf("unable to record FinishScanClient for hub %s, image %s:", job.ImageSpec.Domain, job.ImageSpec.BlackDuckScanName) } - image := m.NewImage(job.ImageSpec.Repository, job.ImageSpec.Tag, m.DockerImageSha(job.ImageSpec.Sha), job.ImageSpec.Priority, job.ImageSpec.HubProjectName, job.ImageSpec.HubProjectVersionName) + image := m.NewImage(job.ImageSpec.Repository, job.ImageSpec.Tag, m.DockerImageSha(job.ImageSpec.Sha), job.ImageSpec.Priority, job.ImageSpec.BlackDuckProjectName, job.ImageSpec.BlackDuckProjectVersionName) pcp.model.FinishScanJob(image, scanErr) }() log.Debugf("handled finished scan job -- %v", job) @@ -354,7 +392,7 @@ func (pcp *Perceptor) PostFinishScan(job api.FinishedScanClientJob) error { // internal use -// PostCommand ..... +// PostCommand resets the circuit breaker func (pcp *Perceptor) PostCommand(command *api.PostCommand) { if command.ResetCircuitBreaker != nil { for _, hub := range pcp.hubManager.HubClients() { @@ -366,14 +404,14 @@ func (pcp *Perceptor) PostCommand(command *api.PostCommand) { // errors -// NotFound ..... +// NotFound logs the http client not found error func (pcp *Perceptor) NotFound(w http.ResponseWriter, r *http.Request) { log.Errorf("HTTPResponder not found from request %+v", r) recordHTTPNotFound(r) http.NotFound(w, r) } -// Error ..... +// Error logs the http client errors func (pcp *Perceptor) Error(w http.ResponseWriter, r *http.Request, err error, statusCode int) { log.Errorf("HTTPResponder error %s with code %d from request %+v", err.Error(), statusCode, r) recordHTTPError(r, err, statusCode) diff --git a/pkg/core/perceptor_test.go b/pkg/core/perceptor_test.go index c96fbe0b..16045cbd 100644 --- a/pkg/core/perceptor_test.go +++ b/pkg/core/perceptor_test.go @@ -22,7 +22,9 @@ under the License. package core import ( + "encoding/json" "fmt" + "os" "sync" "time" @@ -49,7 +51,7 @@ var ( hub3Host = "hub3" ) -func newPerceptor(concurrentScanLimit int, totalScanLimit int) *Perceptor { +func newPerceptor() *Perceptor { stop := make(chan struct{}) manager := NewHubManager(createMockHubClient, stop) timings := &Timings{ @@ -58,26 +60,29 @@ func newPerceptor(concurrentScanLimit int, totalScanLimit int) *Perceptor { StalledScanClientTimeoutHours: 9999, UnknownImagePauseMilliseconds: 500, } - config := &Config{} - pcp, err := NewPerceptor(config, timings, - &ScanScheduler{ - HubManager: manager, - ConcurrentScanLimit: concurrentScanLimit, - TotalScanLimit: totalScanLimit}, - manager) + config := &Config{BlackDuck: &BlackDuckConfig{ConnectionsEnvironmentVariableName: "blackduck.json", TLSVerification: false}} + hosts := map[string]*Host{ + "hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}, + "hub2": {"https", "hub2", 8443, "mock-username", "mock-password", 2}, + "hub3": {"https", "hub3", 8443, "mock-username", "mock-password", 2}, + } + bytes, err := json.Marshal(hosts) + Expect(err).To(BeNil()) + os.Setenv("blackduck.json", string(bytes)) + pcp, err := NewPerceptor(config, timings, &ScanScheduler{HubManager: manager}, manager) Expect(err).To(BeNil()) return pcp } func newPerceptorPrepopulatedClients(fetchUnknownScansPause time.Duration) *Perceptor { - concurrentScanLimit := 2 - totalScanLimit := 5 + // concurrentScanLimit := 2 + // totalScanLimit := 5 scans := map[string][]string{ hub1Host: {image1.Sha, image2.Sha}, hub2Host: {image3.Sha}, hub3Host: {}, } - createClient := func(hubURL string) (*hub.Hub, error) { + createClient := func(scheme string, hubURL string, port int, user string, password string, concurrentScanLimit int) (*hub.Hub, error) { mockRawClient := hub.NewMockRawClient(false, scans[hubURL]) hubTimings := &hub.Timings{ ScanCompletionPause: 1 * time.Minute, @@ -87,7 +92,7 @@ func newPerceptorPrepopulatedClients(fetchUnknownScansPause time.Duration) *Perc LoginPause: hub.DefaultTimings.LoginPause, RefreshScanThreshold: hub.DefaultTimings.RefreshScanThreshold, } - return hub.NewHub("mock-username", "mock-password", hubURL, mockRawClient, hubTimings), nil + return hub.NewHub("mock-username", "mock-password", hubURL, concurrentScanLimit, mockRawClient, hubTimings), nil } stop := make(chan struct{}) @@ -102,57 +107,70 @@ func newPerceptorPrepopulatedClients(fetchUnknownScansPause time.Duration) *Perc Perceptor: &PerceptorConfig{ Timings: &Timings{}, }, - Hub: &HubConfig{}, + BlackDuck: &BlackDuckConfig{ConnectionsEnvironmentVariableName: "blackduck.json", TLSVerification: false}, + } + hosts := map[string]*Host{ + "hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}, + "hub2": {"https", "hub2", 8443, "mock-username", "mock-password", 2}, + "hub3": {"https", "hub3", 8443, "mock-username", "mock-password", 2}, } + bytes, err := json.Marshal(hosts) + Expect(err).To(BeNil()) + os.Setenv("blackduck.json", string(bytes)) pcp, err := NewPerceptor(config, timings, &ScanScheduler{ - HubManager: manager, - ConcurrentScanLimit: concurrentScanLimit, - TotalScanLimit: totalScanLimit}, + HubManager: manager, + // ConcurrentScanLimit: concurrentScanLimit, + // TotalScanLimit: totalScanLimit, + }, manager) Expect(err).To(BeNil()) return pcp } -func makeImageSpec(image *api.Image, hub string) *api.ImageSpec { +func makeImageSpec(image *api.Image, host *Host) *api.ImageSpec { return &api.ImageSpec{ - HubProjectName: image.Repository, - HubProjectVersionName: fmt.Sprintf("%s-%s", image.Tag, image.Sha[:20]), - HubScanName: image.Sha, - HubURL: hub, - Repository: image.Repository, - Sha: image.Sha, - Tag: image.Tag, - Priority: *image.Priority, + BlackDuckProjectName: image.Repository, + BlackDuckProjectVersionName: fmt.Sprintf("%s-%s", image.Tag, image.Sha[:20]), + BlackDuckScanName: image.Sha, + Scheme: host.Scheme, + Domain: host.Domain, + Port: host.Port, + User: host.User, + Password: host.Password, + Repository: image.Repository, + Sha: image.Sha, + Tag: image.Tag, + Priority: *image.Priority, } } func RunTestPerceptor() { Describe("Perceptor", func() { It("should experience unblocked channel communication", func() { - pcp := newPerceptor(2, 5) + pcp := newPerceptor() sha1, err := m.NewDockerImageSha(image1.Sha) Expect(err).To(BeNil()) - imageSpec := makeImageSpec(&image1, "hub1") + imageSpec := makeImageSpec(&image1, &Host{Scheme: "https", Domain: "hub1", Port: 8443, User: "mock-username", Password: "mock-password"}) nextImage := api.NextImage{ImageSpec: imageSpec} Expect(pcp.AddImage(image1)).To(BeNil()) time.Sleep(500 * time.Millisecond) Expect(len(pcp.model.Images)).To(Equal(1)) Expect(pcp.model.Images[sha1].ScanStatus).To(Equal(m.ScanStatusUnknown)) - pcp.hubManager.SetHubs([]string{"hub1"}) + pcp.hubManager.SetHubs(map[string]*Host{"hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}}) time.Sleep(1 * time.Second) Expect(pcp.model.Images[sha1].ScanStatus).To(Equal(m.ScanStatusInQueue)) Expect(pcp.GetNextImage()).To(Equal(nextImage)) - Expect(pcp.PostFinishScan(api.FinishedScanClientJob{ImageSpec: *imageSpec, Err: ""})).To(BeNil()) + Expect(pcp.PostFinishScan(api.FinishedScanClientJob{ImageSpec: imageSpec, Err: ""})).To(BeNil()) time.Sleep(500 * time.Millisecond) Expect(pcp.model.Images[sha1].ScanStatus).To(Equal(m.ScanStatusRunningHubScan)) }) It("should not assign scans when there are no hubs", func() { - pcp := newPerceptor(2, 5) + pcp := newPerceptor() pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1}, }) @@ -160,37 +178,66 @@ func RunTestPerceptor() { }) It("should not assign scans when the concurrent scan limit is 0", func() { - pcp := newPerceptor(0, 5) + pcp := newPerceptor() pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1}, }) - pcp.hubManager.SetHubs([]string{"hub1", "hub2", "hub3"}) + pcp.hubManager.SetHubs(map[string]*Host{ + "hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 0}, + "hub2": {"https", "hub2", 8443, "mock-username", "mock-password", 0}, + "hub3": {"https", "hub3", 8443, "mock-username", "mock-password", 0}, + }) time.Sleep(1 * time.Second) Expect(pcp.GetNextImage()).To(Equal(api.NextImage{})) }) It("should assign scans to different hubs, not exceeding the concurrent scan limit of any hub", func() { - pcp := newPerceptor(1, 5) + pcp := newPerceptor() pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1, image2, image3, image4, image5}, }) - pcp.hubManager.SetHubs([]string{"hub1", "hub2", "hub3"}) + pcp.hubManager.SetHubs(map[string]*Host{ + "hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 1}, + "hub2": {"https", "hub2", 8443, "mock-username", "mock-password", 1}, + "hub3": {"https", "hub3", 8443, "mock-username", "mock-password", 1}, + }) time.Sleep(1 * time.Second) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(5)) next1 := pcp.GetNextImage() - Expect(next1).To(Equal(*api.NewNextImage(makeImageSpec(&image5, next1.ImageSpec.HubURL)))) + Expect(next1).To(Equal(*api.NewNextImage(makeImageSpec(&image5, + &Host{ + Scheme: next1.ImageSpec.Scheme, + Domain: next1.ImageSpec.Domain, + Port: next1.ImageSpec.Port, + User: next1.ImageSpec.User, + Password: next1.ImageSpec.Password, + })))) time.Sleep(500 * time.Millisecond) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(4)) next2 := pcp.GetNextImage() - Expect(next2).To(Equal(*api.NewNextImage(makeImageSpec(&image4, next2.ImageSpec.HubURL)))) + Expect(next2).To(Equal(*api.NewNextImage(makeImageSpec(&image4, + &Host{ + Scheme: next2.ImageSpec.Scheme, + Domain: next2.ImageSpec.Domain, + Port: next2.ImageSpec.Port, + User: next2.ImageSpec.User, + Password: next2.ImageSpec.Password, + })))) time.Sleep(500 * time.Millisecond) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(3)) next3 := pcp.GetNextImage() - Expect(next3).To(Equal(*api.NewNextImage(makeImageSpec(&image3, next3.ImageSpec.HubURL)))) + Expect(next3).To(Equal(*api.NewNextImage(makeImageSpec(&image3, + &Host{ + Scheme: next3.ImageSpec.Scheme, + Domain: next3.ImageSpec.Domain, + Port: next3.ImageSpec.Port, + User: next3.ImageSpec.User, + Password: next3.ImageSpec.Password, + })))) time.Sleep(500 * time.Millisecond) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(2)) @@ -199,11 +246,11 @@ func RunTestPerceptor() { }) It("should handle scan client failure", func() { - pcp := newPerceptor(2, 5) + pcp := newPerceptor() pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1, image2}, }) - pcp.hubManager.SetHubs([]string{"hub1"}) + pcp.hubManager.SetHubs(map[string]*Host{"hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}}) time.Sleep(1 * time.Second) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(2)) @@ -213,7 +260,7 @@ func RunTestPerceptor() { time.Sleep(500 * time.Millisecond) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(1)) - pcp.PostFinishScan(api.FinishedScanClientJob{Err: "planned error", ImageSpec: *next1.ImageSpec}) + pcp.PostFinishScan(api.FinishedScanClientJob{Err: "planned error", ImageSpec: next1.ImageSpec}) time.Sleep(500 * time.Millisecond) Expect(pcp.model.ImageScanQueue.Size()).To(Equal(2)) @@ -227,7 +274,11 @@ func RunTestPerceptor() { pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1, image2, image3, image4, image5}, }) - pcp.hubManager.SetHubs([]string{hub1Host, hub2Host, hub3Host}) + pcp.hubManager.SetHubs(map[string]*Host{ + "hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}, + "hub2": {"https", "hub2", 8443, "mock-username", "mock-password", 2}, + "hub3": {"https", "hub3", 8443, "mock-username", "mock-password", 2}, + }) time.Sleep(1 * time.Second) // jbs, _ := json.MarshalIndent(pcp.GetModel(), "", " ") @@ -242,11 +293,11 @@ func RunTestPerceptor() { }) It("should not return the same image to scan twice", func() { - pcp := newPerceptor(2, 5) + pcp := newPerceptor() pcp.UpdateAllImages(api.AllImages{ Images: []api.Image{image1, image2, image3, image4, image5}, }) - pcp.hubManager.SetHubs([]string{"hub1"}) + pcp.hubManager.SetHubs(map[string]*Host{"hub1": {"https", "hub1", 8443, "mock-username", "mock-password", 2}}) time.Sleep(1 * time.Second) var i1 *api.NextImage diff --git a/pkg/core/scanscheduler.go b/pkg/core/scanscheduler.go index 656e3e10..8505b744 100644 --- a/pkg/core/scanscheduler.go +++ b/pkg/core/scanscheduler.go @@ -27,17 +27,15 @@ import ( "github.com/blackducksoftware/perceptor/pkg/hub" ) -// ScanScheduler ... +// ScanScheduler stores the scan scheduler type ScanScheduler struct { - TotalScanLimit int - ConcurrentScanLimit int - HubManager HubManagerInterface + HubManager HubManagerInterface } // AssignImage finds a Hub that is available to scan `image`. func (s *ScanScheduler) AssignImage(image *m.Image) *hub.Hub { for _, hub := range s.HubManager.HubClients() { - if <-hub.ScansCount() < s.TotalScanLimit && len(<-hub.InProgressScans()) < s.ConcurrentScanLimit { + if len(<-hub.InProgressScans()) < hub.ConcurrentScanLimit() { recordEvent("scanScheduler", "found hub") return hub } @@ -47,8 +45,5 @@ func (s *ScanScheduler) AssignImage(image *m.Image) *hub.Hub { } func (s *ScanScheduler) model() *api.ModelScanScheduler { - return &api.ModelScanScheduler{ - ConcurrentScanLimit: s.ConcurrentScanLimit, - TotalScanLimit: s.TotalScanLimit, - } + return &api.ModelScanScheduler{} } diff --git a/pkg/hub/hub.go b/pkg/hub/hub.go index 03670826..33066191 100644 --- a/pkg/hub/hub.go +++ b/pkg/hub/hub.go @@ -35,12 +35,13 @@ type hubAction struct { apply func() error } -// Hub ..... +// Hub stores the Black Duck configuration type Hub struct { client *Client // basic hub info - host string - status ClientStatus + host string + concurrrentScanLimit int + status ClientStatus // data model *Model errors []error @@ -56,16 +57,17 @@ type Hub struct { actions chan *hubAction } -// NewHub returns a new Hub. It will not be logged in. -func NewHub(username string, password string, host string, rawClient RawClientInterface, timings *Timings) *Hub { +// NewHub returns a new Black Duck. It will not be logged in. +func NewHub(username string, password string, host string, concurrentScanLimit int, rawClient RawClientInterface, timings *Timings) *Hub { hub := &Hub{ - client: NewClient(username, password, host, rawClient), - host: host, - status: ClientStatusDown, - model: nil, - errors: []error{}, - stop: make(chan struct{}), - actions: make(chan *hubAction)} + client: NewClient(username, password, host, rawClient), + host: host, + concurrrentScanLimit: concurrentScanLimit, + status: ClientStatusDown, + model: nil, + errors: []error{}, + stop: make(chan struct{}), + actions: make(chan *hubAction)} // model setup hub.model = NewModel(host, hub.stop, func(scanName string) (*ScanResults, error) { return hub.client.fetchScan(scanName) @@ -98,10 +100,12 @@ func NewHub(username string, password string, host string, rawClient RawClientIn // Private methods +// getStateMetrics get the state metrics func (hub *Hub) getStateMetrics() { hub.model.getStateMetrics() } +// recordError records the error func (hub *Hub) recordError(description string, err error) { if err != nil { log.Errorf("%s: %s", description, err.Error()) @@ -114,7 +118,8 @@ func (hub *Hub) recordError(description string, err error) { } } -func (hub *Hub) apiModel() *api.ModelHub { +// apiModel returns the api Model +func (hub *Hub) apiModel() *api.ModelBlackDuck { errors := make([]string, len(hub.errors)) for ix, err := range hub.errors { errors[ix] = err.Error() @@ -126,6 +131,7 @@ func (hub *Hub) apiModel() *api.ModelHub { return apiModel } +// login logins to the Black Duck instance func (hub *Hub) login() { log.Debugf("starting to login to hub %s", hub.host) err := hub.client.login() @@ -148,6 +154,7 @@ func (hub *Hub) login() { }} } +// fetchAllScans fetches all Black Duck scans func (hub *Hub) fetchAllScans() { log.Debugf("starting to fetch all scans") cls, err := hub.client.listAllCodeLocations() @@ -155,37 +162,42 @@ func (hub *Hub) fetchAllScans() { hub.model.didFetchScans(cls, err) } +// fetchAllScans fetches all unknown Black Duck scans func (hub *Hub) fetchUnknownScans() { log.Debugf("starting to fetch unknown scans") hub.model.fetchUnknownScans() } // Regular jobs - +// startRefreshScansTimer return the start refresh scan timer func (hub *Hub) startRefreshScansTimer(pause time.Duration) *util.Timer { return util.NewTimer(fmt.Sprintf("refresh-scans-%s", hub.host), pause, hub.stop, func() { // TODO implement }) } +// startLoginTimer return the start login timer func (hub *Hub) startLoginTimer(pause time.Duration) *util.Timer { return util.NewRunningTimer(fmt.Sprintf("login-%s", hub.host), pause, hub.stop, true, func() { hub.login() }) } +// startFetchAllScansTimer return the start fetch all scans timer func (hub *Hub) startFetchAllScansTimer(pause time.Duration) *util.Timer { return util.NewTimer(fmt.Sprintf("fetchScans-%s", hub.host), pause, hub.stop, func() { hub.fetchAllScans() }) } +// startFetchAllScansTimer return the start fetch unknown scans timer func (hub *Hub) startFetchUnknownScansTimer(pause time.Duration) *util.Timer { return util.NewTimer(fmt.Sprintf("fetchUnknownScans-%s", hub.host), pause, hub.stop, func() { hub.fetchUnknownScans() }) } +// startFetchAllScansTimer return the start get metrics timer func (hub *Hub) startGetMetricsTimer(pause time.Duration) *util.Timer { name := fmt.Sprintf("getMetrics-%s", hub.host) return util.NewRunningTimer(name, pause, hub.stop, true, func() { @@ -193,6 +205,7 @@ func (hub *Hub) startGetMetricsTimer(pause time.Duration) *util.Timer { }) } +// startFetchAllScansTimer return the start check scans for completion timer func (hub *Hub) startCheckScansForCompletionTimer(pause time.Duration) *util.Timer { name := fmt.Sprintf("checkScansForCompletion-%s", hub.host) return util.NewTimer(name, pause, hub.stop, func() { @@ -202,27 +215,27 @@ func (hub *Hub) startCheckScansForCompletionTimer(pause time.Duration) *util.Tim // Some public API methods ... -// StartScanClient ... +// StartScanClient starts the scan client func (hub *Hub) StartScanClient(scanName string) { hub.model.StartScanClient(scanName) } -// FinishScanClient ... +// FinishScanClient finishes the scan client func (hub *Hub) FinishScanClient(scanName string, scanErr error) { hub.model.FinishScanClient(scanName, scanErr) } -// ScansCount ... +// ScansCount return the Black Duck scan count func (hub *Hub) ScansCount() <-chan int { return hub.model.ScansCount() } -// InProgressScans ... +// InProgressScans return the Inprogress scan count of the Black Duck instance func (hub *Hub) InProgressScans() <-chan []string { return hub.model.InProgressScans() } -// ScanResults ... +// ScanResults return the scan results func (hub *Hub) ScanResults() <-chan map[string]*Scan { return hub.model.ScanResults() } @@ -235,7 +248,7 @@ func (hub *Hub) Updates() <-chan Update { return hub.model.Updates() } -// Stop ... +// Stop stops the Black Duck func (hub *Hub) Stop() { close(hub.stop) } @@ -245,20 +258,25 @@ func (hub *Hub) StopCh() <-chan struct{} { return hub.stop } -// Host ... +// Host return the Host func (hub *Hub) Host() string { return hub.host } -// ResetCircuitBreaker ... +// ConcurrentScanLimit return the concurrent scan limit +func (hub *Hub) ConcurrentScanLimit() int { + return hub.concurrrentScanLimit +} + +// ResetCircuitBreaker resets the circuit breaker func (hub *Hub) ResetCircuitBreaker() { recordEvent(hub.host, "resetCircuitBreaker") hub.client.resetCircuitBreaker() } -// Model ... -func (hub *Hub) Model() <-chan *api.ModelHub { - ch := make(chan *api.ModelHub) +// Model return the model +func (hub *Hub) Model() <-chan *api.ModelBlackDuck { + ch := make(chan *api.ModelBlackDuck) hub.actions <- &hubAction{"getModel", func() error { ch <- hub.apiModel() return nil @@ -266,7 +284,7 @@ func (hub *Hub) Model() <-chan *api.ModelHub { return ch } -// HasFetchedScans ... +// HasFetchedScans return whether there is any fetched scans func (hub *Hub) HasFetchedScans() <-chan bool { return hub.model.HasFetchedScans() } diff --git a/pkg/hub/hub_test.go b/pkg/hub/hub_test.go index 97c3442a..59f4561d 100644 --- a/pkg/hub/hub_test.go +++ b/pkg/hub/hub_test.go @@ -39,7 +39,7 @@ func newClient(ignoreEvents bool) (*MockRawClient, *Hub) { LoginPause: DefaultTimings.LoginPause, RefreshScanThreshold: DefaultTimings.RefreshScanThreshold, } - hub := NewHub("sysadmin", "password", "host1", rawClient, timings) + hub := NewHub("sysadmin", "password", "host1", 2, rawClient, timings) if ignoreEvents { go func() { updates := hub.Updates() diff --git a/pkg/hub/model.go b/pkg/hub/model.go index da91d56b..e95f0871 100644 --- a/pkg/hub/model.go +++ b/pkg/hub/model.go @@ -25,7 +25,6 @@ import ( "fmt" "github.com/blackducksoftware/hub-client-go/hubapi" - "github.com/blackducksoftware/perceptor/pkg/api" log "github.com/sirupsen/logrus" ) @@ -35,7 +34,7 @@ type modelAction struct { apply func() error } -// Model ..... +// Model stores the Black Duck model type Model struct { host string hasFetchedScans bool @@ -46,7 +45,7 @@ type Model struct { actions chan *modelAction } -// NewModel ... +// NewModel return the Black Duck model func NewModel(host string, stop <-chan struct{}, fetchScan func(string) (*ScanResults, error)) *Model { model := &Model{ host: host, @@ -101,7 +100,7 @@ func (model *Model) getStateMetrics() { recordScanStageCounts(model.host, <-ch) } -func (model *Model) apiModel() *api.ModelHub { +func (model *Model) apiModel() *api.ModelBlackDuck { codeLocations := map[string]*api.ModelCodeLocation{} for name, scan := range model.scans { cl := &api.ModelCodeLocation{Stage: scan.Stage.String()} @@ -115,7 +114,7 @@ func (model *Model) apiModel() *api.ModelHub { } codeLocations[name] = cl } - return &api.ModelHub{ + return &api.ModelBlackDuck{ Errors: nil, Status: "", HasLoadedAllCodeLocations: model.scans != nil, @@ -244,7 +243,7 @@ func (model *Model) checkScansForCompletion() { // Some public API methods ... -// StartScanClient ... +// StartScanClient starts the scan client func (model *Model) StartScanClient(scanName string) { model.actions <- &modelAction{"startScanClient", func() error { model.scans[scanName] = &Scan{Stage: ScanStageScanClient} @@ -252,7 +251,7 @@ func (model *Model) StartScanClient(scanName string) { }} } -// FinishScanClient ... +// FinishScanClient finishes the scan client func (model *Model) FinishScanClient(scanName string, scanErr error) { model.actions <- &modelAction{"finishScanClient", func() error { scan, ok := model.scans[scanName] @@ -271,7 +270,7 @@ func (model *Model) FinishScanClient(scanName string, scanErr error) { }} } -// ScansCount ... +// ScansCount returns the scan count func (model *Model) ScansCount() <-chan int { ch := make(chan int) model.actions <- &modelAction{"getScansCount", func() error { @@ -287,7 +286,7 @@ func (model *Model) ScansCount() <-chan int { return ch } -// InProgressScans ... +// InProgressScans returns the inprogress scan count func (model *Model) InProgressScans() <-chan []string { ch := make(chan []string) model.actions <- &modelAction{"getInProgressScans", func() error { @@ -303,7 +302,7 @@ func (model *Model) InProgressScans() <-chan []string { return ch } -// ScanResults ... +// ScanResults returns the scan results func (model *Model) ScanResults() <-chan map[string]*Scan { ch := make(chan map[string]*Scan) model.actions <- &modelAction{"getScanResults", func() error { @@ -317,9 +316,9 @@ func (model *Model) ScanResults() <-chan map[string]*Scan { return ch } -// Model ... -func (model *Model) Model() <-chan *api.ModelHub { - ch := make(chan *api.ModelHub) +// Model returns the model +func (model *Model) Model() <-chan *api.ModelBlackDuck { + ch := make(chan *api.ModelBlackDuck) model.actions <- &modelAction{"getModel", func() error { ch <- model.apiModel() return nil @@ -327,7 +326,7 @@ func (model *Model) Model() <-chan *api.ModelHub { return ch } -// HasFetchedScans ... +// HasFetchedScans checks whether there is any has fetched scans func (model *Model) HasFetchedScans() <-chan bool { ch := make(chan bool) model.actions <- &modelAction{"hasFetchedScans", func() error { @@ -337,7 +336,7 @@ func (model *Model) HasFetchedScans() <-chan bool { return ch } -// Updates ... +// Updates publish the updates func (model *Model) Updates() <-chan Update { return model.publishUpdatesCh }