fix(publisher): control nightly interval from configuration file (#186)

Signed-off-by: wuhuizuo <[email protected]> --------- Signed-off-by: wuhuizuo <[email protected]>
PingCAP-QE · Oct 28, 2024 · 2e1e85d · 2e1e85d
1 parent e877e83
commit 2e1e85d
Show file tree

Hide file tree

Showing 15 changed files with 118 additions and 84 deletions.
diff --git a/publisher/cmd/publisher/init.go b/publisher/cmd/publisher/init.go
@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/PingCAP-QE/ee-apps/publisher/pkg/config"
+)
+
+// Load and parse configuration
+func loadConfig(configFile string) (config.Service, error) {
+	var config config.Service
+	{
+		configData, err := os.ReadFile(configFile)
+		if err != nil {
+			return config, fmt.Errorf("error reading config file: %v", err)
+		}
+		if err := yaml.Unmarshal(configData, &config); err != nil {
+			return config, fmt.Errorf("error parsing config file: %v", err)
+		}
+	}
+	return config, nil
+}
diff --git a/publisher/cmd/publisher/main.go b/publisher/cmd/publisher/main.go
@@ -16,10 +16,8 @@ import (
 	"github.com/segmentio/kafka-go"
 	"goa.design/clue/debug"
 	"goa.design/clue/log"
-	"gopkg.in/yaml.v3"
 
 	gentiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup"
-	"github.com/PingCAP-QE/ee-apps/publisher/pkg/config"
 	"github.com/PingCAP-QE/ee-apps/publisher/pkg/impl/tiup"
 )
 
@@ -54,11 +52,36 @@ func main() {
 		logLevel = zerolog.DebugLevel
 	}
 	zerolog.SetGlobalLevel(logLevel)
+	logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", gentiup.ServiceName).Logger()
 
-	// Initialize the services.
-	tiupSvc, err := initTiupService(*configFile)
+	// Load and parse configuration
+	config, err := loadConfig(*configFile)
 	if err != nil {
-		log.Fatalf(ctx, err, "failed to initialize service")
+		log.Fatalf(ctx, err, "failed to load configuration")
+	}
+
+	// Initialize the services.
+	var (
+		tiupSvc gentiup.Service
+	)
+	{
+		// Configure Kafka kafkaWriter
+		kafkaWriter := kafka.NewWriter(kafka.WriterConfig{
+			Brokers:  config.Kafka.Brokers,
+			Topic:    config.Kafka.Topic,
+			Balancer: &kafka.LeastBytes{},
+			Logger:   kafka.LoggerFunc(logger.Printf),
+		})
+
+		// Configure Redis client
+		redisClient := redis.NewClient(&redis.Options{
+			Addr:     config.Redis.Addr,
+			Password: config.Redis.Password,
+			Username: config.Redis.Username,
+			DB:       config.Redis.DB,
+		})
+
+		tiupSvc = tiup.NewService(&logger, kafkaWriter, redisClient, config.EventSource)
 	}
 
 	// Wrap the services in endpoints that can be invoked from other services
@@ -127,37 +150,3 @@ func main() {
 	wg.Wait()
 	log.Printf(ctx, "exited")
 }
-
-func initTiupService(configFile string) (gentiup.Service, error) {
-	// Load and parse configuration
-	var config config.Service
-	{
-		configData, err := os.ReadFile(configFile)
-		if err != nil {
-			return nil, fmt.Errorf("error reading config file: %v", err)
-		}
-		if err := yaml.Unmarshal(configData, &config); err != nil {
-			return nil, fmt.Errorf("error parsing config file: %v", err)
-		}
-	}
-
-	logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", gentiup.ServiceName).Logger()
-
-	// Configure Kafka kafkaWriter
-	kafkaWriter := kafka.NewWriter(kafka.WriterConfig{
-		Brokers:  config.Kafka.Brokers,
-		Topic:    config.Kafka.Topic,
-		Balancer: &kafka.LeastBytes{},
-		Logger:   kafka.LoggerFunc(logger.Printf),
-	})
-
-	// Configure Redis client
-	redisClient := redis.NewClient(&redis.Options{
-		Addr:     config.Redis.Addr,
-		Password: config.Redis.Password,
-		Username: config.Redis.Username,
-		DB:       config.Redis.DB,
-	})
-
-	return tiup.NewTiup(&logger, kafkaWriter, redisClient, config.EventSource), nil
-}
diff --git a/publisher/cmd/worker/main.go b/publisher/cmd/worker/main.go
@@ -9,6 +9,7 @@ import (
 	"os/signal"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/cloudevents/sdk-go/v2/event"
 	"github.com/go-redis/redis/v8"
@@ -62,7 +63,15 @@ func main() {
 	var handler *tiup.Publisher
 	{
 		var err error
-		handler, err = tiup.NewPublisher(config.MirrorUrl, config.LarkWebhookURL, &log.Logger, redisClient)
+		nigthlyInterval, err := time.ParseDuration(config.Options.NightlyInterval)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Error parsing nightly interval")
+		}
+		handler, err = tiup.NewPublisher(&log.Logger, redisClient, tiup.PublisherOptions{
+			MirrorURL:       config.Options.MirrorURL,
+			LarkWebhookURL:  config.Options.LarkWebhookURL,
+			NightlyInterval: nigthlyInterval,
+		})
 		if err != nil {
 			log.Fatal().Err(err).Msg("Error creating handler")
 		}

diff --git a/publisher/design/design.go b/publisher/design/design.go
@@ -4,7 +4,7 @@ package design
 import (
 	_ "goa.design/plugins/v3/zerologger"
 
-	. "goa.design/goa/v3/dsl"
+	. "goa.design/goa/v3/dsl" //nolint
 )
 
 var _ = API("publisher", func() {
@@ -57,7 +57,7 @@ var _ = Service("tiup", func() {
 			Required("request_id")
 		})
 		Result(String, "request state", func() {
-			Enum("queued", "processing", "success", "failed")
+			Enum("queued", "processing", "success", "failed", "canceled")
 		})
 		HTTP(func() {
 			GET("/publish-request/{request_id}")

diff --git a/publisher/example/config/worker-example.yaml b/publisher/example/config/worker-example.yaml
@@ -3,5 +3,7 @@ kafka:
     - example-bootstrap.kafka:9092
   topic: example-topic
   consumer_group: example-group
-mirror_url: http://tiup.mirror.site
-lark_webhook_url: https://feishu.custom-bot-webhook # create and copy the url then paste here.
+options:
+  mirror_url: http://tiup.mirror.site
+  lark_webhook_url: https://feishu.custom-bot-webhook # create and copy the url then paste here.
+  nightly_interval: 1h
diff --git a/publisher/gen/http/openapi.json b/publisher/gen/http/openapi.json
@@ -1 +1 @@
-{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"[email protected]","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Cupiditate suscipit hic quidem voluptates nostrum necessitatibus."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string","enum":["queued","processing","success","failed"]}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Optio necessitatibus ipsa incidunt."},"request_id":{"type":"string","description":"The request id","example":"Et ullam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Illo harum quis voluptatem."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Expedita et necessitatibus ut molestias."}},"example":{"artifact_url":"Et suscipit et.","request_id":"Corrupti ut natus aut ipsam reprehenderit.","tiup-mirror":"Assumenda tempora autem accusantium est.","version":"Ab expedita repellendus."},"required":["artifact_url","tiup-mirror"]}}}
+{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"[email protected]","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Cupiditate suscipit hic quidem voluptates nostrum necessitatibus."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string","enum":["queued","processing","success","failed","canceled"]}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Optio necessitatibus ipsa incidunt."},"request_id":{"type":"string","description":"The request id","example":"Et ullam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Illo harum quis voluptatem."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Expedita et necessitatibus ut molestias."}},"example":{"artifact_url":"Et suscipit et.","request_id":"Corrupti ut natus aut ipsam reprehenderit.","tiup-mirror":"Assumenda tempora autem accusantium est.","version":"Ab expedita repellendus."},"required":["artifact_url","tiup-mirror"]}}}
diff --git a/publisher/gen/http/openapi.yaml b/publisher/gen/http/openapi.yaml
@@ -64,6 +64,7 @@ paths:
                             - processing
                             - success
                             - failed
+                            - canceled
             schemes:
                 - http
 definitions:

diff --git a/publisher/gen/http/openapi3.json b/publisher/gen/http/openapi3.json
@@ -1 +1 @@
-{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"[email protected]"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"Omnis expedita.","request_id":"Est sequi placeat.","tiup-mirror":"Pariatur rerum consectetur deleniti architecto sunt.","version":"Dicta id perferendis rem a."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Eum saepe nihil omnis dolorem eveniet."},"description":"request track ids","example":["Rerum voluptas dolore.","Eveniet vero voluptas.","Voluptates voluptatem accusamus nisi omnis quia molestias."]},"example":["Pariatur consequuntur itaque est.","Exercitationem atque amet optio."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Aliquam ut laborum nulla."},"example":"Error qui dolores et."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"queued","enum":["queued","processing","success","failed"]},"example":"failed"}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et eum odit eos ratione."},"request_id":{"type":"string","description":"The request id","example":"Voluptas ratione hic libero nisi."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Rerum tempore voluptas."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Laboriosam optio omnis cupiditate magnam nisi."}},"example":{"artifact_url":"Rerum voluptate accusantium optio.","request_id":"Atque vero in molestiae odit consequatur.","tiup-mirror":"Quibusdam nisi quam.","version":"Magni quia adipisci excepturi."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]}
+{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"[email protected]"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"Omnis expedita.","request_id":"Est sequi placeat.","tiup-mirror":"Pariatur rerum consectetur deleniti architecto sunt.","version":"Dicta id perferendis rem a."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Eum saepe nihil omnis dolorem eveniet."},"description":"request track ids","example":["Rerum voluptas dolore.","Eveniet vero voluptas.","Voluptates voluptatem accusamus nisi omnis quia molestias."]},"example":["Pariatur consequuntur itaque est.","Exercitationem atque amet optio."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Aliquam ut laborum nulla."},"example":"Error qui dolores et."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"processing","enum":["queued","processing","success","failed","canceled"]},"example":"queued"}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et eum odit eos ratione."},"request_id":{"type":"string","description":"The request id","example":"Voluptas ratione hic libero nisi."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Rerum tempore voluptas."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Laboriosam optio omnis cupiditate magnam nisi."}},"example":{"artifact_url":"Rerum voluptate accusantium optio.","request_id":"Atque vero in molestiae odit consequatur.","tiup-mirror":"Quibusdam nisi quam.","version":"Magni quia adipisci excepturi."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]}
diff --git a/publisher/gen/http/openapi3.yaml b/publisher/gen/http/openapi3.yaml
@@ -69,13 +69,14 @@ paths:
                             schema:
                                 type: string
                                 description: request state
-                                example: queued
+                                example: processing
                                 enum:
                                     - queued
                                     - processing
                                     - success
                                     - failed
-                            example: failed
+                                    - canceled
+                            example: queued
 components:
     schemas:
         RequestToPublishRequestBody:

diff --git a/publisher/gen/http/tiup/client/encode_decode.go b/publisher/gen/http/tiup/client/encode_decode.go
diff --git a/publisher/pkg/config/config.go b/publisher/pkg/config/config.go
@@ -5,9 +5,12 @@ type Worker struct {
 		KafkaBasic    `yaml:",inline" json:",inline"`
 		ConsumerGroup string `yaml:"consumer_group" json:"consumer_group,omitempty"`
 	} `yaml:"kafka" json:"kafka,omitempty"`
-	Redis          Redis  `yaml:"redis" json:"redis,omitempty"`
-	MirrorUrl      string `yaml:"mirror_url" json:"mirror_url,omitempty"`
-	LarkWebhookURL string `yaml:"lark_webhook_url" json:"lark_webhook_url,omitempty"`
+	Redis   Redis `yaml:"redis" json:"redis,omitempty"`
+	Options struct {
+		MirrorURL       string `yaml:"mirror_url" json:"mirror_url,omitempty"`
+		LarkWebhookURL  string `yaml:"lark_webhook_url" json:"lark_webhook_url,omitempty"`
+		NightlyInterval string `yaml:"nightly_interval" json:"nightly_interval,omitempty"`
+	}
 }
 
 type Service struct {

diff --git a/publisher/pkg/impl/tiup/funcs.go b/publisher/pkg/impl/tiup/funcs.go
@@ -229,7 +229,7 @@ func AnalyzeFromOciArtifact(repo, tag string) ([]PublishRequest, error) {
 	return publishRequests, nil
 }
 
-func AnalyzeFromOciArtifactUrl(url string) ([]PublishRequest, error) {
+func analyzeFromOciArtifactUrl(url string) ([]PublishRequest, error) {
 	repo, tag, err := splitRepoAndTag(url)
 	if err != nil {
 		return nil, err