From 428b540a3e2e456a8ca5a922f1641bd3e1157748 Mon Sep 17 00:00:00 2001 From: Chris Battarbee Date: Wed, 27 Mar 2024 09:45:45 +0000 Subject: [PATCH] Add apiserver --- apiserver/internal/server/caches.go | 37 +++++++ apiserver/internal/server/current_status.go | 103 ++++++++++++++++++ apiserver/internal/server/incidents.go | 94 ++++++++++++++++ apiserver/internal/server/server.go | 18 ++- apiserver/internal/server/status_pages.go | 20 ++++ apiserver/main.go | 1 + common/api/api.go | 2 +- common/db/db.go | 12 ++ .../scraper/providers/statusio/statusio.go | 14 ++- 9 files changed, 290 insertions(+), 11 deletions(-) create mode 100644 apiserver/internal/server/caches.go create mode 100644 apiserver/internal/server/current_status.go create mode 100644 apiserver/internal/server/incidents.go create mode 100644 apiserver/internal/server/status_pages.go diff --git a/apiserver/internal/server/caches.go b/apiserver/internal/server/caches.go new file mode 100644 index 0000000..0cebea0 --- /dev/null +++ b/apiserver/internal/server/caches.go @@ -0,0 +1,37 @@ +package server + +import ( + "context" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" + "time" +) + +func (s *Server) StartCaches(ctx context.Context) { + go s.updateStatusPageCache(ctx) +} + +const statusPageCacheRefreshInterval = 5 * time.Minute + +func (s *Server) updateStatusPageCache(ctx context.Context) { + ticker := time.NewTicker(statusPageCacheRefreshInterval) + s.updateStatusPageCacheInner(ctx) + for { + select { + case <-ticker.C: + s.updateStatusPageCacheInner(ctx) + } + } +} + +func (s *Server) updateStatusPageCacheInner(ctx context.Context) { + statusPages, err := s.dbClient.GetAllStatusPages(ctx) + if err != nil { + s.logger.Error("failed to get status pages", zap.Error(err)) + return + } + + for _, statusPage := range statusPages { + s.statusPageCache.Set(statusPage.URL, statusPage, cache.DefaultExpiration) + } +} diff --git a/apiserver/internal/server/current_status.go b/apiserver/internal/server/current_status.go new file mode 100644 index 0000000..861bee2 --- /dev/null +++ b/apiserver/internal/server/current_status.go @@ -0,0 +1,103 @@ +package server + +import ( + "context" + "github.com/gin-gonic/gin" + "github.com/metoro-io/statusphere/common/api" + "github.com/patrickmn/go-cache" + "github.com/pkg/errors" + "go.uber.org/zap" + "net/http" +) + +type CurrentStatusResponse struct { + IsOkay bool `json:"isOkay"` +} + +// currentStatus is a handler for the /current-status endpoint. +// It has a required query parameter of statusPageUrl +func (s *Server) currentStatus(context *gin.Context) { + ctx := context.Request.Context() + statusPageUrl := context.Query("statusPageUrl") + if statusPageUrl == "" { + context.JSON(http.StatusBadRequest, gin.H{"error": "statusPageUrl is required"}) + return + } + + // Attempt to get the incidents from the cache + incidents, found, err := s.getCurrentIncidentsFromCache(ctx, statusPageUrl) + if err != nil { + s.logger.Error("failed to get incidents from cache", zap.Error(err)) + context.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get incidents from cache"}) + return + } + if found { + if len(incidents) > 0 { + context.JSON(http.StatusOK, CurrentStatusResponse{IsOkay: false}) + return + } + context.JSON(http.StatusOK, CurrentStatusResponse{IsOkay: true}) + return + } + + // Attempt to get the incidents from the database + incidents, found, err = s.getCurrentIncidentsFromDatabase(ctx, statusPageUrl) + if err != nil { + s.logger.Error("failed to get incidents from database", zap.Error(err)) + context.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get incidents from database"}) + return + } + if !found { + context.JSON(http.StatusNotFound, gin.H{"error": "status page not indexed"}) + return + } + + s.currentIncidentCache.Set(statusPageUrl, incidents, cache.DefaultExpiration) + if len(incidents) > 0 { + context.JSON(http.StatusOK, CurrentStatusResponse{IsOkay: false}) + return + } + context.JSON(http.StatusOK, CurrentStatusResponse{IsOkay: true}) +} + +// getCurrentIncidentsFromCache attempts to get the current incidents from the cache. +// If the incidents are found in the cache, it returns them. +// If the incidents are not found in the cache, it returns false for the second return value. + +func (s *Server) getCurrentIncidentsFromCache(ctx context.Context, statusPageUrl string) ([]api.Incident, bool, error) { + incidents, found := s.currentIncidentCache.Get(statusPageUrl) + if !found { + return nil, false, nil + } + + incidentsCasted, ok := incidents.([]api.Incident) + if !ok { + return nil, false, errors.New("failed to cast incidents to []api.Incident") + } + + return incidentsCasted, true, nil +} + +// getCurrentIncidentsFromDatabase attempts to get the current incidents from the database. +// If the incidents are found in the database, it returns them. +// If the incidents are not found in the database, it returns false for the second return value. +func (s *Server) getCurrentIncidentsFromDatabase(ctx context.Context, statusPageUrl string) ([]api.Incident, bool, error) { + incidents, err := s.dbClient.GetCurrentIncidents(ctx, statusPageUrl) + if err != nil { + return nil, false, err + } + + if len(incidents) == 0 { + // See if the status page exists + statusPage, err := s.dbClient.GetStatusPage(ctx, statusPageUrl) + if err != nil { + return nil, false, err + } + if statusPage == nil { + // The status page does not exist + return nil, false, nil + } + } + + return incidents, true, nil +} diff --git a/apiserver/internal/server/incidents.go b/apiserver/internal/server/incidents.go new file mode 100644 index 0000000..9300bbc --- /dev/null +++ b/apiserver/internal/server/incidents.go @@ -0,0 +1,94 @@ +package server + +import ( + "context" + "github.com/gin-gonic/gin" + "github.com/metoro-io/statusphere/common/api" + "github.com/patrickmn/go-cache" + "github.com/pkg/errors" + "go.uber.org/zap" + "net/http" +) + +type IncidentsResponse struct { + Incidents []api.Incident `json:"incidents"` +} + +// incidents is a handler for the /incidents endpoint. +// It has a required query parameter of statusPageUrl +func (s *Server) incidents(context *gin.Context) { + ctx := context.Request.Context() + statusPageUrl := context.Query("statusPageUrl") + if statusPageUrl == "" { + context.JSON(http.StatusBadRequest, gin.H{"error": "statusPageUrl is required"}) + return + } + + // Attempt to get the incidents from the cache + incidents, found, err := s.getIncidentsFromCache(ctx, statusPageUrl) + if err != nil { + s.logger.Error("failed to get incidents from cache", zap.Error(err)) + context.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get incidents from cache"}) + return + } + if found { + context.JSON(http.StatusOK, IncidentsResponse{Incidents: incidents}) + return + } + + // Attempt to get the incidents from the database + incidents, found, err = s.getIncidentsFromDatabase(ctx, statusPageUrl) + if err != nil { + s.logger.Error("failed to get incidents from database", zap.Error(err)) + context.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get incidents from database"}) + return + } + if !found { + context.JSON(http.StatusNotFound, gin.H{"error": "status page not indexed"}) + return + } + + s.incidentCache.Set(statusPageUrl, incidents, cache.DefaultExpiration) + context.JSON(http.StatusOK, IncidentsResponse{Incidents: incidents}) +} + +// getIncidentsFromCache attempts to get the incidents from the cache. +// If the incidents are found in the cache, it returns them. +// If the incidents are not found in the cache, it returns false for the second return value. +func (s *Server) getIncidentsFromCache(ctx context.Context, statusPageUrl string) ([]api.Incident, bool, error) { + incidents, found := s.incidentCache.Get(statusPageUrl) + if !found { + return nil, false, nil + } + + incidentsCasted, ok := incidents.([]api.Incident) + if !ok { + return nil, false, errors.New("failed to cast incidents to []api.Incident") + } + + return incidentsCasted, true, nil +} + +// getIncidentsFromDatabase attempts to get the incidents from the database. +// If the incidents are found in the database, it returns them. +// If the incidents are not found in the database, it returns false for the second return value. +func (s *Server) getIncidentsFromDatabase(ctx context.Context, statusPageUrl string) ([]api.Incident, bool, error) { + incidents, err := s.dbClient.GetIncidents(ctx, statusPageUrl) + if err != nil { + return nil, false, err + } + + if len(incidents) == 0 { + // See if the status page exists + statusPage, err := s.dbClient.GetStatusPage(ctx, statusPageUrl) + if err != nil { + return nil, false, err + } + if statusPage == nil { + // The status page does not exist + return nil, false, nil + } + } + + return incidents, true, nil +} diff --git a/apiserver/internal/server/server.go b/apiserver/internal/server/server.go index 136ba1b..a2f2e7b 100644 --- a/apiserver/internal/server/server.go +++ b/apiserver/internal/server/server.go @@ -6,20 +6,27 @@ import ( "github.com/gin-gonic/gin" "github.com/metoro-io/statusphere/common/db" "github.com/metoro-io/statusphere/common/utils" + "github.com/patrickmn/go-cache" "github.com/pkg/errors" "go.uber.org/zap" "time" ) type Server struct { - logger *zap.Logger - dbClient *db.DbClient + logger *zap.Logger + dbClient *db.DbClient + statusPageCache *cache.Cache + incidentCache *cache.Cache + currentIncidentCache *cache.Cache } func NewServer(logger *zap.Logger, dbClient *db.DbClient) *Server { return &Server{ - logger: logger, - dbClient: dbClient, + logger: logger, + dbClient: dbClient, + statusPageCache: cache.New(15*time.Minute, 15*time.Minute), + incidentCache: cache.New(1*time.Minute, 1*time.Minute), + currentIncidentCache: cache.New(1*time.Minute, 1*time.Minute), } } @@ -37,6 +44,9 @@ func (s *Server) Serve() error { apiV1 := r.Group("/api/v1") { apiV1.Use(addNoIndexHeader()) + apiV1.GET("/statusPages", s.statusPages) + apiV1.GET("/incidents", s.incidents) + apiV1.GET("/currentStatus", s.currentStatus) } s.addFrontendRoutes(r) diff --git a/apiserver/internal/server/status_pages.go b/apiserver/internal/server/status_pages.go new file mode 100644 index 0000000..2d251dd --- /dev/null +++ b/apiserver/internal/server/status_pages.go @@ -0,0 +1,20 @@ +package server + +import ( + "github.com/gin-gonic/gin" + "github.com/metoro-io/statusphere/common/api" + "net/http" +) + +type StatusPageResponse struct { + StatusPages []api.StatusPage `json:"statusPages"` +} + +func (s *Server) statusPages(context *gin.Context) { + var statusPages []api.StatusPage + for _, statusPage := range s.statusPageCache.Items() { + statusPages = append(statusPages, statusPage.Object.(api.StatusPage)) + } + + context.JSON(http.StatusOK, StatusPageResponse{StatusPages: statusPages}) +} diff --git a/apiserver/main.go b/apiserver/main.go index c514726..f8408b4 100644 --- a/apiserver/main.go +++ b/apiserver/main.go @@ -28,6 +28,7 @@ func main() { } s := server.NewServer(logger, dbClient) + s.StartCaches(ctx) go func() { if err := s.Serve(); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/common/api/api.go b/common/api/api.go index 04e18c2..2ad4194 100644 --- a/common/api/api.go +++ b/common/api/api.go @@ -46,7 +46,7 @@ type Incident struct { Components []string `gorm:"column:components;type:jsonb"` Events IncidentEventArray `gorm:"column:events;type:jsonb"` StartTime time.Time `gorm:"secondarykey"` - EndTime *time.Time + EndTime *time.Time `gorm:"secondarykey"` Description *string DeepLink string `gorm:"primarykey"` Impact Impact `gorm:"secondarykey"` diff --git a/common/db/db.go b/common/db/db.go index 73be601..0d846bb 100644 --- a/common/db/db.go +++ b/common/db/db.go @@ -130,6 +130,9 @@ func (d *DbClient) GetStatusPage(ctx context.Context, url string) (*api.StatusPa var statusPage api.StatusPage result := d.db.Table(fmt.Sprintf(fmt.Sprintf("%s.%s", schemaName, statusPageTableName))).Where("url = ?", url).First(&statusPage) if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, nil + } return nil, result.Error } return &statusPage, nil @@ -160,6 +163,15 @@ func (d *DbClient) GetIncidents(ctx context.Context, statusPageUrl string) ([]ap return incidents, nil } +func (d *DbClient) GetCurrentIncidents(ctx context.Context, statusPageUrl string) ([]api.Incident, error) { + var incidents []api.Incident + result := d.db.Table(fmt.Sprintf("%s.%s", schemaName, incidentsTableName)).Where("status_page_url = ? AND end_time IS NULL", statusPageUrl).Find(&incidents) + if result.Error != nil { + return nil, result.Error + } + return incidents, nil +} + func (d *DbClient) CreateOrUpdateIncidents(ctx context.Context, incidents []api.Incident) error { result := d.db.Table(fmt.Sprintf("%s.%s", schemaName, incidentsTableName)).Clauses( clause.OnConflict{ diff --git a/scraper/internal/scraper/providers/statusio/statusio.go b/scraper/internal/scraper/providers/statusio/statusio.go index d05d384..a5fe710 100644 --- a/scraper/internal/scraper/providers/statusio/statusio.go +++ b/scraper/internal/scraper/providers/statusio/statusio.go @@ -159,12 +159,13 @@ func (s *StatusioProvider) parseIncidents(url string, html string) ([]api.Incide // Example transformation, customize as needed incident := api.Incident{ - Title: inc.Name, - Description: &inc.Message, - StartTime: startTime, - EndTime: endTime, - Impact: api.Impact(inc.Impact), - DeepLink: link, + Title: inc.Name, + Description: &inc.Message, + StartTime: startTime, + EndTime: endTime, + Impact: api.Impact(inc.Impact), + DeepLink: link, + StatusPageUrl: url, } incidents = append(incidents, incident) } @@ -220,6 +221,7 @@ func (s *StatusioProvider) parseCurrentIncidents(url string, html string) ([]api deepLink := selection.Find(".incident-title a").First().AttrOr("href", "") deepLink = url + deepLink incident.DeepLink = deepLink + incident.StatusPageUrl = url var minTime *time.Time = nil selection.Find(".update").Each(func(i int, sel *goquery.Selection) {