From f40c0b171d99f0a28248187944a12e55dcf5a937 Mon Sep 17 00:00:00 2001 From: kunalvirwal Date: Mon, 4 Nov 2024 21:51:00 +0530 Subject: [PATCH 1/5] Deprecate docker.ListContainers and update with appmaker.FetchAllApplicationNames Signed-off-by: kunalvirwal --- helper.go | 29 +++++++++++++---------------- lib/docker/container.go | 2 ++ lib/mongo/read.go | 2 +- services/appmaker/helper.go | 10 ++++++---- services/appmaker/monitor.go | 25 +++++++++++-------------- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/helper.go b/helper.go index 4eebbcac..5d230d0b 100644 --- a/helper.go +++ b/helper.go @@ -11,6 +11,7 @@ import ( "github.com/sdslabs/gasper/lib/database" "github.com/sdslabs/gasper/lib/docker" "github.com/sdslabs/gasper/lib/utils" + "github.com/sdslabs/gasper/services/appmaker" "google.golang.org/grpc" ) @@ -58,42 +59,38 @@ func buildHTTPServer(handler http.Handler, port int) *http.Server { } func setupDatabaseContainer(serviceName string) { - containers, err := docker.ListContainers() - if err != nil { - utils.LogError("Main-Helper-6", err) - os.Exit(1) - } + containers := appmaker.FetchAllApplicationNames() if !utils.Contains(containers, serviceName) { - utils.LogInfo("Main-Helper-7", "No %s instance found in host. Building the instance.", strings.Title(serviceName)) + utils.LogInfo("Main-Helper-6", "No %s instance found in host. Building the instance.", strings.Title(serviceName)) containerID, err := database.SetupDBInstance(serviceName) if err != nil { - utils.Log("Main-Helper-8", fmt.Sprintf("There was a problem deploying %s service.", strings.Title(serviceName)), utils.ErrorTAG) - utils.LogError("Main-Helper-9", err) + utils.Log("Main-Helper-7", fmt.Sprintf("There was a problem deploying %s service.", strings.Title(serviceName)), utils.ErrorTAG) + utils.LogError("Main-Helper-8", err) } else { - utils.LogInfo("Main-Helper-10", "%s Container has been deployed with ID:\t%s", strings.Title(serviceName), containerID) + utils.LogInfo("Main-Helper-9", "%s Container has been deployed with ID:\t%s", strings.Title(serviceName), containerID) } } else { containerStatus, err := docker.InspectContainerState(serviceName) if err != nil { - utils.Log("Main-Helper-11", "Error in fetching container state. Deleting container and deploying again.", utils.ErrorTAG) - utils.LogError("Main-Helper-12", err) + utils.Log("Main-Helper-10", "Error in fetching container state. Deleting container and deploying again.", utils.ErrorTAG) + utils.LogError("Main-Helper-11", err) err := docker.DeleteContainer(serviceName) if err != nil { - utils.LogError("Main-Helper-13", err) + utils.LogError("Main-Helper-12", err) } containerID, err := database.SetupDBInstance(serviceName) if err != nil { - utils.Log("Main-Helper-14", fmt.Sprintf("There was a problem deploying %s service even after restart.", + utils.Log("Main-Helper-13", fmt.Sprintf("There was a problem deploying %s service even after restart.", strings.Title(serviceName)), utils.ErrorTAG) - utils.LogError("Main-Helper-15", err) + utils.LogError("Main-Helper-14", err) } else { - utils.LogInfo("Main-Helper-16", "Container has been deployed with ID:\t%s", containerID) + utils.LogInfo("Main-Helper-15", "Container has been deployed with ID:\t%s", containerID) } } if !containerStatus.Running { if err := docker.StartContainer(serviceName); err != nil { - utils.LogError("Main-Helper-17", err) + utils.LogError("Main-Helper-16", err) } } } diff --git a/lib/docker/container.go b/lib/docker/container.go index 393584a9..b57fbb3f 100644 --- a/lib/docker/container.go +++ b/lib/docker/container.go @@ -126,6 +126,8 @@ func StopContainer(containerID string) error { } // ListContainers lists all containers +// Deprecated : It returns all the containers running on the system, even those which are not monitored by gasper +// Instead use appmaker.FetchAllApplicationNames() func ListContainers() ([]string, error) { ctx := context.Background() containers, err := cli.ContainerList(ctx, dockerTypes.ContainerListOptions{All: true}) diff --git a/lib/mongo/read.go b/lib/mongo/read.go index f00d91a7..40146a9d 100644 --- a/lib/mongo/read.go +++ b/lib/mongo/read.go @@ -197,7 +197,7 @@ func CountInstancesByUser(owner string, instance string) int { return len(FetchInstances(filter)) } -//CountInstanceInTimeFrame returns the number of instances created by a user +// CountInstanceInTimeFrame returns the number of instances created by a user func CountInstanceInTimeFrame(owner string, instance string, rate time.Duration) int { filter := types.M{ diff --git a/services/appmaker/helper.go b/services/appmaker/helper.go index bc11a44e..7c925e77 100644 --- a/services/appmaker/helper.go +++ b/services/appmaker/helper.go @@ -2,13 +2,14 @@ package appmaker import ( "fmt" + "os" + "path/filepath" + "github.com/sdslabs/gasper/lib/docker" "github.com/sdslabs/gasper/lib/mongo" "github.com/sdslabs/gasper/lib/redis" "github.com/sdslabs/gasper/lib/utils" "github.com/sdslabs/gasper/types" - "os" - "path/filepath" ) var path, _ = os.Getwd() @@ -56,7 +57,8 @@ func stateCleanup(appName string) { } } -func fetchAllApplicationNames() []string { +func FetchAllApplicationNames() []string { + apps := mongo.FetchDocs(mongo.InstanceCollection, types.M{ mongo.InstanceTypeKey: mongo.AppInstance, }) @@ -65,4 +67,4 @@ func fetchAllApplicationNames() []string { appNames = append(appNames, app[mongo.NameKey].(string)) } return appNames -} \ No newline at end of file +} diff --git a/services/appmaker/monitor.go b/services/appmaker/monitor.go index f9d51ceb..f69df5aa 100644 --- a/services/appmaker/monitor.go +++ b/services/appmaker/monitor.go @@ -14,23 +14,20 @@ import ( ) func registerMetrics() { - apps, err := docker.ListContainers() - if err != nil { - utils.LogError("AppMaker-Monitor-1", err) - return - } + apps := FetchAllApplicationNames() + var parsedMetricsList []interface{} for _, app := range apps { metrics, err := docker.ContainerStats(app) if err != nil { - utils.LogError("AppMaker-Monitor-2", err) + utils.LogError("AppMaker-Monitor-1", err) continue } containerStatus, err := docker.InspectContainerState(app) if err != nil { - utils.LogError("AppMaker-Monitor-3", err) + utils.LogError("AppMaker-Monitor-2", err) continue } @@ -39,7 +36,7 @@ func registerMetrics() { maxUsage := metrics.Memory.MaxUsage memoryLimit := metrics.Memory.Limit if memoryLimit == 0 { - utils.Log("AppMaker-Monitor-4", fmt.Sprintf("Container %s has stopped", app), utils.ErrorTAG) + utils.Log("AppMaker-Monitor-3", fmt.Sprintf("Container %s has stopped", app), utils.ErrorTAG) // error needs to be handled in a better way continue } @@ -48,7 +45,7 @@ func registerMetrics() { cpuTime := metrics.CPU.CPUUsage.TotalUsage onlineCPUs := metrics.CPU.OnlineCPUs if onlineCPUs == 0 { - utils.Log("AppMaker-Monitor-5", fmt.Sprintf("Container %s has stopped", app), utils.ErrorTAG) + utils.Log("AppMaker-Monitor-4", fmt.Sprintf("Container %s has stopped", app), utils.ErrorTAG) // error needs to be handled in a better way continue } @@ -68,7 +65,7 @@ func registerMetrics() { parsedMetricsList = append(parsedMetricsList, parsedMetrics) } - if _, err = mongo.BulkRegisterMetrics(parsedMetricsList); err != nil { + if _, err := mongo.BulkRegisterMetrics(parsedMetricsList); err != nil { utils.Log("AppMaker-Monitor-6", "Failed to register metrics", utils.ErrorTAG) utils.LogError("AppMaker-Monitor-7", err) } @@ -82,8 +79,8 @@ func ScheduleMetricsCollection() { } // checkContainerHealth checks the health of the containers and restarts the unhealthy ones -func checkContainerHealth(){ - apps := fetchAllApplicationNames() +func checkContainerHealth() { + apps := FetchAllApplicationNames() for _, app := range apps { containerStatus, err := docker.InspectContainerHealth(app) if err != nil { @@ -91,7 +88,7 @@ func checkContainerHealth(){ continue } // If container is unhealthy, log the error and restart the container - if containerStatus == docker.Container_Unhealthy{ + if containerStatus == docker.Container_Unhealthy { utils.Log("AppMaker-Monitor-10", fmt.Sprintf("Container %s has stopped", app), utils.ErrorTAG) if err := docker.ContainerRestart(app); err != nil { utils.LogError("AppMaker-Monitor-11", err) @@ -105,4 +102,4 @@ func ScheduleHealthCheck() { interval := configs.ServiceConfig.AppMaker.HealthInterval * time.Second scheduler := utils.NewScheduler(interval, checkContainerHealth) scheduler.RunAsync() -} \ No newline at end of file +} From 6fbc716b33966d2530a9f1708bc429d66aac2721 Mon Sep 17 00:00:00 2001 From: kunalvirwal Date: Mon, 11 Nov 2024 02:52:37 +0530 Subject: [PATCH 2/5] Add missing Import statement Signed-off-by: kunalvirwal --- helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.go b/helper.go index a45c27da..ca849103 100644 --- a/helper.go +++ b/helper.go @@ -12,10 +12,10 @@ import ( "github.com/sdslabs/gasper/lib/docker" "github.com/sdslabs/gasper/lib/utils" "github.com/sdslabs/gasper/services/appmaker" + "github.com/sdslabs/gasper/types" "google.golang.org/grpc" ) - func startGrpcServer(server *grpc.Server, port int) error { lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { From 25b8bf247ebf282b5ec847003587c0173e607d66 Mon Sep 17 00:00:00 2001 From: kunalvirwal Date: Sat, 14 Dec 2024 15:56:29 +0530 Subject: [PATCH 3/5] Use ExecProcessWthStream for git init and clone commands Signed-off-by: kunalvirwal --- lib/api/application.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/api/application.go b/lib/api/application.go index 6db223ec..de414912 100644 --- a/lib/api/application.go +++ b/lib/api/application.go @@ -95,7 +95,7 @@ func SetupApplication(app types.Application) types.ResponseError { } } - _, err = docker.ExecProcess(app.GetContainerID(), []string{"git", "init"}) + _, err = docker.ExecProcessWthStream(app.GetContainerID(), []string{"git", "init"}) if err != nil { return types.NewResErr(500, "git init unsuccessful", err) } @@ -103,17 +103,17 @@ func SetupApplication(app types.Application) types.ResponseError { var cloneURL string if len(app.GetGitAccessToken()) > 0 { split := strings.Split(app.GetGitRepositoryURL(), "//") - cloneURL = fmt.Sprintf("https://oauth2:%s@%s", app.GetGitAccessToken() , split[1]) + cloneURL = fmt.Sprintf("https://oauth2:%s@%s", app.GetGitAccessToken(), split[1]) } else { cloneURL = app.GetGitRepositoryURL() } - _, err = docker.ExecProcess(app.GetContainerID(), []string{"git", "remote", "add", "origin", cloneURL}) + _, err = docker.ExecProcessWthStream(app.GetContainerID(), []string{"git", "remote", "add", "origin", cloneURL}) if err != nil { return types.NewResErr(500, "setting remote unsuccessful", err) } - _, err = docker.ExecProcess(app.GetContainerID(), []string{"git", "pull", "origin", app.GetGitRepositoryBranch()}) + _, err = docker.ExecProcessWthStream(app.GetContainerID(), []string{"git", "pull", "origin", app.GetGitRepositoryBranch()}) if err != nil { return types.NewResErr(500, "pulling contents unsuccessful", err) } From fc3c437e4c71cf1fcd6f37480d3f862ce3e66500 Mon Sep 17 00:00:00 2001 From: kunalvirwal Date: Wed, 18 Dec 2024 15:30:53 +0530 Subject: [PATCH 4/5] Update Docker Hub repo names in config.sample.toml Signed-off-by: kunalvirwal --- config.sample.toml | 18 +++++++++--------- services/master/controllers/application.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config.sample.toml b/config.sample.toml index fa415433..119dc582 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -99,16 +99,16 @@ public_ip = "" # IPv4 address for Cloudflare's DNS records to point to. ################################### [images] -static = "docker.io/sdsws/static:latest" -php = "docker.io/sdsws/php:latest" -nodejs = "docker.io/sdsws/node:latest" -python2 = "docker.io/sdsws/python2:latest" -python3 = "docker.io/sdsws/python3:latest" -golang = "docker.io/sdsws/golang:latest" -ruby = "docker.io/sdsws/ruby:latest" -rust = "docker.io/sdsws/rust:latest" +static = "docker.io/sdslabs/static:latest" +php = "docker.io/sdslabs/php:latest" +nodejs = "docker.io/sdslabs/node:latest" +python2 = "docker.io/sdslabs/python2:latest" +python3 = "docker.io/sdslabs/python3:latest" +golang = "docker.io/sdslabs/golang:latest" +ruby = "docker.io/sdslabs/ruby:latest" +rust = "docker.io/sdslabs/rust:latest" mysql = "docker.io/mysql:latest" -mongodb = "docker.io/sdsws/alpine-mongo:latest" +mongodb = "docker.io/sdslabs/alpine-mongo:latest" postgresql = "docker.io/postgres:latest" redis = "docker.io/redis:6.0-rc3-alpine3.11" diff --git a/services/master/controllers/application.go b/services/master/controllers/application.go index d58fc916..e0ad9524 100644 --- a/services/master/controllers/application.go +++ b/services/master/controllers/application.go @@ -292,7 +292,7 @@ func FetchMetrics(c *gin.Context) { } downtimeIntensity = 0 CPURecord = append(CPURecord, metrics[i]["cpu_usage"].(float64)) - memoryRecord = append(memoryRecord, metrics[i]["memory_usage"].(float64)) + memoryRecord = append(memoryRecord, metrics[i]["memory_usage"].(float64)) } } From a4795f92a84fbe58f335e34f0a6e83df66efda91 Mon Sep 17 00:00:00 2001 From: kunalvirwal Date: Tue, 24 Dec 2024 21:34:53 +0530 Subject: [PATCH 5/5] Fix application rebuild route to run build and run commands Signed-off-by: kunalvirwal --- lib/api/application.go | 14 +++++++++++++- services/appmaker/controller.go | 15 +++++++++------ services/master/controllers/application.go | 4 ++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/lib/api/application.go b/lib/api/application.go index de414912..63dbfbfa 100644 --- a/lib/api/application.go +++ b/lib/api/application.go @@ -118,12 +118,17 @@ func SetupApplication(app types.Application) types.ResponseError { return types.NewResErr(500, "pulling contents unsuccessful", err) } + return RunBuildAndStartCommands(app) + +} + +func RunBuildAndStartCommands(app types.Application) types.ResponseError { if app.HasRcFile() { cmd := []string{"sh", "-c", fmt.Sprintf(`chmod 755 ./%s &> /proc/1/fd/1 && ./%s &> /proc/1/fd/1`, configs.GasperConfig.RcFile, configs.GasperConfig.RcFile)} - _, err = docker.ExecDetachedProcess(app.GetContainerID(), cmd) + _, err := docker.ExecDetachedProcess(app.GetContainerID(), cmd) if err != nil { // this error cannot be ignored; the chances of error here are very less // but if an error arises, this means there's some issue with "execing" @@ -134,6 +139,13 @@ func SetupApplication(app types.Application) types.ResponseError { } else { go buildAndRun(app) } + return nil +} +func StopAllProcesses(app types.Application) types.ResponseError { + _, err := docker.ExecProcessWthStream(app.GetContainerID(), []string{"kill", "-TERM", "-1"}) + if err != nil { + return types.NewResErr(500, "Unable to stop processes running inside the container", err) + } return nil } diff --git a/services/appmaker/controller.go b/services/appmaker/controller.go index db03db20..8d16d8bf 100644 --- a/services/appmaker/controller.go +++ b/services/appmaker/controller.go @@ -69,12 +69,12 @@ func (s *server) Create(ctx context.Context, body *pb.RequestBody) (*pb.Response utils.LogError("AppMaker-Controller-1", fmt.Errorf("GenDNS instance %s is of invalid format", nameServer)) } } - + if pipeline[language] == nil { return nil, fmt.Errorf("language `%s` is not supported", language) } utils.LogError("AppMaker-Controller-1", fmt.Errorf("%s", app.GetDockerImage())) - + if app.GetDockerImage() != "" { utils.LogError("AppMaker-Controller-1", fmt.Errorf("docker img")) docker.CheckAndPullImages(app.GetDockerImage()) @@ -82,7 +82,7 @@ func (s *server) Create(ctx context.Context, body *pb.RequestBody) (*pb.Response if err != nil { return nil, types.NewResErr(500, "No free port available", err) } - app.SetContainerPort(containerPort) + app.SetContainerPort(containerPort) errList := api.CreateBasicApplication(app) for _, err := range errList { @@ -159,7 +159,7 @@ func (s *server) Create(ctx context.Context, body *pb.RequestBody) (*pb.Response return &pb.ResponseBody{Data: response}, err } -// Rebuild rebuilds an application +// Rebuild stops all currently running processes ,rebuilds and restarts an application func (s *server) Rebuild(ctx context.Context, body *pb.NameHolder) (*pb.ResponseBody, error) { appName := body.GetName() app, err := mongo.FetchSingleApp(appName) @@ -167,9 +167,12 @@ func (s *server) Rebuild(ctx context.Context, body *pb.NameHolder) (*pb.Response return nil, err } - pullChanges := []string{"git", "pull", "origin", app.GetGitRepositoryBranch()} - _, err = docker.ExecProcess(app.ContainerID, pullChanges) + err = api.StopAllProcesses(app) + if err != nil { + return nil, err + } + err = api.RunBuildAndStartCommands(app) if err != nil { return nil, err } diff --git a/services/master/controllers/application.go b/services/master/controllers/application.go index e0ad9524..c5767585 100644 --- a/services/master/controllers/application.go +++ b/services/master/controllers/application.go @@ -73,9 +73,9 @@ func BulkUpdateApps(c *gin.Context) { // UpdateAppByName updates the app getting name from url params func UpdateAppByName(c *gin.Context) { - app := c.Param("app") + appName := c.Param("app") filter := types.M{ - mongo.NameKey: app, + mongo.NameKey: appName, mongo.InstanceTypeKey: mongo.AppInstance, } var data types.M