Added a better caching system for the pulling of catalog machines

.vscode/launch.json

@@ -12,6 +12,18 @@
       // "args": [
       //   "--port=5570",
       // ]
+    },
+    {
+      "name": "Test Parallels Desktop Api Catalog Providers",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceFolder}/src/main.go",
+      "envFile": "${workspaceFolder}/.env",
+      "args": [
+        "--test",
+        "--catalog-providers",
+      ]
     }
   ]
 }

src/catalog/providers/azurestorageaccount/main.go

@@ -2,7 +2,6 @@ package azurestorageaccount
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
 	"net/url"
 	"os"
@@ -12,6 +11,7 @@ import (
 
 	"github.com/Parallels/pd-api-service/basecontext"
 	"github.com/Parallels/pd-api-service/catalog/common"
+	"github.com/Parallels/pd-api-service/helpers"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
 )
@@ -105,10 +105,24 @@ func (s *AzureStorageAccountProvider) PushFile(ctx basecontext.ApiContext, rootL
 
 	defer file.Close()
 
+	md5, err := helpers.GetFileMD5Checksum(localFilePath)
+	if err != nil {
+		return err
+	}
+	// md5Hash := base64.StdEncoding.EncodeToString([]byte(md5))
+
 	_, err = azblob.UploadFileToBlockBlob(ctx.Context(), file, blobUrl, azblob.UploadToBlockBlobOptions{
 		BlockSize:   4 * 1024 * 1024,
 		Parallelism: 16,
 	})
+	if err != nil {
+		return err
+	}
+
+	_, err = blobUrl.SetHTTPHeaders(ctx.Context(), azblob.BlobHTTPHeaders{
+		ContentType: "application/octet-stream",
+		ContentMD5:  []byte(md5),
+	}, azblob.BlobAccessConditions{})
 
 	return err
 }
@@ -175,7 +189,7 @@ func (s *AzureStorageAccountProvider) FileChecksum(ctx basecontext.ApiContext, p
 
 	props, err := blobUrl.GetProperties(ctx.Context(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 
-	fileCheckSum := hex.EncodeToString(props.ContentMD5()[:])
+	fileCheckSum := string(props.ContentMD5())
 	return fileCheckSum, err
 }
 

src/catalog/pull.go

@@ -2,11 +2,14 @@ package catalog
 
 import (
 	"fmt"
+	"log"
+	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/Parallels/pd-api-service/basecontext"
 	"github.com/Parallels/pd-api-service/catalog/models"
+	"github.com/Parallels/pd-api-service/config"
 	"github.com/Parallels/pd-api-service/constants"
 	"github.com/Parallels/pd-api-service/data"
 	"github.com/Parallels/pd-api-service/errors"
@@ -60,6 +63,8 @@ func (s *CatalogManifestService) Pull(ctx basecontext.ApiContext, r *models.Pull
 
 	var manifest *models.VirtualMachineCatalogManifest
 	provider := models.CatalogManifestProvider{}
+	cfg := config.NewConfig()
+
 	if err := provider.Parse(r.Connection); err != nil {
 		response.AddError(err)
 		return response
@@ -220,16 +225,53 @@ func (s *CatalogManifestService) Pull(ctx basecontext.ApiContext, r *models.Pull
 					continue
 				}
 
-				if err := rs.PullFile(ctx, file.Path, file.Name, r.Path); err != nil {
-					ctx.LogError("Error pulling file %v: %v", file.Name, err)
+				destinationFolder := r.Path
+				fileName := file.Name
+				fileChecksum, err := rs.FileChecksum(ctx, file.Path, file.Name)
+				if err != nil {
+					ctx.LogError("Error getting file %v checksum: %v", fileName, err)
 					response.AddError(err)
 					break
 				}
 
-				response.CleanupRequest.AddLocalFileCleanupOperation(filepath.Join(r.Path, file.Name), false)
-				ctx.LogInfo("Decompressing file %v", file.Name)
-				if err := s.decompressMachine(ctx, filepath.Join(r.Path, file.Name), r.LocalMachineFolder); err != nil {
-					ctx.LogError("Error decompressing file %v: %v", file.Name, err)
+				cacheFileName := fmt.Sprintf("%s.pdpack", fileChecksum)
+				needsPulling := false
+				if cfg.IsCatalogCachingEnable() {
+					destinationFolder, err = cfg.GetCatalogCacheFolder()
+					if err != nil {
+						destinationFolder = r.Path
+					}
+					if helper.FileExists(filepath.Join(destinationFolder, cacheFileName)) {
+						ctx.LogInfo("File %v already exists in cache", fileName)
+					} else {
+						needsPulling = true
+					}
+				} else {
+					needsPulling = true
+				}
+
+				if needsPulling {
+					if err := rs.PullFile(ctx, file.Path, file.Name, destinationFolder); err != nil {
+						ctx.LogError("Error pulling file %v: %v", fileName, err)
+						response.AddError(err)
+						break
+					}
+					if cfg.IsCatalogCachingEnable() {
+						err := os.Rename(filepath.Join(destinationFolder, file.Name), filepath.Join(destinationFolder, cacheFileName))
+						if err != nil {
+							log.Fatal(err)
+						}
+					}
+				}
+
+				if !cfg.IsCatalogCachingEnable() {
+					cacheFileName = file.Name
+					response.CleanupRequest.AddLocalFileCleanupOperation(filepath.Join(destinationFolder, file.Name), false)
+				}
+
+				ctx.LogInfo("Decompressing file %v", cacheFileName)
+				if err := s.decompressMachine(ctx, filepath.Join(destinationFolder, cacheFileName), r.LocalMachineFolder); err != nil {
+					ctx.LogError("Error decompressing file %v: %v", fileName, err)
 					response.AddError(err)
 					break
 				}

src/config/main.go

@@ -2,10 +2,15 @@ package config
 
 import (
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 
+	"github.com/Parallels/pd-api-service/basecontext"
 	"github.com/Parallels/pd-api-service/common"
 	"github.com/Parallels/pd-api-service/constants"
+	"github.com/Parallels/pd-api-service/helpers"
+	"github.com/Parallels/pd-api-service/serviceprovider/system"
 
 	log "github.com/cjlapao/common-go-logger"
 	"github.com/cjlapao/common-go/helper"
@@ -128,5 +133,68 @@ func (c *Config) TLSEnabled() bool {
 }
 
 func (c *Config) GetTokenDurationMinutes() int {
-	return constants.TOKEN_DURATION_MINUTES
+	tokenDuration := os.Getenv(constants.TOKEN_DURATION_MINUTES_ENV_VAR)
+	if tokenDuration != "" {
+		return constants.DEFAULT_TOKEN_DURATION_MINUTES
+	}
+
+	intVal, err := strconv.Atoi(tokenDuration)
+	if err != nil {
+		return constants.DEFAULT_TOKEN_DURATION_MINUTES
+	}
+	return intVal
+}
+
+func (c *Config) GetRootFolder() (string, error) {
+	srv := system.Get()
+	ctx := basecontext.NewRootBaseContext()
+	currentUser, err := srv.GetCurrentUser(ctx)
+	if err != nil {
+		currentUser = "root"
+	}
+
+	if currentUser == "root" {
+		folder := "/etc/parallels-api-service"
+		err := helpers.CreateDirIfNotExist(folder)
+		if err != nil {
+			return "", err
+		}
+
+		return folder, nil
+	} else {
+		userHome, err := srv.GetUserHome(ctx, currentUser)
+		if err != nil {
+			return "", err
+		}
+		folder := userHome + "/.parallels-api-service"
+		err = helpers.CreateDirIfNotExist(folder)
+		if err != nil {
+			return "", err
+		}
+
+		return folder, nil
+	}
+}
+
+func (c *Config) GetCatalogCacheFolder() (string, error) {
+	rootFolder, err := c.GetRootFolder()
+	if err != nil {
+		return "", err
+	}
+	cacheFolder := filepath.Join(rootFolder, constants.DEFAULT_CATALOG_CACHE_FOLDER)
+	err = helpers.CreateDirIfNotExist(cacheFolder)
+	if err != nil {
+		return "", err
+	}
+
+	return cacheFolder, nil
+}
+
+func (c *Config) IsCatalogCachingEnable() bool {
+	envVar := os.Getenv(constants.DISABLE_CATALOG_CACHING_ENV_VAR)
+	if envVar == "true" || envVar == "1" {
+		return false
+	}
+
+	return true
 }

src/constants/main.go

@@ -21,33 +21,38 @@ const (
 )
 
 const (
-	DEFAULT_API_PREFIX   = "/api"
-	DEFAULT_API_PORT     = "80"
-	DEFAULT_API_TLS_PORT = "443"
-	CURRENT_USER_ENV_VAR = "PD_CURRENT_USER"
+	DEFAULT_API_PREFIX             = "/api"
+	DEFAULT_API_PORT               = "80"
+	DEFAULT_API_TLS_PORT           = "443"
+	CURRENT_USER_ENV_VAR           = "PD_CURRENT_USER"
+	DEFAULT_TOKEN_DURATION_MINUTES = 60
+	DEFAULT_CATALOG_CACHE_FOLDER   = "./catalog_cache"
 )
 
 const (
-	HMAC_SECRET_ENV_VAR     = "HMAC_SECRET"
-	LOG_LEVEL_ENV_VAR       = "LOG_LEVEL"
-	SECURITY_KEY_ENV_VAR    = "SECURITY_PRIVATE_KEY"
-	TLS_ENABLED_ENV_VAR     = "TLS_ENABLED"
-	TLS_PORT_ENV_VAR        = "TLS_PORT"
-	TLS_CERTIFICATE_ENV_VAR = "TLS_CERTIFICATE"
-	TLS_PRIVATE_KEY_ENV_VAR = "TLS_PRIVATE_KEY"
-	TOKEN_DURATION_MINUTES  = 60
-	API_PORT_ENV_VAR        = "API_PORT"
-	API_PREFIX_ENV_VAR      = "API_PREFIX"
-	ROOT_PASSWORD_ENV_VAR   = "ROOT_PASSWORD"
+	HMAC_SECRET_ENV_VAR             = "HMAC_SECRET"
+	LOG_LEVEL_ENV_VAR               = "LOG_LEVEL"
+	SECURITY_KEY_ENV_VAR            = "SECURITY_PRIVATE_KEY"
+	TLS_ENABLED_ENV_VAR             = "TLS_ENABLED"
+	TLS_PORT_ENV_VAR                = "TLS_PORT"
+	TLS_CERTIFICATE_ENV_VAR         = "TLS_CERTIFICATE"
+	TLS_PRIVATE_KEY_ENV_VAR         = "TLS_PRIVATE_KEY"
+	API_PORT_ENV_VAR                = "API_PORT"
+	API_PREFIX_ENV_VAR              = "API_PREFIX"
+	ROOT_PASSWORD_ENV_VAR           = "ROOT_PASSWORD"
+	DISABLE_CATALOG_CACHING_ENV_VAR = "DISABLE_CATALOG_CACHING"
+	TOKEN_DURATION_MINUTES_ENV_VAR  = "TOKEN_DURATION_MINUTES"
 )
 
 const (
-	API_PORT_FLAG              = "port"
-	UPDATE_ROOT_PASSWORD_FLAG  = "update-root-pass"
-	GENERATE_SECURITY_KEY_FLAG = "gen-rsa"
-	FILE_FLAG                  = "file"
-	INSTALL_SERVICE_FLAG       = "install"
-	UNINSTALL_SERVICE_FLAG     = "uninstall"
+	API_PORT_FLAG               = "port"
+	UPDATE_ROOT_PASSWORD_FLAG   = "update-root-pass"
+	GENERATE_SECURITY_KEY_FLAG  = "gen-rsa"
+	FILE_FLAG                   = "file"
+	INSTALL_SERVICE_FLAG        = "install"
+	UNINSTALL_SERVICE_FLAG      = "uninstall"
+	TEST_FLAG                   = "test"
+	TEST_CATALOG_PROVIDERS_FLAG = "catalog-providers"
 )
 
 const (

src/controllers/authorization.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/Parallels/pd-api-service/basecontext"
 	"github.com/Parallels/pd-api-service/config"
-	"github.com/Parallels/pd-api-service/constants"
 	"github.com/Parallels/pd-api-service/data"
 	"github.com/Parallels/pd-api-service/helpers"
 	"github.com/Parallels/pd-api-service/models"
@@ -99,7 +98,7 @@ func GetTokenHandler() restapi.ControllerHandler {
 			claims = append(claims, claim.Name)
 		}
 
-		expiresAt := time.Now().Add(time.Minute * constants.TOKEN_DURATION_MINUTES).Unix()
+		expiresAt := time.Now().Add(time.Minute * time.Duration(cfg.GetTokenDurationMinutes())).Unix()
 		token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
 			"email":  request.Email,
 			"roles":  roles,

src/main.go

@@ -4,7 +4,6 @@ import (
 	"os"
 
 	"github.com/Parallels/pd-api-service/basecontext"
-	"github.com/Parallels/pd-api-service/catalog/tester"
 	"github.com/Parallels/pd-api-service/common"
 	"github.com/Parallels/pd-api-service/config"
 	"github.com/Parallels/pd-api-service/constants"
@@ -15,6 +14,7 @@ import (
 	"github.com/Parallels/pd-api-service/security"
 	"github.com/Parallels/pd-api-service/serviceprovider"
 	"github.com/Parallels/pd-api-service/startup"
+	"github.com/Parallels/pd-api-service/tests"
 
 	"github.com/cjlapao/common-go/helper"
 	"github.com/cjlapao/common-go/version"
@@ -66,6 +66,18 @@ func main() {
 	ctx := basecontext.NewRootBaseContext()
 	cfg := config.NewConfig()
 
+	// Checking if we just want to test
+	if helper.GetFlagSwitch(constants.TEST_FLAG, false) {
+		if helper.GetFlagSwitch(constants.TEST_CATALOG_PROVIDERS_FLAG, false) {
+			if err := tests.TestCatalogProviders(ctx); err != nil {
+				ctx.LogError(err.Error())
+				os.Exit(1)
+			}
+		}
+
+		os.Exit(0)
+	}
+
 	if helper.GetFlagSwitch(constants.GENERATE_SECURITY_KEY_FLAG, false) {
 		ctx.LogInfo("Generating security key")
 		filename := "private.key"
@@ -150,23 +162,6 @@ func main() {
 		}
 	}
 
-	if os.Getenv("ARTIFACTORY_TEST_CONNECTION") != "" {
-		ctx.LogInfo("Testing connection to Artifactory")
-		test := tester.NewTestProvider(ctx, os.Getenv("ARTIFACTORY_TEST_CONNECTION"))
-		err := test.Test()
-		if err != nil {
-			ctx.LogError(err.Error())
-			os.Exit(1)
-		} else {
-			ctx.LogInfo("Connection to Artifactory successful")
-			os.Exit(0)
-		}
-	}
-
-	// filePath := "/Users/cjlapao/VagrantTest/Vagrantfile"
-	// vs := serviceprovider.Get().VagrantService
-	// vs.UpdateVagrantFile(ctx, filePath)
-
 	// Serve the API
 	for {
 		listener := startup.InitApi()