From 70f8b4018ce41feb8f52fd71d46df7e3978b0953 Mon Sep 17 00:00:00 2001 From: mircearoata Date: Sat, 6 May 2023 14:27:59 +0200 Subject: [PATCH 01/25] fix: update converter_windows to match linux counterpart --- util/converter/converter_windows.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/util/converter/converter_windows.go b/util/converter/converter_windows.go index 02a77a7b..9ab1eb71 100755 --- a/util/converter/converter_windows.go +++ b/util/converter/converter_windows.go @@ -2,28 +2,39 @@ package converter import ( "bytes" + "context" + "image" + "github.com/chai2010/webp" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "image" + + // GIF Support _ "image/gif" + // JPEG Support _ "image/jpeg" + // PNG Support _ "image/png" ) -func ConvertAnyImageToWebp(imageAsBytes []byte) ([]byte, error) { - imageData, _, err := image.Decode(bytes.NewReader(imageAsBytes)) - +func ConvertAnyImageToWebp(ctx context.Context, imageAsBytes []byte) ([]byte, error) { + imageData, imageType, err := image.Decode(bytes.NewReader(imageAsBytes)) if err != nil { - err := errors.Wrap(err, "error converting image to webp") - log.Error(err) - return nil, err + message := "error converting image to webp" + log.Err(err).Msg(message) + return nil, errors.Wrap(err, message) } result := bytes.NewBuffer(make([]byte, 0)) + if imageType == "gif" { + message := "converting gif to webp not supported on windows" + log.Err(err).Msg(message) + return nil, errors.Wrap(err, message) + } + if err := webp.Encode(result, imageData, nil); err != nil { - return nil, err + return nil, errors.Wrap(err, "error converting image to webp") } return result.Bytes(), nil From ace91c3897edd3bda045ef979320df4040bb775e Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 12:10:44 +0200 Subject: [PATCH 02/25] feat: validate multi-target plugin, detect targets and plugin type --- validation/validation.go | 105 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/validation/validation.go b/validation/validation.go index 839317bb..19a8f577 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "path" "path/filepath" "strconv" "strings" @@ -20,11 +21,22 @@ import ( "github.com/xeipuuv/gojsonschema" ) +// TODO UE5: WindowsNoEditor -> Windows +var AllowedTargets = []string{"WindowsNoEditor", "WindowsServer", "LinuxServer"} + type ModObject struct { Path string `json:"path"` Type string `json:"type"` } +type ModType int + +const ( + DataJson ModType = iota + UEPlugin = 1 + MultiTargetUEPlugin = 2 +) + type ModInfo struct { Dependencies map[string]string `json:"dependencies"` OptionalDependencies map[string]string `json:"optional_dependencies"` @@ -36,6 +48,8 @@ type ModInfo struct { Objects []ModObject `json:"objects"` Metadata []map[string]map[string][]interface{} `json:"-"` Size int64 `json:"-"` + Targets []string `json:"-"` + Type ModType `json:"-"` } var ( @@ -78,7 +92,7 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal dataFile = v break } - if v.Name == "WindowsNoEditor/"+modReference+".uplugin" { + if v.Name == modReference+".uplugin" { uPlugin = v break } @@ -101,7 +115,16 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal } if modInfo == nil { - return nil, errors.New("missing WindowsNoEditor/" + modReference + ".uplugin or data.json") + + // Neither data.json nor .uplugin found, try multi-target .uplugin + modInfo, err = validateMultiTargetPlugin(archive, withValidation, modReference) + if err != nil { + return nil, err + } + } + + if modInfo == nil { + return nil, errors.New("missing " + modReference + ".uplugin or data.json") } if withMetadata { @@ -244,6 +267,8 @@ func validateDataJSON(archive *zip.Reader, dataFile *zip.File, withValidation bo } } + modInfo.Type = DataJson + return &modInfo, nil } @@ -357,5 +382,81 @@ func validateUPluginJSON(archive *zip.Reader, uPluginFile *zip.File, withValidat return nil, errors.New(uPluginFile.Name + " doesn't contain SML as a dependency.") } + modInfo.Type = UEPlugin + return &modInfo, nil } + +func validateMultiTargetPlugin(archive *zip.Reader, withValidation bool, modReference string) (*ModInfo, error) { + var targets []string + var uPluginFiles []*zip.File + for _, file := range archive.File { + if path.Base(file.Name) == modReference+".uplugin" && path.Dir(file.Name) != "." { + targets = append(targets, path.Dir(file.Name)) + uPluginFiles = append(uPluginFiles, file) + } + } + + if withValidation { + for _, target := range targets { + found := false + for _, allowedTarget := range AllowedTargets { + if target == allowedTarget { + found = true + break + } + } + if !found { + return nil, errors.New("multi-target plugin contains invalid target: " + target) + } + } + + for _, file := range archive.File { + found := false + for _, target := range targets { + if strings.HasPrefix(file.Name, target+"/") { + found = true + break + } + } + if !found { + return nil, errors.New("multi-target plugin contains file outside of target directories: " + file.Name) + } + } + } + + if len(uPluginFiles) == 0 { + return nil, errors.New("multi-target plugin doesn't contain any .uplugin files") + } + + if withValidation { + var lastData []byte + for _, uPluginFile := range uPluginFiles { + file, err := uPluginFile.Open() + if err != nil { + return nil, errors.Wrap(err, "failed to open .uplugin file") + } + data, err := io.ReadAll(file) + file.Close() + if err != nil { + return nil, errors.Wrap(err, "failed to read .uplugin file") + } + + if lastData != nil && !bytes.Equal(lastData, data) { + return nil, errors.New("multi-target plugin contains different .uplugin files") + } + lastData = data + } + } + + // All the .uplugin files should be the same at this point (assuming validation is enabled) + modInfo, err := validateUPluginJSON(archive, uPluginFiles[0], withValidation, modReference) + if err != nil { + return nil, errors.Wrap(err, "failed to validate multi-target plugin") + } + + modInfo.Targets = targets + modInfo.Type = MultiTargetUEPlugin + + return modInfo, nil +} From 4bdfce8993626ca4c07e826b0477c10577230d8f Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 12:35:07 +0200 Subject: [PATCH 03/25] fix: keep the original uploaded file, remove "combined" arch --- gql/resolver_versions.go | 2 -- gql/versions.go | 6 ++-- .../consumer_scan_mod_on_virus_total.go | 9 +----- storage/storage.go | 28 +++++++++---------- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index f551b381..4fe464df 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -184,8 +184,6 @@ func (r *mutationResolver) ApproveVersion(ctx context.Context, versionID string) postgres.Save(newCtx, &mod) go integrations.NewVersion(util.ReWrapCtx(ctx), dbVersion) - go storage.DeleteModArch(ctx, dbVersion.ModID, mod.Name, versionID, "Combined") - go storage.DeleteModArch(ctx, dbVersion.ModID, mod.Name, dbVersion.Version, "Combined") return true, nil } diff --git a/gql/versions.go b/gql/versions.go index 97e2c0ae..05b0cd89 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -67,6 +67,8 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI ModID: mod.ID, Stability: string(version.Stability), ModReference: &modInfo.ModReference, + Size: &modInfo.Size, + Hash: &modInfo.Hash, VersionMajor: &versionMajor, VersionMinor: &versionMinor, VersionPatch: &versionPatch, @@ -162,8 +164,6 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &dbVersion) postgres.Save(ctx, &mod) - storage.DeleteVersion(ctx, mod.ID, mod.Name, versionID) - if autoApproved { mod := postgres.GetModByID(ctx, dbVersion.ModID) now := time.Now() @@ -171,8 +171,6 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &mod) go integrations.NewVersion(util.ReWrapCtx(ctx), dbVersion) - storage.DeleteModArch(ctx, mod.ID, mod.Name, versionID, "Combined") - storage.DeleteModArch(ctx, mod.ID, mod.Name, dbVersion.Version, "Combined") } else { l.Info().Msg("Submitting version job for virus scan") jobs.SubmitJobScanModOnVirusTotalTask(ctx, mod.ID, dbVersion.ID, true) diff --git a/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go b/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go index b3d7a76c..32ca8213 100644 --- a/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go +++ b/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go @@ -38,10 +38,7 @@ func ScanModOnVirusTotalConsumer(ctx context.Context, payload []byte) error { log.Info().Msgf("starting virus scan of mod %s version %s", task.ModID, task.VersionID) version := postgres.GetVersion(ctx, task.VersionID) - mod := postgres.GetModByID(ctx, version.ModID) - - modArch := postgres.GetModArchByPlatform(ctx, task.VersionID, "Combined") - link := storage.GenerateDownloadLink(modArch.Key) + link := storage.GenerateDownloadLink(version.Key) response, _ := http.Get(link) @@ -92,9 +89,5 @@ func ScanModOnVirusTotalConsumer(ctx context.Context, payload []byte) error { go integrations.NewVersion(util.ReWrapCtx(ctx), version) } - go storage.DeleteModArch(ctx, mod.ID, mod.Name, version.ID, "Combined") - go storage.DeleteModArch(ctx, mod.ID, mod.Name, version.Version, "Combined") - go postgres.Delete(ctx, modArch) - return nil } diff --git a/storage/storage.go b/storage/storage.go index af214dd5..0a460936 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -402,31 +402,29 @@ func SeparateMod(ctx context.Context, body []byte, modID, name string, versionID return false } - ModPlatforms := []string{"Combined", "WindowsNoEditor", "WindowsServer", "LinuxServer"} + ModPlatforms := []string{"WindowsNoEditor", "WindowsServer", "LinuxServer"} cleanName := cleanModName(name) bufPlatform := bytes.NewBuffer(body) for _, ModPlatform := range ModPlatforms { - if ModPlatform != "Combined" { - bufPlatform = new(bytes.Buffer) - zipWriter := zip.NewWriter(bufPlatform) + bufPlatform = new(bytes.Buffer) + zipWriter := zip.NewWriter(bufPlatform) - for _, file := range zipReader.File { - if strings.HasPrefix(file.Name, ".pdb") || strings.HasPrefix(file.Name, ".debug") || !strings.Contains(file.Name, ModPlatform) { - continue - } + for _, file := range zipReader.File { + if strings.HasPrefix(file.Name, ".pdb") || strings.HasPrefix(file.Name, ".debug") || !strings.Contains(file.Name, ModPlatform) { + continue + } - err = WriteZipFile(ctx, file, ModPlatform, zipWriter) + err = WriteZipFile(ctx, file, ModPlatform, zipWriter) - if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to write zip to " + ModPlatform + " smod") - return false - } + if err != nil { + log.Ctx(ctx).Err(err).Msg("Failed to write zip to " + ModPlatform + " smod") + return false } - - zipWriter.Close() } + zipWriter.Close() + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+ModPlatform+"-"+modVersion) err = WriteModArch(ctx, key, versionID, ModPlatform, bufPlatform) From 022aeed03f8e6f56c32c93801053ee341c289d04 Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 12:54:31 +0200 Subject: [PATCH 04/25] refactor: separate ModArch creation from storage --- gql/versions.go | 108 +++++++++++++++++++++++++----------- storage/storage.go | 135 +++++++++++++-------------------------------- 2 files changed, 115 insertions(+), 128 deletions(-) diff --git a/gql/versions.go b/gql/versions.go index 05b0cd89..ca4a3c48 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -122,45 +122,50 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &dbVersion) } - separated := storage.SeparateMod(ctx, fileData, mod.ID, mod.Name, dbVersion.ID, modInfo.Version) - - if !separated { - for modID, condition := range modInfo.Dependencies { - dependency := postgres.VersionDependency{ - VersionID: dbVersion.ID, - ModID: modID, - Condition: condition, - Optional: false, - } - - postgres.DeleteForced(ctx, &dependency) - } + // TODO: Should legacy plugins be supported? + var modArchs []*postgres.ModArch - for modID, condition := range modInfo.OptionalDependencies { - dependency := postgres.VersionDependency{ - VersionID: dbVersion.ID, - ModID: modID, - Condition: condition, - Optional: true, - } + for _, target := range modInfo.Targets { + dbModArch, _ := postgres.CreateModArch(ctx, &postgres.ModArch{ + ModVersionID: dbVersion.ID, + Platform: target, + }) - postgres.DeleteForced(ctx, &dependency) - } + modArchs = append(modArchs, dbModArch) + } - postgres.DeleteForced(ctx, &dbVersion) - storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + separateSuccess := true + for _, modArch := range modArchs { + log.Info().Str("modArch", modArch.Platform).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") + success, key, hash, size := storage.SeparateModPlatform(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, modArch.Platform) - for _, dbModArch := range dbVersion.Arch { - postgres.DeleteForced(ctx, &dbModArch) + if !success { + separateSuccess = false + break } - return nil, errors.New("failed to upload mod") + + modArch.Key = key + modArch.Hash = hash + modArch.Size = size + + postgres.Save(ctx, modArch) + } + + if !separateSuccess { + removeMod(ctx, modInfo, mod, dbVersion) + + return nil, errors.New("failed to separate mod") } - dbModArch := postgres.GetModArchByPlatform(ctx, dbVersion.ID, "WindowsNoEditor") + success, key := storage.RenameVersion(ctx, mod.ID, mod.Name, versionID, modInfo.Version) - dbVersion.Key = dbModArch.Key - dbVersion.Hash = &dbModArch.Hash - dbVersion.Size = &dbModArch.Size + if !success { + removeMod(ctx, modInfo, mod, dbVersion) + + return nil, errors.New("failed to upload mod") + } + + dbVersion.Key = key postgres.Save(ctx, &dbVersion) postgres.Save(ctx, &mod) @@ -181,3 +186,44 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI Version: DBVersionToGenerated(dbVersion), }, nil } + +func removeMod(ctx context.Context, modInfo *validation.ModInfo, mod *postgres.Mod, dbVersion *postgres.Version) { + for modID, condition := range modInfo.Dependencies { + dependency := postgres.VersionDependency{ + VersionID: dbVersion.ID, + ModID: modID, + Condition: condition, + Optional: false, + } + + postgres.DeleteForced(ctx, &dependency) + } + + for modID, condition := range modInfo.OptionalDependencies { + dependency := postgres.VersionDependency{ + VersionID: dbVersion.ID, + ModID: modID, + Condition: condition, + Optional: true, + } + + postgres.DeleteForced(ctx, &dependency) + } + + for _, target := range modInfo.Targets { + // TODO: ModArch should have VersionID and Platform as primary key + dbModArch := postgres.ModArch{ + ModVersionID: dbVersion.ID, + Platform: target, + } + + postgres.DeleteForced(ctx, &dbModArch) + } + + postgres.DeleteForced(ctx, &dbVersion) + + storage.DeleteMod(ctx, mod.ID, mod.Name, dbVersion.ID) + for _, target := range modInfo.Targets { + storage.DeleteModPlatform(ctx, mod.ID, mod.Name, dbVersion.ID, target) + } +} diff --git a/storage/storage.go b/storage/storage.go index 0a460936..a77c27a5 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -14,8 +14,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" - - "github.com/satisfactorymodding/smr-api/db/postgres" ) type Storage interface { @@ -273,7 +271,7 @@ func RenameVersion(ctx context.Context, modID string, name string, versionID str return true, fmt.Sprintf("/mods/%s/%s.smod", modID, EncodeName(cleanName)+"-"+version) } -func DeleteVersion(ctx context.Context, modID string, name string, versionID string) bool { +func DeleteMod(ctx context.Context, modID string, name string, versionID string) bool { if storage == nil { return false } @@ -291,35 +289,7 @@ func DeleteVersion(ctx context.Context, modID string, name string, versionID str return true } -func DeleteMod(ctx context.Context, modID string, name string, versionID string) bool { - if storage == nil { - return false - } - - cleanName := cleanModName(name) - - query := postgres.GetModVersion(ctx, modID, versionID) - - if query != nil && len(query.Arch) != 0 { - for _, link := range query.Arch { - if success := DeleteModArch(ctx, modID, cleanName, versionID, link.Platform); !success { - return false - } - } - } else { - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+versionID) - - log.Info().Str("key", key).Msg("deleting mod") - if err := storage.Delete(key); err != nil { - log.Ctx(ctx).Err(err).Msg("failed to delete version") - return false - } - } - - return true -} - -func DeleteModArch(ctx context.Context, modID string, name string, versionID string, platform string) bool { +func DeleteModPlatform(ctx context.Context, modID string, name string, versionID string, platform string) bool { if storage == nil { return false } @@ -327,9 +297,9 @@ func DeleteModArch(ctx context.Context, modID string, name string, versionID str cleanName := cleanModName(name) key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+platform+"-"+versionID) - log.Info().Str("key", key).Msg("deleting mod arch") + log.Info().Str("key", key).Msg("deleting mod platform") if err := storage.Delete(key); err != nil { - log.Ctx(ctx).Err(err).Msg("failed to delete version link") + log.Err(err).Msg("failed to delete version link") return false } @@ -396,102 +366,73 @@ func EncodeName(name string) string { return result } -func SeparateMod(ctx context.Context, body []byte, modID, name string, versionID string, modVersion string) bool { +func SeparateModPlatform(ctx context.Context, body []byte, modID, name, modVersion, platform string) (bool, string, string, int64) { zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { - return false + return false, "", "", 0 } - ModPlatforms := []string{"WindowsNoEditor", "WindowsServer", "LinuxServer"} cleanName := cleanModName(name) - bufPlatform := bytes.NewBuffer(body) - - for _, ModPlatform := range ModPlatforms { - bufPlatform = new(bytes.Buffer) - zipWriter := zip.NewWriter(bufPlatform) - - for _, file := range zipReader.File { - if strings.HasPrefix(file.Name, ".pdb") || strings.HasPrefix(file.Name, ".debug") || !strings.Contains(file.Name, ModPlatform) { - continue - } - err = WriteZipFile(ctx, file, ModPlatform, zipWriter) + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) - if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to write zip to " + ModPlatform + " smod") - return false - } + for _, file := range zipReader.File { + if !strings.HasPrefix(file.Name, platform+"/") && file.Name != platform+"/" { + continue } - zipWriter.Close() - - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+ModPlatform+"-"+modVersion) + err = copyModFileToArchZip(file, zipWriter, strings.TrimPrefix(file.Name, platform+"/")) - err = WriteModArch(ctx, key, versionID, ModPlatform, bufPlatform) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to save " + ModPlatform + " smod") - return false + log.Err(err).Msg("failed to add file to " + platform + " archive") + return false, "", "", 0 } + } - return true -} + zipWriter.Close() -func WriteZipFile(ctx context.Context, file *zip.File, platform string, zipWriter *zip.Writer) error { - fileName := strings.ReplaceAll(file.Name, platform+"/", "") - zipFile, err := zipWriter.Create(fileName) - if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to create smod file for " + platform) - return errors.Wrap(err, "Failed to open smod file for "+platform) - } + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+platform+"-"+modVersion) - rawFile, err := file.Open() + _, err = storage.Put(ctx, key, bytes.NewReader(buf.Bytes())) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to open smod file for " + platform) - return errors.Wrap(err, "Failed to open smod file for "+platform) + log.Err(err).Msg("failed to save " + platform + " archive") + return false, "", "", 0 } - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(rawFile) + hash := sha256.New() + hash.Write(buf.Bytes()) - if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to read from buffer for " + platform) - return errors.Wrap(err, "Failed to read from buffer for "+platform) - } + return true, key, hex.EncodeToString(hash.Sum(nil)), int64(buf.Len()) +} - _, err = zipFile.Write(buf.Bytes()) +func copyModFileToArchZip(file *zip.File, zipWriter *zip.Writer, newName string) error { + fileHeader := file.FileHeader + fileHeader.Name = newName + zipFile, err := zipWriter.CreateHeader(&fileHeader) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to write to smod file: " + platform) - return errors.Wrap(err, "Failed to write smod file for "+platform) + return errors.Wrap(err, "failed to create file") } - return nil -} - -func WriteModArch(ctx context.Context, key string, versionID string, platform string, buffer *bytes.Buffer) error { - _, err := storage.Put(ctx, key, bytes.NewReader(buffer.Bytes())) + rawFile, err := file.Open() if err != nil { - log.Ctx(ctx).Err(err).Msg("failed to write smod: " + key) - return errors.Wrap(err, "Failed to load smod:"+key) + return errors.Wrap(err, "failed to open file") } + defer rawFile.Close() - hash := sha256.New() - hash.Write(buffer.Bytes()) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(rawFile) - dbModArch := &postgres.ModArch{ - ModVersionID: versionID, - Platform: platform, - Key: key, - Hash: hex.EncodeToString(hash.Sum(nil)), - Size: int64(len(buffer.Bytes())), + if err != nil { + return errors.Wrap(err, "failed to read file") } - _, err = postgres.CreateModArch(ctx, dbModArch) + _, err = zipFile.Write(buf.Bytes()) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to create ModArch: " + versionID + "-" + platform) - return errors.Wrap(err, "Failed to create ModArch: "+versionID+"-"+platform) + return errors.Wrap(err, "failed to write file") } return nil From 40ca1d08bed85f8210dae19e9ba644ff39cd09db Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 18:15:53 +0200 Subject: [PATCH 05/25] chore: rename all "arch/platform" consistently, remove ID from VersionTargets The "arch/platform" entities are now called targets The "platform" field is now targetName VersionTarget and SMLVersionTarget have VersionID and TargetName as their primary keys, rather than a separate ID Removed unnecessary and unused GQL endpoints related to Targets --- dataloader/loaders.go | 4 +- db/postgres/mod.go | 8 +- db/postgres/mod_archs.go | 122 ------------------ db/postgres/postgres_types.go | 34 ++--- db/postgres/sml_archs.go | 99 -------------- db/postgres/sml_version.go | 25 ++-- db/postgres/version.go | 44 +++++-- gql/gql_types.go | 51 ++++---- gql/resolver.go | 4 +- gql/resolver_mod_archs.go | 34 ----- gql/resolver_sml_archs.go | 101 --------------- gql/resolver_sml_versions.go | 75 ++++------- gql/resolver_versions.go | 6 + gql/versions.go | 39 +++--- gqlgen.yml | 13 +- .../sql/000022_update_mod_targets.down.sql | 62 +++++++++ .../sql/000022_update_mod_targets.up.sql | 29 +++++ models/filters.go | 82 ------------ nodes/mod.go | 20 +-- nodes/routes.go | 4 +- nodes/version.go | 22 ++-- schemas/mod_archs.graphql | 38 ------ schemas/sml_archs.graphql | 46 ------- schemas/sml_version.graphql | 22 +++- schemas/version.graphql | 10 +- storage/storage.go | 18 +-- 26 files changed, 294 insertions(+), 718 deletions(-) delete mode 100644 db/postgres/mod_archs.go delete mode 100644 db/postgres/sml_archs.go delete mode 100644 gql/resolver_mod_archs.go delete mode 100644 gql/resolver_sml_archs.go create mode 100644 migrations/sql/000022_update_mod_targets.down.sql create mode 100644 migrations/sql/000022_update_mod_targets.up.sql delete mode 100644 schemas/mod_archs.graphql delete mode 100644 schemas/sml_archs.graphql diff --git a/dataloader/loaders.go b/dataloader/loaders.go index 8fc03032..861ed84c 100644 --- a/dataloader/loaders.go +++ b/dataloader/loaders.go @@ -109,7 +109,7 @@ func Middleware() func(handlerFunc echo.HandlerFunc) echo.HandlerFunc { var entities []postgres.Version reqCtx := c.Request().Context() - postgres.DBCtx(reqCtx).Preload("Arch").Where("approved = ? AND denied = ? AND mod_id IN ?", true, false, fetchIds).Order("created_at desc").Find(&entities) + postgres.DBCtx(reqCtx).Preload("Targets").Where("approved = ? AND denied = ? AND mod_id IN ?", true, false, fetchIds).Order("created_at desc").Find(&entities) for _, entity := range entities { byID[entity.ModID] = append(byID[entity.ModID], entity) @@ -145,7 +145,7 @@ func Middleware() func(handlerFunc echo.HandlerFunc) echo.HandlerFunc { var entities []postgres.Version reqCtx := c.Request().Context() - postgres.DBCtx(reqCtx).Preload("Arch").Select( + postgres.DBCtx(reqCtx).Preload("Targets").Select( "id", "created_at", "updated_at", diff --git a/db/postgres/mod.go b/db/postgres/mod.go index 1c92c555..438ada7d 100644 --- a/db/postgres/mod.go +++ b/db/postgres/mod.go @@ -26,7 +26,7 @@ func GetModByID(ctx context.Context, modID string) *Mod { func GetModByIDNoCache(ctx context.Context, modID string) *Mod { var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "id = ?", modID) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "id = ?", modID) if mod.ID == "" { return nil @@ -44,7 +44,7 @@ func GetModByReference(ctx context.Context, modReference string) *Mod { } var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "mod_reference = ?", modReference) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "mod_reference = ?", modReference) if mod.ID == "" { return nil @@ -213,7 +213,7 @@ func NewModQuery(ctx context.Context, filter *models.ModFilter, unapproved bool, } query = query.Where("approved = ? AND denied = ?", !unapproved, false) - query = query.Preload("Tags").Preload("Versions.Arch") + query = query.Preload("Tags").Preload("Versions.Targets") if filter != nil { if filter.Search != nil && *filter.Search != "" { cleanSearch := strings.ReplaceAll(strings.TrimSpace(*filter.Search), " ", " & ") @@ -270,7 +270,7 @@ func GetModByIDOrReference(ctx context.Context, modIDOrReference string) *Mod { } var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "mod_reference = ? OR id = ?", modIDOrReference, modIDOrReference) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "mod_reference = ? OR id = ?", modIDOrReference, modIDOrReference) if mod.ID == "" { return nil diff --git a/db/postgres/mod_archs.go b/db/postgres/mod_archs.go deleted file mode 100644 index aa6bde95..00000000 --- a/db/postgres/mod_archs.go +++ /dev/null @@ -1,122 +0,0 @@ -package postgres - -import ( - "context" - "strings" - - "github.com/patrickmn/go-cache" - - "github.com/satisfactorymodding/smr-api/models" - "github.com/satisfactorymodding/smr-api/util" -) - -func CreateModArch(ctx context.Context, modArch *ModArch) (*ModArch, error) { - modArch.ID = util.GenerateUniqueID() - DBCtx(ctx).Create(&modArch) - return modArch, nil -} - -func GetModArch(ctx context.Context, modArchID string) *ModArch { - cacheKey := "GetModArch_" + modArchID - - if modArch, ok := dbCache.Get(cacheKey); ok { - return modArch.(*ModArch) - } - - var modArch ModArch - DBCtx(ctx).Find(&modArch, "id = ?", modArchID) - - if modArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &modArch, cache.DefaultExpiration) - - return &modArch -} - -func GetModArchs(ctx context.Context, filter *models.ModArchFilter) []ModArch { - var modArchs []ModArch - query := DBCtx(ctx) - - if filter != nil { - query = query.Limit(*filter.Limit). - Offset(*filter.Offset). - Order(string(*filter.OrderBy) + " " + string(*filter.Order)) - - if filter.Search != nil && *filter.Search != "" { - query = query.Where("to_tsvector(name) @@ to_tsquery(?)", strings.ReplaceAll(*filter.Search, " ", " & ")) - } - } - - query.Find(&modArchs) - return modArchs -} - -func GetVersionModArchs(ctx context.Context, versionID string) []ModArch { - var modArchs []ModArch - query := DBCtx(ctx).Find(&modArchs, "mod_version_arch_id = ?", versionID) - - query.Find(&modArchs) - return modArchs -} - -func GetModArchByID(ctx context.Context, modArchID string) *ModArch { - cacheKey := "GetModArch_" + modArchID - - if modArch, ok := dbCache.Get(cacheKey); ok { - return modArch.(*ModArch) - } - - var modArch ModArch - DBCtx(ctx).Find(&modArch, "id = ?", modArchID) - - if modArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &modArch, cache.DefaultExpiration) - - return &modArch -} - -func GetModArchsByID(ctx context.Context, modArchIds []string) []ModArch { - var modArchs []ModArch - - DBCtx(ctx).Find(&modArchs, "id in (?)", modArchIds) - - if len(modArchIds) != len(modArchs) { - return nil - } - - return modArchs -} - -func GetModArchByPlatform(ctx context.Context, versionID string, platform string) *ModArch { - cacheKey := "GetModArch_" + versionID + "_" + platform - if modplatform, ok := dbCache.Get(cacheKey); ok { - return modplatform.(*ModArch) - } - - var modplatform ModArch - DBCtx(ctx).First(&modplatform, "mod_version_arch_id = ? AND platform = ?", versionID, platform) - - if modplatform.ModVersionID == "" { - return nil - } - - dbCache.Set(cacheKey, &modplatform, cache.DefaultExpiration) - - return &modplatform -} - -func GetModArchDownload(ctx context.Context, versionID string, platform string) string { - var modPlatform ModArch - DBCtx(ctx).First(&modPlatform, "mod_version_arch_id = ? AND platform = ?", versionID, platform) - - if modPlatform.ModVersionID == "" { - return "" - } - - return modPlatform.Key -} diff --git a/db/postgres/postgres_types.go b/db/postgres/postgres_types.go index 84e0622f..8f865852 100644 --- a/db/postgres/postgres_types.go +++ b/db/postgres/postgres_types.go @@ -87,7 +87,7 @@ type Version struct { SMLVersion string `gorm:"type:varchar(16)"` Version string `gorm:"type:varchar(16)"` ModID string - Arch []ModArch `gorm:"foreignKey:ModVersionID;preload:true"` + Targets []VersionTarget `gorm:"foreignKey:VersionID;preload:true"` Hotness uint Downloads uint Denied bool `gorm:"default:false;not null"` @@ -120,7 +120,7 @@ type SMLVersion struct { Stability string `sql:"type:version_stability"` Link string Changelog string - Arch []SMLArch `gorm:"foreignKey:SMLVersionID;preload:true"` + Targets []SMLVersionTarget `gorm:"foreignKey:VersionID;preload:true"` SatisfactoryVersion int } @@ -179,26 +179,16 @@ type Compatibility struct { Note string } -type ModArch struct { - ID string `gorm:"primary_key;type:varchar(16)"` - ModVersionID string `gorm:"column:mod_version_arch_id"` - Platform string - Key string - Hash string - Size int64 -} - -func (ModArch) TableName() string { - return "mod_archs" -} - -type SMLArch struct { - ID string `gorm:"primary_key;type:varchar(14)"` - SMLVersionID string `gorm:"column:sml_version_arch_id"` - Platform string - Link string +type VersionTarget struct { + VersionID string `gorm:"primary_key;type:varchar(14)"` + TargetName string `gorm:"primary_key;type:varchar(16)"` + Key string + Hash string + Size int64 } -func (SMLArch) TableName() string { - return "sml_archs" +type SMLVersionTarget struct { + VersionID string `gorm:"primary_key;type:varchar(14)"` + TargetName string `gorm:"primary_key;type:varchar(16)"` + Link string } diff --git a/db/postgres/sml_archs.go b/db/postgres/sml_archs.go deleted file mode 100644 index f8b0108c..00000000 --- a/db/postgres/sml_archs.go +++ /dev/null @@ -1,99 +0,0 @@ -package postgres - -import ( - "context" - "strings" - - "github.com/patrickmn/go-cache" - - "github.com/satisfactorymodding/smr-api/models" - "github.com/satisfactorymodding/smr-api/util" -) - -func CreateSMLArch(ctx context.Context, smlArch *SMLArch) (*SMLArch, error) { - smlArch.ID = util.GenerateUniqueID() - - DBCtx(ctx).Create(&smlArch) - - return smlArch, nil -} - -func GetSMLArch(ctx context.Context, smlLinkID string) *SMLArch { - cacheKey := "GetSMLArch_" + smlLinkID - - if smlArch, ok := dbCache.Get(cacheKey); ok { - return smlArch.(*SMLArch) - } - - var smlArch SMLArch - DBCtx(ctx).Find(&smlArch, "id = ?", smlLinkID) - - if smlArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &smlArch, cache.DefaultExpiration) - - return &smlArch -} - -func GetSMLArchs(ctx context.Context, filter *models.SMLArchFilter) []SMLArch { - var smlLinks []SMLArch - query := DBCtx(ctx) - - if filter != nil { - query = query.Limit(*filter.Limit). - Offset(*filter.Offset). - Order(string(*filter.OrderBy) + " " + string(*filter.Order)) - - if filter.Search != nil && *filter.Search != "" { - query = query.Where("to_tsvector(name) @@ to_tsquery(?)", strings.ReplaceAll(*filter.Search, " ", " & ")) - } - } - - query.Find(&smlLinks) - return smlLinks -} - -func GetSMLArchByID(ctx context.Context, smlLinkID string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "id in ?", smlLinkID) - - if len(smlArchs) != 0 { - return nil - } - - return smlArchs -} - -func GetSMLArchsByID(ctx context.Context, smlArchIds []string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "id in (?)", smlArchIds) - - if len(smlArchIds) != len(smlArchs) { - return nil - } - - return smlArchs -} - -func GetSMLArchBySMLID(ctx context.Context, smlVersionID string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "sml_version_arch_id = ?", smlVersionID) - - return smlArchs -} - -func GetSMLArchDownload(ctx context.Context, smlVersionID string, platform string) string { - var smlPlatform SMLArch - DBCtx(ctx).First(&smlPlatform, "sml_version_arch_id = ? AND platform = ?", smlVersionID, platform) - - if smlPlatform.ID == "" { - return "" - } - - return smlPlatform.Link -} diff --git a/db/postgres/sml_version.go b/db/postgres/sml_version.go index 762e923d..46f6e213 100644 --- a/db/postgres/sml_version.go +++ b/db/postgres/sml_version.go @@ -13,21 +13,12 @@ func CreateSMLVersion(ctx context.Context, smlVersion *SMLVersion) (*SMLVersion, DBCtx(ctx).Create(&smlVersion) - for _, link := range smlVersion.Arch { - DBCtx(ctx).Create(&SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: smlVersion.ID, - Platform: link.Platform, - Link: link.Link, - }) - } - return smlVersion, nil } func GetSMLVersionByID(ctx context.Context, smlVersionID string) *SMLVersion { var smlVersion SMLVersion - DBCtx(ctx).Preload("Arch").Find(&smlVersion, "id in (?)", smlVersionID) + DBCtx(ctx).Preload("Targets").Find(&smlVersion, "id in (?)", smlVersionID) if smlVersion.ID == "" { return nil @@ -50,14 +41,14 @@ func GetSMLVersions(ctx context.Context, filter *models.SMLVersionFilter) []SMLV } } - query.Preload("Arch").Find(&smlVersions) + query.Preload("Targets").Find(&smlVersions) return smlVersions } func GetSMLVersionsByID(ctx context.Context, smlVersionIds []string) []SMLVersion { var smlVersions []SMLVersion - DBCtx(ctx).Preload("Arch").Find(&smlVersions, "id in (?)", smlVersionIds) + DBCtx(ctx).Preload("Targets").Find(&smlVersions, "id in (?)", smlVersionIds) if len(smlVersionIds) != len(smlVersions) { return nil @@ -83,9 +74,17 @@ func GetSMLVersionCount(ctx context.Context, filter *models.SMLVersionFilter) in func GetSMLLatestVersions(ctx context.Context) *[]SMLVersion { var smlVersions []SMLVersion - DBCtx(ctx).Preload("Arch").Select("distinct on (stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (stability) *"). Order("stability, created_at desc"). Find(&smlVersions) return &smlVersions } + +func GetSMLVersionTargets(ctx context.Context, smlVersionID string) []SMLVersionTarget { + var smlVersionTargets []SMLVersionTarget + + DBCtx(ctx).Find(&smlVersionTargets, "version_id = ?", smlVersionID) + + return smlVersionTargets +} diff --git a/db/postgres/version.go b/db/postgres/version.go index 54400553..041e347e 100644 --- a/db/postgres/version.go +++ b/db/postgres/version.go @@ -24,7 +24,7 @@ func GetVersionsByID(ctx context.Context, versionIds []string) []Version { } var versions []Version - DBCtx(ctx).Preload("Arch").Find(&versions, "id in (?)", versionIds) + DBCtx(ctx).Preload("Targets").Find(&versions, "id in (?)", versionIds) if len(versionIds) != len(versions) { return nil @@ -43,7 +43,7 @@ func GetModLatestVersions(ctx context.Context, modID string, unapproved bool) *[ var versions []Version - DBCtx(ctx).Preload("Arch").Select("distinct on (mod_id, stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (mod_id, stability) *"). Where("mod_id = ?", modID). Where("approved = ? AND denied = ?", !unapproved, false). Order("mod_id, stability, created_at desc"). @@ -62,7 +62,7 @@ func GetModsLatestVersions(ctx context.Context, modIds []string, unapproved bool var versions []Version - DBCtx(ctx).Preload("Arch").Select("distinct on (mod_id, stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (mod_id, stability) *"). Where("mod_id in (?)", modIds). Where("approved = ? AND denied = ?", !unapproved, false). Order("mod_id, stability, created_at desc"). @@ -80,7 +80,7 @@ func GetModVersions(ctx context.Context, modID string, limit int, offset int, or } var versions []Version - DBCtx(ctx).Preload("Arch").Limit(limit).Offset(offset).Order(orderBy+" "+order).Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) + DBCtx(ctx).Preload("Targets").Limit(limit).Offset(offset).Order(orderBy+" "+order).Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -98,7 +98,7 @@ func GetModVersionsNew(ctx context.Context, modID string, filter *models.Version } var versions []Version - query := DBCtx(ctx).Preload("Arch") + query := DBCtx(ctx).Preload("Targets") if filter != nil { query = query.Limit(*filter.Limit). @@ -106,7 +106,7 @@ func GetModVersionsNew(ctx context.Context, modID string, filter *models.Version Order(string(*filter.OrderBy) + " " + string(*filter.Order)) } - query.Preload("Arch").Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) + query.Preload("Targets").Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) if cacheKey != "" { dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -122,7 +122,7 @@ func GetModVersion(ctx context.Context, modID string, versionID string) *Version } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "mod_id = ? AND id = ?", modID, versionID) + DBCtx(ctx).Preload("Targets").First(&version, "mod_id = ? AND id = ?", modID, versionID) if version.ID == "" { return nil @@ -140,7 +140,7 @@ func GetModVersionByName(ctx context.Context, modID string, versionName string) } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "mod_id = ? AND version = ?", modID, versionName) + DBCtx(ctx).Preload("Targets").First(&version, "mod_id = ? AND version = ?", modID, versionName) if version.ID == "" { return nil @@ -186,7 +186,7 @@ func GetVersion(ctx context.Context, versionID string) *Version { } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "id = ?", versionID) + DBCtx(ctx).Preload("Targets").First(&version, "id = ?", versionID) if version.ID == "" { return nil @@ -208,7 +208,7 @@ func GetVersionsNew(ctx context.Context, filter *models.VersionFilter, unapprove } var versions []Version - query := DBCtx(ctx).Preload("Arch").Where("approved = ? AND denied = ?", !unapproved, false) + query := DBCtx(ctx).Preload("Targets").Where("approved = ? AND denied = ?", !unapproved, false) if filter != nil { query = query.Limit(*filter.Limit). @@ -224,7 +224,7 @@ func GetVersionsNew(ctx context.Context, filter *models.VersionFilter, unapprove } } - query.Preload("Arch").Find(&versions) + query.Preload("Targets").Find(&versions) if cacheKey != "" { dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -261,6 +261,24 @@ func GetVersionCountNew(ctx context.Context, filter *models.VersionFilter, unapp return versionCount } +func GetVersionTarget(ctx context.Context, versionID string, target string) *VersionTarget { + cacheKey := "GetVersionTarget_" + versionID + "_" + target + if versionTarget, ok := dbCache.Get(cacheKey); ok { + return versionTarget.(*VersionTarget) + } + + var versionTarget VersionTarget + DBCtx(ctx).First(&versionTarget, "version_id = ? AND target = ?", versionID, target) + + if versionTarget.VersionID == "" { + return nil + } + + dbCache.Set(cacheKey, &versionTarget, cache.DefaultExpiration) + + return &versionTarget +} + func GetVersionDependencies(ctx context.Context, versionID string) []VersionDependency { var versionDependencies []VersionDependency DBCtx(ctx).Where("version_id = ?", versionID).Find(&versionDependencies) @@ -288,7 +306,7 @@ func GetModVersionsConstraint(ctx context.Context, modID string, constraint stri return nil } - query := DBCtx(ctx).Preload("Arch").Where("mod_id", modID) + query := DBCtx(ctx).Preload("Targets").Where("mod_id", modID) /* <=1.2.3 @@ -357,6 +375,6 @@ func GetModVersionsConstraint(ctx context.Context, modID string, constraint stri } var versions []Version - query.Preload("Arch").Find(&versions) + query.Preload("Targets").Find(&versions) return versions } diff --git a/gql/gql_types.go b/gql/gql_types.go index 7a7f8778..e3b126ae 100644 --- a/gql/gql_types.go +++ b/gql/gql_types.go @@ -85,7 +85,7 @@ func DBVersionToGenerated(version *postgres.Version) *generated.Version { Changelog: version.Changelog, Downloads: int(version.Downloads), Stability: generated.VersionStabilities(version.Stability), - Arch: DBModArchsToGeneratedSlice(version.Arch), + Targets: DBVersionTargetsToGeneratedSlice(version.Targets), Approved: version.Approved, UpdatedAt: version.UpdatedAt.Format(time.RFC3339Nano), CreatedAt: version.CreatedAt.Format(time.RFC3339Nano), @@ -134,7 +134,7 @@ func DBSMLVersionToGenerated(smlVersion *postgres.SMLVersion) *generated.SMLVers BootstrapVersion: smlVersion.BootstrapVersion, Stability: generated.VersionStabilities(smlVersion.Stability), Link: smlVersion.Link, - Arch: DBSMLArchsToGeneratedSlice(smlVersion.Arch), + Targets: DBSMLVersionTargetToGeneratedSlice(smlVersion.Targets), Changelog: smlVersion.Changelog, Date: smlVersion.Date.Format(time.RFC3339Nano), UpdatedAt: smlVersion.UpdatedAt.Format(time.RFC3339Nano), @@ -211,47 +211,46 @@ func DBTagsToGeneratedSlice(tags []postgres.Tag) []*generated.Tag { return converted } -func DBModArchToGenerated(modArch *postgres.ModArch) *generated.ModArch { - if modArch == nil { +func DBVersionTargetToGenerated(versionTarget *postgres.VersionTarget) *generated.VersionTarget { + if versionTarget == nil { return nil } - size := int(modArch.Size) + hash := versionTarget.Hash + size := int(versionTarget.Size) - return &generated.ModArch{ - ID: modArch.ID, - ModVersionID: modArch.ModVersionID, - Platform: modArch.Platform, - Hash: &modArch.Hash, - Size: &size, + return &generated.VersionTarget{ + VersionID: versionTarget.VersionID, + TargetName: versionTarget.TargetName, + Hash: &hash, + Size: &size, } } -func DBModArchsToGeneratedSlice(modArchs []postgres.ModArch) []*generated.ModArch { - converted := make([]*generated.ModArch, len(modArchs)) - for i, modArch := range modArchs { - converted[i] = DBModArchToGenerated(&modArch) +func DBVersionTargetsToGeneratedSlice(versionTargets []postgres.VersionTarget) []*generated.VersionTarget { + converted := make([]*generated.VersionTarget, len(versionTargets)) + for i, versionTarget := range versionTargets { + converted[i] = DBVersionTargetToGenerated(&versionTarget) } return converted } -func DBSMLArchToGenerated(smlArch *postgres.SMLArch) *generated.SMLArch { - if smlArch == nil { +func DBSMLVersionTargetToGenerated(smlVersionTarget *postgres.SMLVersionTarget) *generated.SMLVersionTarget { + if smlVersionTarget == nil { return nil } - return &generated.SMLArch{ - ID: smlArch.ID, - SMLVersionID: smlArch.SMLVersionID, - Platform: smlArch.Platform, - Link: smlArch.Link, + return &generated.SMLVersionTarget{ + VersionID: smlVersionTarget.VersionID, + TargetName: smlVersionTarget.TargetName, + Link: smlVersionTarget.Link, } } -func DBSMLArchsToGeneratedSlice(smlLinks []postgres.SMLArch) []*generated.SMLArch { - converted := make([]*generated.SMLArch, len(smlLinks)) - for i, smlArch := range smlLinks { - converted[i] = DBSMLArchToGenerated(&smlArch) +func DBSMLVersionTargetToGeneratedSlice(smlLinks []postgres.SMLVersionTarget) []*generated.SMLVersionTarget { + converted := make([]*generated.SMLVersionTarget, len(smlLinks)) + for i, smlVersionTarget := range smlLinks { + converted[i] = DBSMLVersionTargetToGenerated(&smlVersionTarget) } return converted } diff --git a/gql/resolver.go b/gql/resolver.go index 51c0c370..43547809 100755 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -26,8 +26,8 @@ func (r *Resolver) UserMod() generated.UserModResolver { return &userModResolver{r} } -func (r *Resolver) ModArch() generated.ModArchResolver { - return &modlinkResolver{r} +func (r *Resolver) VersionTarget() generated.VersionTargetResolver { + return &versionTargetResolver{r} } func (r *Resolver) Version() generated.VersionResolver { diff --git a/gql/resolver_mod_archs.go b/gql/resolver_mod_archs.go deleted file mode 100644 index 0181328e..00000000 --- a/gql/resolver_mod_archs.go +++ /dev/null @@ -1,34 +0,0 @@ -package gql - -import ( - "context" - - "github.com/satisfactorymodding/smr-api/db/postgres" - "github.com/satisfactorymodding/smr-api/generated" -) - -type modlinkResolver struct{ *Resolver } - -func (r *modlinkResolver) Asset(_ context.Context, obj *generated.ModArch) (string, error) { - return "/v1/version/" + obj.ModVersionID + "/" + obj.Platform + "/download", nil -} - -func (r *queryResolver) GetModArch(ctx context.Context, modArchID string) (*generated.ModArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getModArch") - defer wrapper.end() - modArch := postgres.GetModArch(newCtx, modArchID) - return DBModArchToGenerated(modArch), nil -} - -func (r *queryResolver) GetModArchs(ctx context.Context, filter map[string]interface{}) (*generated.GetModArchs, error) { - wrapper, _ := WrapQueryTrace(ctx, "getModArchs") - defer wrapper.end() - return &generated.GetModArchs{}, nil -} - -func (r *queryResolver) GetModArchByID(ctx context.Context, modArchID string) (*generated.ModArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getModArchByID") - defer wrapper.end() - modArch := postgres.GetModArchByID(newCtx, modArchID) - return DBModArchToGenerated(modArch), nil -} diff --git a/gql/resolver_sml_archs.go b/gql/resolver_sml_archs.go deleted file mode 100644 index be4e7763..00000000 --- a/gql/resolver_sml_archs.go +++ /dev/null @@ -1,101 +0,0 @@ -package gql - -import ( - "context" - - "github.com/pkg/errors" - "gopkg.in/go-playground/validator.v9" - - "github.com/satisfactorymodding/smr-api/db/postgres" - "github.com/satisfactorymodding/smr-api/generated" - "github.com/satisfactorymodding/smr-api/util" -) - -func (r *mutationResolver) CreateSMLArch(ctx context.Context, smlArch generated.NewSMLArch) (*generated.SMLArch, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "createSMLArch") - defer wrapper.end() - - val := ctx.Value(util.ContextValidator{}).(*validator.Validate) - if err := val.Struct(&smlArch); err != nil { - return nil, errors.Wrap(err, "validation failed") - } - - dbSMLArchs := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArchs) - if err != nil { - return nil, err - } - - return DBSMLArchToGenerated(resultSMLArch), nil -} - -func (r *mutationResolver) DeleteSMLArch(ctx context.Context, linksID string) (bool, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "deleteSMLArch") - defer wrapper.end() - - dbSMLArch := postgres.GetSMLArchByID(newCtx, linksID) - - if dbSMLArch == nil { - return false, errors.New("SML Link not found") - } - - postgres.Delete(newCtx, &dbSMLArch) - - return true, nil -} - -func (r *mutationResolver) UpdateSMLArch(ctx context.Context, smlLinkID string, smlArch generated.UpdateSMLArch) (*generated.SMLArch, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "updateSMLArch") - defer wrapper.end() - val := ctx.Value(util.ContextValidator{}).(*validator.Validate) - if err := val.Struct(&smlArch); err != nil { - return nil, errors.Wrap(err, "validation failed") - } - - dbSMLArch := postgres.GetSMLArch(newCtx, smlLinkID) - - if dbSMLArch == nil { - return nil, errors.New("sml link not found") - } - - SetStringINNOE(&smlArch.Platform, &dbSMLArch.Platform) - SetStringINNOE(&smlArch.Link, &dbSMLArch.Link) - - postgres.Save(newCtx, &dbSMLArch) - - return DBSMLArchToGenerated(dbSMLArch), nil -} - -func (r *queryResolver) GetSMLArch(ctx context.Context, smlLinkID string) (*generated.SMLArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getSMLArch") - defer wrapper.end() - - smlArch := postgres.GetSMLArch(newCtx, smlLinkID) - - return DBSMLArchToGenerated(smlArch), nil -} - -func (r *queryResolver) GetSMLArchs(ctx context.Context, filter map[string]interface{}) (*generated.GetSMLArchs, error) { - wrapper, _ := WrapQueryTrace(ctx, "getSMLArchs") - defer wrapper.end() - return &generated.GetSMLArchs{}, nil -} - -func (r *queryResolver) GetSMLArchBySMLID(ctx context.Context, smlVersionID string) ([]postgres.SMLArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "GetSMLArchBySMLID") - defer wrapper.end() - smlArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) - return smlArch, nil -} - -func (r *queryResolver) GetSMLDownload(ctx context.Context, smlVersionID string, platform string) (string, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getSMLDownload") - defer wrapper.end() - smlArch := postgres.GetSMLArchDownload(newCtx, smlVersionID, platform) - return smlArch, nil -} diff --git a/gql/resolver_sml_versions.go b/gql/resolver_sml_versions.go index c04c6c1c..959256e3 100644 --- a/gql/resolver_sml_versions.go +++ b/gql/resolver_sml_versions.go @@ -40,20 +40,12 @@ func (r *mutationResolver) CreateSMLVersion(ctx context.Context, smlVersion gene resultSMLVersion, err := postgres.CreateSMLVersion(newCtx, dbSMLVersion) - for _, smlArch := range smlVersion.Arch { - dbSMLArchs := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: resultSMLVersion.ID, - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArchs) - if err != nil { - return nil, err - } - - DBSMLArchToGenerated(resultSMLArch) + for _, smlVersionTarget := range smlVersion.Targets { + postgres.Save(newCtx, &postgres.SMLVersionTarget{ + VersionID: resultSMLVersion.ID, + TargetName: smlVersionTarget.TargetName, + Link: smlVersionTarget.Link, + }) } if err != nil { @@ -86,43 +78,30 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog) SetDateINN(smlVersion.Date, &dbSMLVersion.Date) - dbSMLArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) + dbSMLTargets := postgres.GetSMLVersionTargets(newCtx, smlVersionID) - if len(dbSMLArch) == len(smlVersion.Arch) { - for i, smlArch := range smlVersion.Arch { - SetStringINNOE(&smlArch.Platform, &dbSMLArch[i].Platform) - SetStringINNOE(&smlArch.Link, &dbSMLArch[i].Link) + for _, dbSMLTarget := range dbSMLTargets { + found := false - postgres.Save(newCtx, dbSMLArch) - } - } else { - for _, smlArch := range dbSMLVersion.Arch { - dbSMLArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) - - if dbSMLVersion == nil { - return nil, errors.New("smlArch not found" + smlArch.Platform) + for _, smlTarget := range smlVersion.Targets { + if dbSMLTarget.TargetName == smlTarget.TargetName { + found = true } - - postgres.Delete(newCtx, &dbSMLArch) } - for _, smlArch := range smlVersion.Arch { - dbSMLArch := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: smlVersionID, - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArch) - if err != nil { - return nil, err - } - - DBSMLArchToGenerated(resultSMLArch) + if !found { + postgres.Delete(ctx, &dbSMLTargets) } } + for _, smlTarget := range smlVersion.Targets { + postgres.Save(newCtx, &postgres.SMLVersionTarget{ + VersionID: smlVersionID, + TargetName: smlTarget.TargetName, + Link: smlTarget.Link, + }) + } + postgres.Save(newCtx, &dbSMLVersion) return DBSMLVersionToGenerated(dbSMLVersion), nil @@ -138,14 +117,10 @@ func (r *mutationResolver) DeleteSMLVersion(ctx context.Context, smlVersionID st return false, errors.New("smlVersion not found") } - for _, smlArch := range dbSMLVersion.Arch { - dbSMLArch := postgres.GetSMLArch(newCtx, smlArch.ID) - - if dbSMLVersion == nil { - return false, errors.New("smlArch not found") - } + dbSMLVersionTargets := postgres.GetSMLVersionTargets(newCtx, smlVersionID) - postgres.Delete(newCtx, &dbSMLArch) + for _, dbSMLVersionTarget := range dbSMLVersionTargets { + postgres.Delete(newCtx, &dbSMLVersionTarget) } postgres.Delete(newCtx, &dbSMLVersion) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index 4fe464df..f34ea424 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -367,6 +367,12 @@ func (r *versionResolver) Dependencies(ctx context.Context, obj *generated.Versi return converted, nil } +type versionTargetResolver struct{ *Resolver } + +func (r *versionTargetResolver) Link(_ context.Context, obj *generated.VersionTarget) (string, error) { + return "/v1/version/" + obj.VersionID + "/" + obj.TargetName + "/download", nil +} + type getMyVersionsResolver struct{ *Resolver } func (r *getMyVersionsResolver) Versions(ctx context.Context, _ *generated.GetMyVersions) ([]*generated.Version, error) { diff --git a/gql/versions.go b/gql/versions.go index ca4a3c48..811cec61 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -123,32 +123,34 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI } // TODO: Should legacy plugins be supported? - var modArchs []*postgres.ModArch + var targets []*postgres.VersionTarget for _, target := range modInfo.Targets { - dbModArch, _ := postgres.CreateModArch(ctx, &postgres.ModArch{ - ModVersionID: dbVersion.ID, - Platform: target, - }) + dbVersionTarget := &postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: target, + } + + postgres.Save(ctx, dbVersionTarget) - modArchs = append(modArchs, dbModArch) + targets = append(targets, dbVersionTarget) } separateSuccess := true - for _, modArch := range modArchs { - log.Info().Str("modArch", modArch.Platform).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") - success, key, hash, size := storage.SeparateModPlatform(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, modArch.Platform) + for _, target := range targets { + log.Info().Str("target", target.TargetName).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") + success, key, hash, size := storage.SeparateModTarget(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, target.TargetName) if !success { separateSuccess = false break } - modArch.Key = key - modArch.Hash = hash - modArch.Size = size + target.Key = key + target.Hash = hash + target.Size = size - postgres.Save(ctx, modArch) + postgres.Save(ctx, target) } if !separateSuccess { @@ -211,19 +213,18 @@ func removeMod(ctx context.Context, modInfo *validation.ModInfo, mod *postgres.M } for _, target := range modInfo.Targets { - // TODO: ModArch should have VersionID and Platform as primary key - dbModArch := postgres.ModArch{ - ModVersionID: dbVersion.ID, - Platform: target, + dbVersionTarget := postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: target, } - postgres.DeleteForced(ctx, &dbModArch) + postgres.DeleteForced(ctx, &dbVersionTarget) } postgres.DeleteForced(ctx, &dbVersion) storage.DeleteMod(ctx, mod.ID, mod.Name, dbVersion.ID) for _, target := range modInfo.Targets { - storage.DeleteModPlatform(ctx, mod.ID, mod.Name, dbVersion.ID, target) + storage.DeleteModTarget(ctx, mod.ID, mod.Name, dbVersion.ID, target) } } diff --git a/gqlgen.yml b/gqlgen.yml index ecb11180..b240c6e7 100755 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -20,10 +20,6 @@ models: UpdateMod: model: github.com/satisfactorymodding/smr-api/generated.UpdateMod - ModArchFilter: - model: "map[string]interface{}" - SMLArchFilter: - model: "map[string]interface{}" VersionFilter: model: "map[string]interface{}" ModFilter: @@ -57,11 +53,6 @@ models: latestVersions: resolver: true - ModArch: - fields: - asset: - resolver: true - UserMod: fields: user: @@ -77,6 +68,10 @@ models: resolver: true dependencies: resolver: true + VersionTarget: + fields: + link: + resolver: true GetMods: fields: diff --git a/migrations/sql/000022_update_mod_targets.down.sql b/migrations/sql/000022_update_mod_targets.down.sql new file mode 100644 index 00000000..fb401fb8 --- /dev/null +++ b/migrations/sql/000022_update_mod_targets.down.sql @@ -0,0 +1,62 @@ +Create or replace function update_mod_platform_down_random_string(length integer) returns text as +$$ +declare + chars text[] := '{0,1,2,3,4,5,6,7,8,9,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}'; + result text := ''; + i integer := 0; +begin + if length < 0 then + raise exception 'Given length cannot be less than 0'; + end if; + for i in 1..length loop + result := result || chars[1+random()*(array_length(chars, 1)-1)]; + end loop; + return result; +end; +$$ language plpgsql; + +ALTER TABLE version_targets RENAME TO mod_archs; + +ALTER TABLE mod_archs + RENAME COLUMN version_id TO mod_version_arch_id; +ALTER TABLE mod_archs + RENAME COLUMN target_name TO platform; +ALTER TABLE mod_archs + ADD COLUMN id varchar(14); + +-- This is not as random as the original ID, but it should be good enough +UPDATE mod_archs SET id = update_mod_platform_down_random_string(14) WHERE true; + +ALTER TABLE mod_archs + ALTER COLUMN id SET NOT NULL; + +ALTER TABLE mod_archs + DROP CONSTRAINT version_targets_version_id_fkey, + DROP CONSTRAINT version_targets_pkey, + ADD CONSTRAINT mod_archs_pkey PRIMARY KEY (id); + +CREATE INDEX IF NOT EXISTS idx_mod_arch_id ON mod_archs (mod_version_arch_id, platform); + +ALTER TABLE sml_version_targets RENAME TO sml_archs; + +ALTER TABLE sml_archs + RENAME COLUMN version_id TO sml_version_arch_id; +ALTER TABLE sml_archs + RENAME COLUMN target_name TO platform; +ALTER TABLE sml_archs + ADD COLUMN id varchar(14); + +-- This is not as random as the original ID, but it should be good enough +UPDATE sml_archs SET id = update_mod_platform_down_random_string(14) WHERE true; + +ALTER TABLE sml_archs + ALTER COLUMN id SET NOT NULL; + +ALTER TABLE sml_archs + DROP CONSTRAINT sml_version_targets_version_id_fkey, + DROP CONSTRAINT sml_version_targets_pkey, + ADD CONSTRAINT sml_archs_pkey PRIMARY KEY (id); + +CREATE INDEX IF NOT EXISTS idx_sml_archs_id ON sml_archs (sml_version_arch_id, platform); + +DROP FUNCTION update_mod_platform_down_random_string(length integer); \ No newline at end of file diff --git a/migrations/sql/000022_update_mod_targets.up.sql b/migrations/sql/000022_update_mod_targets.up.sql new file mode 100644 index 00000000..23bfc8bb --- /dev/null +++ b/migrations/sql/000022_update_mod_targets.up.sql @@ -0,0 +1,29 @@ +ALTER TABLE mod_archs RENAME TO version_targets; + +DROP INDEX idx_mod_arch_id; + +ALTER TABLE version_targets + RENAME COLUMN mod_version_arch_id TO version_id; +ALTER TABLE version_targets + RENAME COLUMN platform TO target_name; +ALTER TABLE version_targets + DROP COLUMN id; + +ALTER TABLE version_targets + ADD CONSTRAINT version_targets_version_id_fkey FOREIGN KEY (version_id) REFERENCES versions (id), + ADD CONSTRAINT version_targets_pkey PRIMARY KEY (version_id, target_name); + +ALTER TABLE sml_archs RENAME TO sml_version_targets; + +DROP INDEX idx_sml_archs_id; + +ALTER TABLE sml_version_targets + RENAME COLUMN sml_version_arch_id TO version_id; +ALTER TABLE sml_version_targets + RENAME COLUMN platform TO target_name; +ALTER TABLE sml_version_targets + DROP COLUMN id; + +ALTER TABLE sml_version_targets + ADD CONSTRAINT sml_version_targets_version_id_fkey FOREIGN KEY (version_id) REFERENCES sml_versions (id), + ADD CONSTRAINT sml_version_targets_pkey PRIMARY KEY (version_id, target_name); \ No newline at end of file diff --git a/models/filters.go b/models/filters.go index 885e9ce1..c7fd5d61 100644 --- a/models/filters.go +++ b/models/filters.go @@ -328,85 +328,3 @@ func ProcessBootstrapVersionFilter(filter map[string]interface{}) (*BootstrapVer return base, nil } - -type SMLArchFilter struct { - Limit *int `json:"limit" validate:"omitempty,min=1,max=100"` - Offset *int `json:"offset" validate:"omitempty,min=0"` - OrderBy *generated.SMLArchFields `json:"order_by"` - Order *generated.Order `json:"order"` - Search *string `json:"search" validate:"omitempty,min=3"` - Ids []string `json:"ids" validate:"omitempty,max=100"` -} - -func DefaultSMLArchFilter() *SMLArchFilter { - limit := 10 - offset := 0 - order := generated.OrderDesc - orderBy := generated.SMLArchFieldsPlatform - return &SMLArchFilter{ - Limit: &limit, - Offset: &offset, - Ids: nil, - Order: &order, - OrderBy: &orderBy, - } -} - -func ProcessSMLArchFilter(filter map[string]interface{}) (*SMLArchFilter, error) { - base := DefaultSMLArchFilter() - - if filter == nil { - return base, nil - } - - if err := ApplyChanges(filter, base); err != nil { - return nil, err - } - - if err := dataValidator.Struct(base); err != nil { - return nil, errors.Wrap(err, "failed to validate SMLArchFilter") - } - - return base, nil -} - -type ModArchFilter struct { - Limit *int `json:"limit" validate:"omitempty,min=1,max=100"` - Offset *int `json:"offset" validate:"omitempty,min=0"` - OrderBy *generated.ModArchFields `json:"order_by"` - Order *generated.Order `json:"order"` - Search *string `json:"search" validate:"omitempty,min=3"` - Ids []string `json:"ids" validate:"omitempty,max=100"` -} - -func DefaultModArchFilter() *ModArchFilter { - limit := 10 - offset := 0 - order := generated.OrderDesc - orderBy := generated.ModArchFieldsPlatform - return &ModArchFilter{ - Limit: &limit, - Offset: &offset, - Ids: nil, - Order: &order, - OrderBy: &orderBy, - } -} - -func ProcessModArchFilter(filter map[string]interface{}) (*ModArchFilter, error) { - base := DefaultModArchFilter() - - if filter == nil { - return base, nil - } - - if err := ApplyChanges(filter, base); err != nil { - return nil, err - } - - if err := dataValidator.Struct(base); err != nil { - return nil, errors.Wrap(err, "failed to validate ModArchFilter") - } - - return base, nil -} diff --git a/nodes/mod.go b/nodes/mod.go index 725fb109..d589ebc1 100644 --- a/nodes/mod.go +++ b/nodes/mod.go @@ -289,20 +289,20 @@ func downloadModVersion(c echo.Context) error { return c.Redirect(302, storage.GenerateDownloadLink(version.Key)) } -// @Summary Download a Mod Version by Platform +// @Summary Download a Mod Version by TargetName // @Tags Mod -// @Description Download a mod version by mod ID and version ID and Platform +// @Description Download a mod version by mod ID and version ID and TargetName // @Accept json // @Produce json // @Param modId path string true "Mod ID" // @Param versionId path string true "Version ID" -// @Param versionId path string true "Platform" +// @Param target path string true "TargetName" // @Success 200 -// @Router /mod/{modId}/versions/{versionId}/{platform}/download [get] -func downloadModVersionArch(c echo.Context) error { +// @Router /mod/{modId}/versions/{versionId}/{target}/download [get] +func downloadModVersionTarget(c echo.Context) error { modID := c.Param("modId") versionID := c.Param("versionId") - platform := c.Param("platform") + target := c.Param("target") mod := postgres.GetModByID(c.Request().Context(), modID) @@ -316,15 +316,15 @@ func downloadModVersionArch(c echo.Context) error { return c.String(404, "version not found, modID:"+modID+" versionID:"+versionID) } - arch := postgres.GetModArchByPlatform(c.Request().Context(), versionID, platform) + versionTarget := postgres.GetVersionTarget(c.Request().Context(), versionID, target) - if arch == nil { - return c.String(404, "platform not found, modID:"+modID+" versionID:"+versionID+" platform:"+platform) + if versionTarget == nil { + return c.String(404, "target not found, modID:"+modID+" versionID:"+versionID+" target:"+target) } if redis.CanIncrement(c.RealIP(), "download", "version:"+versionID, time.Hour*4) { postgres.IncrementVersionDownloads(c.Request().Context(), version) } - return c.Redirect(302, storage.GenerateDownloadLink(arch.Key)) + return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key)) } diff --git a/nodes/routes.go b/nodes/routes.go index 8cd75b83..e84b94fb 100755 --- a/nodes/routes.go +++ b/nodes/routes.go @@ -15,7 +15,7 @@ func RegisterModRoutes(router *echo.Group) { router.GET("/:modId/versions/:versionId", dataWrapper(getModVersion)) router.GET("/:modId/versions/:versionId/download", downloadModVersion) - router.GET("/:modId/versions/:versionId/:platform/download", downloadModVersionArch) + router.GET("/:modId/versions/:versionId/:target/download", downloadModVersionTarget) } func RegisterModsRoutes(router *echo.Group) { @@ -48,7 +48,7 @@ func RegisterUsersRoutes(router *echo.Group) { func RegisterVersionRoutes(router *echo.Group) { router.GET("/:versionId", dataWrapper(getVersion)) router.GET("/:versionId/download", downloadVersion) - router.GET("/:versionId/:platform/download", downloadModArch) + router.GET("/:versionId/:target/download", downloadModTarget) } func RegisterSMLRoutes(router *echo.Group) { diff --git a/nodes/version.go b/nodes/version.go index 178064f0..1da2eeef 100644 --- a/nodes/version.go +++ b/nodes/version.go @@ -54,19 +54,19 @@ func downloadVersion(c echo.Context) error { return c.Redirect(302, storage.GenerateDownloadLink(version.Key)) } -// @Summary Download a Platform +// @Summary Download a TargetName // @Tags Version -// @Tags Platform -// @Description Download a mod version by version ID and Platform +// @Tags TargetName +// @Description Download a mod version by version ID and TargetName // @Accept json // @Produce json // @Param versionId path string true "Version ID" -// @Param versionId path string true "Version ID" +// @Param target path string true "TargetName" // @Success 200 -// @Router /versions/{versionId}/{platform}/download [get] -func downloadModArch(c echo.Context) error { +// @Router /versions/{versionId}/{target}/download [get] +func downloadModTarget(c echo.Context) error { versionID := c.Param("versionId") - platform := c.Param("platform") + target := c.Param("target") version := postgres.GetVersion(c.Request().Context(), versionID) @@ -74,15 +74,15 @@ func downloadModArch(c echo.Context) error { return c.String(404, "version not found, versionID:"+versionID) } - arch := postgres.GetModArchByPlatform(c.Request().Context(), versionID, platform) + versionTarget := postgres.GetVersionTarget(c.Request().Context(), versionID, target) - if arch == nil { - return c.String(404, "platform not found, versionID:"+versionID+" platform:"+platform) + if versionTarget == nil { + return c.String(404, "target not found, versionID:"+versionID+" target:"+target) } if redis.CanIncrement(c.RealIP(), "download", "version:"+versionID, time.Hour*4) { postgres.IncrementVersionDownloads(c.Request().Context(), version) } - return c.Redirect(302, storage.GenerateDownloadLink(arch.Key)) + return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key)) } diff --git a/schemas/mod_archs.graphql b/schemas/mod_archs.graphql deleted file mode 100644 index 648fc180..00000000 --- a/schemas/mod_archs.graphql +++ /dev/null @@ -1,38 +0,0 @@ -### Types - -scalar ModArchID - -type ModArch { - id: ModArchID! - ModVersionID: String! - platform: String! - asset: String! - size: Int - hash: String -} - -enum ModArchFields { - platform -} - -type GetModArchs { - arch: [ModArch!]! -} - -### Inputs - -input ModArchFilter { - limit: Int - offset: Int - order_by: ModArchFields - order: Order - search: String - ids: [String!] -} - -### Queries - -extend type Query { - getModArch(id: ModArchID!): ModArch - getModArchs(filter: ModArchFilter): GetModArchs! -} \ No newline at end of file diff --git a/schemas/sml_archs.graphql b/schemas/sml_archs.graphql deleted file mode 100644 index e3cdb327..00000000 --- a/schemas/sml_archs.graphql +++ /dev/null @@ -1,46 +0,0 @@ -### Types - -scalar SMLArchID - -type SMLArch { - id: SMLArchID! - SMLVersionID: String! - platform: String! - link: String! -} - -enum SMLArchFields { - platform -} - -type GetSMLArchs { - arch: [SMLArch!]! -} - -### Inputs - -input SMLArchFilter { - limit: Int - offset: Int - order_by: SMLArchFields - order: Order - search: String - ids: [String!] -} - -input NewSMLArch { - platform: String! - link: String! -} - -input UpdateSMLArch { - platform: String! - link: String! -} - -### Queries - -extend type Query { - getSMLArch(smlVersionId: SMLVersionID!): SMLArch - getSMLArchs(filter: SMLArchFilter): GetSMLArchs! -} \ No newline at end of file diff --git a/schemas/sml_version.graphql b/schemas/sml_version.graphql index ef207ca2..854875a1 100755 --- a/schemas/sml_version.graphql +++ b/schemas/sml_version.graphql @@ -8,7 +8,7 @@ type SMLVersion { satisfactory_version: Int! stability: VersionStabilities! link: String! - arch: [SMLArch]! + targets: [SMLVersionTarget]! changelog: String! date: Date! bootstrap_version: String @@ -17,6 +17,12 @@ type SMLVersion { created_at: Date! } +type SMLVersionTarget { + VersionID: SMLVersionID! + targetName: String! + link: String! +} + type GetSMLVersions { sml_versions: [SMLVersion!]! count: Int! @@ -37,7 +43,7 @@ input NewSMLVersion { satisfactory_version: Int! stability: VersionStabilities! link: String! - arch: [NewSMLArch!]! + targets: [NewSMLVersionTarget!]! changelog: String! date: Date! bootstrap_version: String @@ -48,12 +54,22 @@ input UpdateSMLVersion { satisfactory_version: Int stability: VersionStabilities link: String - arch: [UpdateSMLArch]! + targets: [UpdateSMLVersionTarget]! changelog: String date: Date bootstrap_version: String } +input NewSMLVersionTarget { + targetName: String! + link: String! +} + +input UpdateSMLVersionTarget { + targetName: String! + link: String! +} + input SMLVersionFilter { limit: Int offset: Int diff --git a/schemas/version.graphql b/schemas/version.graphql index 2513639f..ec4880b3 100755 --- a/schemas/version.graphql +++ b/schemas/version.graphql @@ -32,7 +32,7 @@ type Version { updated_at: Date! created_at: Date! link: String! - arch: [ModArch]! + targets: [VersionTarget]! metadata: String size: Int hash: String @@ -41,6 +41,14 @@ type Version { dependencies: [VersionDependency!]! } +type VersionTarget { + VersionID: VersionID! + targetName: String! + link: String! + size: Int + hash: String +} + type CreateVersionResponse { auto_approved: Boolean! version: Version diff --git a/storage/storage.go b/storage/storage.go index a77c27a5..0b8aa8f7 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -289,15 +289,15 @@ func DeleteMod(ctx context.Context, modID string, name string, versionID string) return true } -func DeleteModPlatform(ctx context.Context, modID string, name string, versionID string, platform string) bool { +func DeleteModTarget(ctx context.Context, modID string, name string, versionID string, target string) bool { if storage == nil { return false } cleanName := cleanModName(name) - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+platform+"-"+versionID) + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+target+"-"+versionID) - log.Info().Str("key", key).Msg("deleting mod platform") + log.Info().Str("key", key).Msg("deleting mod target") if err := storage.Delete(key); err != nil { log.Err(err).Msg("failed to delete version link") return false @@ -366,7 +366,7 @@ func EncodeName(name string) string { return result } -func SeparateModPlatform(ctx context.Context, body []byte, modID, name, modVersion, platform string) (bool, string, string, int64) { +func SeparateModTarget(ctx context.Context, body []byte, modID, name, modVersion, target string) (bool, string, string, int64) { zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { return false, "", "", 0 @@ -378,14 +378,14 @@ func SeparateModPlatform(ctx context.Context, body []byte, modID, name, modVersi zipWriter := zip.NewWriter(buf) for _, file := range zipReader.File { - if !strings.HasPrefix(file.Name, platform+"/") && file.Name != platform+"/" { + if !strings.HasPrefix(file.Name, target+"/") && file.Name != target+"/" { continue } - err = copyModFileToArchZip(file, zipWriter, strings.TrimPrefix(file.Name, platform+"/")) + err = copyModFileToArchZip(file, zipWriter, strings.TrimPrefix(file.Name, target+"/")) if err != nil { - log.Err(err).Msg("failed to add file to " + platform + " archive") + log.Err(err).Msg("failed to add file to " + target + " archive") return false, "", "", 0 } @@ -393,11 +393,11 @@ func SeparateModPlatform(ctx context.Context, body []byte, modID, name, modVersi zipWriter.Close() - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+platform+"-"+modVersion) + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+target+"-"+modVersion) _, err = storage.Put(ctx, key, bytes.NewReader(buf.Bytes())) if err != nil { - log.Err(err).Msg("failed to save " + platform + " archive") + log.Err(err).Msg("failed to save " + target + " archive") return false, "", "", 0 } From 0e7f9a50847a03ca3e7cb320d8cfec7a714c1731 Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 19:42:18 +0200 Subject: [PATCH 06/25] feat: backwards compatibility for Version fields Link, Hash and Size These fields in the db store the original uploaded file, which for single-target plugins (WindowsNoEditor) is valid, but for multi-target plugins is not Return the WindowsNoEditor target information, if available, otherwise return the version's --- gql/resolver_versions.go | 74 ++++++++++++++++++++++++++++++++++++++-- gqlgen.yml | 5 +++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index f34ea424..08238549 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -319,8 +319,34 @@ func (r *getVersionsResolver) Count(ctx context.Context, _ *generated.GetVersion type versionResolver struct{ *Resolver } -func (r *versionResolver) Link(_ context.Context, obj *generated.Version) (string, error) { - return "/v1/version/" + obj.ID + "/download", nil +func findWindowsTarget(obj *generated.Version) *generated.VersionTarget { + var windowsTarget *generated.VersionTarget + for _, target := range obj.Targets { + if target.TargetName == "WindowsNoEditor" { + windowsTarget = target + break + } + // TODO UE5: Also check for Windows target + // if target.TargetName == "Windows" { + // windowsTarget = target + // break + // } + } + return windowsTarget +} + +func (r *versionResolver) Link(ctx context.Context, obj *generated.Version) (string, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.link") + defer wrapper.end() + + link := "/v1/version/" + obj.ID + "/download" + + windowsTarget := findWindowsTarget(obj) + if windowsTarget != nil { + link, _ = r.VersionTarget().Link(ctx, windowsTarget) + } + + return link, nil } func (r *versionResolver) Mod(ctx context.Context, obj *generated.Version) (*generated.Mod, error) { @@ -330,6 +356,50 @@ func (r *versionResolver) Mod(ctx context.Context, obj *generated.Version) (*gen return DBModToGenerated(postgres.GetModByID(newCtx, obj.ModID)), nil } +func (r *versionResolver) Hash(ctx context.Context, obj *generated.Version) (*string, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.hash") + defer wrapper.end() + + hash := "" + + windowsTarget := findWindowsTarget(obj) + if windowsTarget == nil { + if obj.Hash == nil { + return nil, nil + } + hash = *obj.Hash + } else { + if windowsTarget.Hash == nil { + return nil, nil + } + hash = *windowsTarget.Hash + } + + return &hash, nil +} + +func (r *versionResolver) Size(ctx context.Context, obj *generated.Version) (*int, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.size") + defer wrapper.end() + + size := 0 + + windowsTarget := findWindowsTarget(obj) + if windowsTarget == nil { + if obj.Size == nil { + return nil, nil + } + size = *obj.Size + } else { + if windowsTarget.Size == nil { + return nil, nil + } + size = *windowsTarget.Size + } + + return &size, nil +} + var versionDependencyCache, _ = ristretto.NewCache(&ristretto.Config{ NumCounters: 1e6, // number of keys to track frequency of (1M). MaxCost: 1e6, // maximum cost of cache (1M). diff --git a/gqlgen.yml b/gqlgen.yml index b240c6e7..7c2148e4 100755 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -68,6 +68,11 @@ models: resolver: true dependencies: resolver: true + size: + resolver: true + hash: + resolver: true + VersionTarget: fields: link: From d33815dd82c1ebc5d64075fa176501c4ffa88ccb Mon Sep 17 00:00:00 2001 From: mircearoata Date: Mon, 8 May 2023 20:01:55 +0200 Subject: [PATCH 07/25] chore: lint --- gql/versions.go | 2 +- storage/storage.go | 1 - validation/validation.go | 7 +++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/gql/versions.go b/gql/versions.go index 811cec61..7fa1e8bb 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -123,7 +123,7 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI } // TODO: Should legacy plugins be supported? - var targets []*postgres.VersionTarget + targets := make([]*postgres.VersionTarget, 0) for _, target := range modInfo.Targets { dbVersionTarget := &postgres.VersionTarget{ diff --git a/storage/storage.go b/storage/storage.go index 0b8aa8f7..e246a3cc 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -388,7 +388,6 @@ func SeparateModTarget(ctx context.Context, body []byte, modID, name, modVersion log.Err(err).Msg("failed to add file to " + target + " archive") return false, "", "", 0 } - } zipWriter.Close() diff --git a/validation/validation.go b/validation/validation.go index 19a8f577..06fbd780 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -32,7 +32,7 @@ type ModObject struct { type ModType int const ( - DataJson ModType = iota + DataJSON ModType = iota UEPlugin = 1 MultiTargetUEPlugin = 2 ) @@ -47,8 +47,8 @@ type ModInfo struct { SMLVersion string `json:"sml_version"` Objects []ModObject `json:"objects"` Metadata []map[string]map[string][]interface{} `json:"-"` - Size int64 `json:"-"` Targets []string `json:"-"` + Size int64 `json:"-"` Type ModType `json:"-"` } @@ -115,7 +115,6 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal } if modInfo == nil { - // Neither data.json nor .uplugin found, try multi-target .uplugin modInfo, err = validateMultiTargetPlugin(archive, withValidation, modReference) if err != nil { @@ -267,7 +266,7 @@ func validateDataJSON(archive *zip.Reader, dataFile *zip.File, withValidation bo } } - modInfo.Type = DataJson + modInfo.Type = DataJSON return &modInfo, nil } From e5a8fd5a3f3957402e71e06517f421924ddc24e8 Mon Sep 17 00:00:00 2001 From: mircearoata Date: Thu, 11 May 2023 10:34:36 +0200 Subject: [PATCH 08/25] fix: update a missed target->target_name --- db/postgres/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/postgres/version.go b/db/postgres/version.go index 041e347e..e3f99fc1 100644 --- a/db/postgres/version.go +++ b/db/postgres/version.go @@ -268,7 +268,7 @@ func GetVersionTarget(ctx context.Context, versionID string, target string) *Ver } var versionTarget VersionTarget - DBCtx(ctx).First(&versionTarget, "version_id = ? AND target = ?", versionID, target) + DBCtx(ctx).First(&versionTarget, "version_id = ? AND target_name = ?", versionID, target) if versionTarget.VersionID == "" { return nil From 1925d55b3079f3ba059dd6acfc78f604e8c0c57a Mon Sep 17 00:00:00 2001 From: mircearoata Date: Thu, 11 May 2023 10:34:54 +0200 Subject: [PATCH 09/25] chore: update more missed link->target --- gql/gql_types.go | 6 +++--- storage/storage.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gql/gql_types.go b/gql/gql_types.go index e3b126ae..28595e34 100644 --- a/gql/gql_types.go +++ b/gql/gql_types.go @@ -247,9 +247,9 @@ func DBSMLVersionTargetToGenerated(smlVersionTarget *postgres.SMLVersionTarget) } } -func DBSMLVersionTargetToGeneratedSlice(smlLinks []postgres.SMLVersionTarget) []*generated.SMLVersionTarget { - converted := make([]*generated.SMLVersionTarget, len(smlLinks)) - for i, smlVersionTarget := range smlLinks { +func DBSMLVersionTargetToGeneratedSlice(smlVersionTargets []postgres.SMLVersionTarget) []*generated.SMLVersionTarget { + converted := make([]*generated.SMLVersionTarget, len(smlVersionTargets)) + for i, smlVersionTarget := range smlVersionTargets { converted[i] = DBSMLVersionTargetToGenerated(&smlVersionTarget) } return converted diff --git a/storage/storage.go b/storage/storage.go index e246a3cc..180cf37f 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -299,7 +299,7 @@ func DeleteModTarget(ctx context.Context, modID string, name string, versionID s log.Info().Str("key", key).Msg("deleting mod target") if err := storage.Delete(key); err != nil { - log.Err(err).Msg("failed to delete version link") + log.Err(err).Msg("failed to delete version target") return false } From cc9bc6212a9d11adb7cc47f363fc4dba453b75d3 Mon Sep 17 00:00:00 2001 From: porisius <47161774+porisius@users.noreply.github.com> Date: Mon, 29 May 2023 19:02:09 +0000 Subject: [PATCH 10/25] feat: VirusTotal parallelization --- .github/workflows/build.yml | 1 + go.mod | 1 + go.sum | 1 + validation/virustotal.go | 91 ++++++++++++++++++++++++++----------- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe345398..f06bb0ac 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: + version: v1.49.0 skip-pkg-cache: true skip-build-cache: true args: --timeout 5m diff --git a/go.mod b/go.mod index 2a6ed37f..43a278e6 100755 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa golang.org/x/net v0.0.0-20220728030405-41545e8bf201 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/go-playground/validator.v9 v9.31.0 gorm.io/driver/postgres v1.3.5 gorm.io/gorm v1.23.5 diff --git a/go.sum b/go.sum index 42c970ce..7828d761 100755 --- a/go.sum +++ b/go.sum @@ -1677,6 +1677,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/validation/virustotal.go b/validation/virustotal.go index f9104ed7..e12f4eb6 100644 --- a/validation/virustotal.go +++ b/validation/virustotal.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" ) var client *vt.Client @@ -32,44 +33,82 @@ type AnalysisResults struct { } func ScanFiles(ctx context.Context, files []io.Reader, names []string) (bool, error) { - for i, file := range files { - scan, err := client.NewFileScanner().Scan(file, names[i], nil) - if err != nil { - return false, errors.Wrap(err, "failed to scan file") + errs, gctx := errgroup.WithContext(context.Background()) + fileCount := len(files) + + c := make(chan bool) + + for i := 0; i < fileCount; i++ { + count := i + errs.Go(func() error { + ok, err := scanFile(gctx, files[count], names[count]) + if err != nil { + return errors.Wrap(err, "failed to scan file") + } + c <- ok + return nil + }) + } + go func() { + _ = errs.Wait() + close(c) + }() + + success := true + for res := range c { + if !res { + success = false + break } + } - analysisID := scan.ID() + if err := errs.Wait(); err != nil { + return false, errors.Wrap(err, "failed to scan file") + } - log.Info().Msgf("uploaded virus scan for file %s and analysis ID: %s", names[i], analysisID) + return success, nil +} - for { - time.Sleep(time.Second * 15) +func scanFile(ctx context.Context, file io.Reader, name string) (bool, error) { + scan, err := client.NewFileScanner().Scan(file, name, nil) + if err != nil { + return false, errors.Wrap(err, "failed to scan file") + } - var target AnalysisResults - _, err = client.GetData(vt.URL("analyses/%s", analysisID), &target) + analysisID := scan.ID() - if err != nil { - return false, errors.Wrap(err, "failed to get analysis results") - } + log.Info().Msgf("uploaded virus scan for file %s and analysis ID: %s", name, analysisID) - if target.Attributes.Status != "completed" { - continue - } + for { + time.Sleep(time.Second * 15) - if target.Attributes.Stats == nil { - return false, nil - } + var target AnalysisResults + _, err = client.GetData(vt.URL("analyses/%s", analysisID), &target) - if target.Attributes.Stats.Malicious == nil || target.Attributes.Stats.Suspicious == nil { - return false, nil - } + if err != nil { + return false, errors.Wrap(err, "failed to get analysis results") + } - if *target.Attributes.Stats.Malicious > 0 || *target.Attributes.Stats.Suspicious > 0 { - return false, nil - } + if target.Attributes.Status != "completed" { + continue + } - break + if target.Attributes.Stats == nil { + log.Error().Msgf("no stats available. failing file: %s", name) + return false, nil + } + + if target.Attributes.Stats.Malicious == nil || target.Attributes.Stats.Suspicious == nil { + log.Error().Msgf("unable to determine malicious or suspicious File: %s", name) + return false, nil } + + if *target.Attributes.Stats.Malicious > 0 || *target.Attributes.Stats.Suspicious > 0 { + log.Error().Msgf("suspicious or malicious file found: %s", name) + return false, nil + } + + break } return true, nil From a042c4d6f1cd0e43054ec93d682a53049787973c Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sat, 19 Aug 2023 15:59:52 +0300 Subject: [PATCH 11/25] chore: rename WindowsNoEditor to Windows --- gql/resolver_versions.go | 7 +------ validation/validation.go | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index 08238549..7ef07595 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -322,15 +322,10 @@ type versionResolver struct{ *Resolver } func findWindowsTarget(obj *generated.Version) *generated.VersionTarget { var windowsTarget *generated.VersionTarget for _, target := range obj.Targets { - if target.TargetName == "WindowsNoEditor" { + if target.TargetName == "Windows" { windowsTarget = target break } - // TODO UE5: Also check for Windows target - // if target.TargetName == "Windows" { - // windowsTarget = target - // break - // } } return windowsTarget } diff --git a/validation/validation.go b/validation/validation.go index 06fbd780..d5b955d9 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -21,8 +21,7 @@ import ( "github.com/xeipuuv/gojsonschema" ) -// TODO UE5: WindowsNoEditor -> Windows -var AllowedTargets = []string{"WindowsNoEditor", "WindowsServer", "LinuxServer"} +var AllowedTargets = []string{"Windows", "WindowsServer", "LinuxServer"} type ModObject struct { Path string `json:"path"` From 24c46eebce4bd2e3738ca144eba4823f9e13b1bc Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sat, 19 Aug 2023 16:03:21 +0300 Subject: [PATCH 12/25] feat: enum for TargetName --- gql/gql_types.go | 4 ++-- gql/resolver_sml_versions.go | 6 +++--- gql/resolver_versions.go | 2 +- schemas/sml_version.graphql | 6 +++--- schemas/version.graphql | 2 +- schemas/version_target.graphql | 5 +++++ 6 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 schemas/version_target.graphql diff --git a/gql/gql_types.go b/gql/gql_types.go index 28595e34..0fa9756f 100644 --- a/gql/gql_types.go +++ b/gql/gql_types.go @@ -221,7 +221,7 @@ func DBVersionTargetToGenerated(versionTarget *postgres.VersionTarget) *generate return &generated.VersionTarget{ VersionID: versionTarget.VersionID, - TargetName: versionTarget.TargetName, + TargetName: generated.TargetName(versionTarget.TargetName), Hash: &hash, Size: &size, } @@ -242,7 +242,7 @@ func DBSMLVersionTargetToGenerated(smlVersionTarget *postgres.SMLVersionTarget) return &generated.SMLVersionTarget{ VersionID: smlVersionTarget.VersionID, - TargetName: smlVersionTarget.TargetName, + TargetName: generated.TargetName(smlVersionTarget.TargetName), Link: smlVersionTarget.Link, } } diff --git a/gql/resolver_sml_versions.go b/gql/resolver_sml_versions.go index 959256e3..5e1f4580 100644 --- a/gql/resolver_sml_versions.go +++ b/gql/resolver_sml_versions.go @@ -43,7 +43,7 @@ func (r *mutationResolver) CreateSMLVersion(ctx context.Context, smlVersion gene for _, smlVersionTarget := range smlVersion.Targets { postgres.Save(newCtx, &postgres.SMLVersionTarget{ VersionID: resultSMLVersion.ID, - TargetName: smlVersionTarget.TargetName, + TargetName: string(smlVersionTarget.TargetName), Link: smlVersionTarget.Link, }) } @@ -84,7 +84,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st found := false for _, smlTarget := range smlVersion.Targets { - if dbSMLTarget.TargetName == smlTarget.TargetName { + if dbSMLTarget.TargetName == string(smlTarget.TargetName) { found = true } } @@ -97,7 +97,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st for _, smlTarget := range smlVersion.Targets { postgres.Save(newCtx, &postgres.SMLVersionTarget{ VersionID: smlVersionID, - TargetName: smlTarget.TargetName, + TargetName: string(smlTarget.TargetName), Link: smlTarget.Link, }) } diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index 7ef07595..8454add1 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -435,7 +435,7 @@ func (r *versionResolver) Dependencies(ctx context.Context, obj *generated.Versi type versionTargetResolver struct{ *Resolver } func (r *versionTargetResolver) Link(_ context.Context, obj *generated.VersionTarget) (string, error) { - return "/v1/version/" + obj.VersionID + "/" + obj.TargetName + "/download", nil + return "/v1/version/" + obj.VersionID + "/" + string(obj.TargetName) + "/download", nil } type getMyVersionsResolver struct{ *Resolver } diff --git a/schemas/sml_version.graphql b/schemas/sml_version.graphql index 854875a1..3e300aa8 100755 --- a/schemas/sml_version.graphql +++ b/schemas/sml_version.graphql @@ -19,7 +19,7 @@ type SMLVersion { type SMLVersionTarget { VersionID: SMLVersionID! - targetName: String! + targetName: TargetName! link: String! } @@ -61,12 +61,12 @@ input UpdateSMLVersion { } input NewSMLVersionTarget { - targetName: String! + targetName: TargetName! link: String! } input UpdateSMLVersionTarget { - targetName: String! + targetName: TargetName! link: String! } diff --git a/schemas/version.graphql b/schemas/version.graphql index ec4880b3..168d0de8 100755 --- a/schemas/version.graphql +++ b/schemas/version.graphql @@ -43,7 +43,7 @@ type Version { type VersionTarget { VersionID: VersionID! - targetName: String! + targetName: TargetName! link: String! size: Int hash: String diff --git a/schemas/version_target.graphql b/schemas/version_target.graphql new file mode 100644 index 00000000..2d77e3d1 --- /dev/null +++ b/schemas/version_target.graphql @@ -0,0 +1,5 @@ +enum TargetName { + Windows, + WindowsServer, + LinuxServer +} \ No newline at end of file From ecf604a4ab2123e5692f8777479027c50f70ecb0 Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sat, 19 Aug 2023 16:47:54 +0300 Subject: [PATCH 13/25] fix: delete SML version targets when removed --- gql/resolver_sml_versions.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/gql/resolver_sml_versions.go b/gql/resolver_sml_versions.go index 5e1f4580..b8102ff8 100644 --- a/gql/resolver_sml_versions.go +++ b/gql/resolver_sml_versions.go @@ -64,20 +64,6 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st return nil, errors.Wrap(err, "validation failed") } - dbSMLVersion := postgres.GetSMLVersionByID(newCtx, smlVersionID) - - if dbSMLVersion == nil { - return nil, errors.New("smlVersion not found") - } - - SetStringINNOE(smlVersion.Version, &dbSMLVersion.Version) - SetINN(smlVersion.SatisfactoryVersion, &dbSMLVersion.SatisfactoryVersion) - SetStringINNOE(smlVersion.BootstrapVersion, dbSMLVersion.BootstrapVersion) - SetStabilityINN(smlVersion.Stability, &dbSMLVersion.Stability) - SetStringINNOE(smlVersion.Link, &dbSMLVersion.Link) - SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog) - SetDateINN(smlVersion.Date, &dbSMLVersion.Date) - dbSMLTargets := postgres.GetSMLVersionTargets(newCtx, smlVersionID) for _, dbSMLTarget := range dbSMLTargets { @@ -90,7 +76,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st } if !found { - postgres.Delete(ctx, &dbSMLTargets) + postgres.Delete(newCtx, &dbSMLTarget) } } @@ -102,6 +88,20 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st }) } + dbSMLVersion := postgres.GetSMLVersionByID(newCtx, smlVersionID) + + if dbSMLVersion == nil { + return nil, errors.New("smlVersion not found") + } + + SetStringINNOE(smlVersion.Version, &dbSMLVersion.Version) + SetINN(smlVersion.SatisfactoryVersion, &dbSMLVersion.SatisfactoryVersion) + SetStringINNOE(smlVersion.BootstrapVersion, dbSMLVersion.BootstrapVersion) + SetStabilityINN(smlVersion.Stability, &dbSMLVersion.Stability) + SetStringINNOE(smlVersion.Link, &dbSMLVersion.Link) + SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog) + SetDateINN(smlVersion.Date, &dbSMLVersion.Date) + postgres.Save(newCtx, &dbSMLVersion) return DBSMLVersionToGenerated(dbSMLVersion), nil From 66ee7db8b1cfdb890a404435895e3952e16ebc1f Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sun, 20 Aug 2023 16:23:01 +0300 Subject: [PATCH 14/25] fix: don't automatically preload version targets --- db/postgres/postgres_types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/postgres/postgres_types.go b/db/postgres/postgres_types.go index 8f865852..0635ffdc 100644 --- a/db/postgres/postgres_types.go +++ b/db/postgres/postgres_types.go @@ -87,7 +87,7 @@ type Version struct { SMLVersion string `gorm:"type:varchar(16)"` Version string `gorm:"type:varchar(16)"` ModID string - Targets []VersionTarget `gorm:"foreignKey:VersionID;preload:true"` + Targets []VersionTarget `gorm:"foreignKey:VersionID"` Hotness uint Downloads uint Denied bool `gorm:"default:false;not null"` @@ -120,7 +120,7 @@ type SMLVersion struct { Stability string `sql:"type:version_stability"` Link string Changelog string - Targets []SMLVersionTarget `gorm:"foreignKey:VersionID;preload:true"` + Targets []SMLVersionTarget `gorm:"foreignKey:VersionID"` SatisfactoryVersion int } From 2eb02630b6c62a2290122eda382d9f452ac39066 Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sun, 20 Aug 2023 17:55:37 +0300 Subject: [PATCH 15/25] chore: simplify Version.Link resolver --- gql/resolver_versions.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index 8454add1..4a775acc 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -334,14 +334,12 @@ func (r *versionResolver) Link(ctx context.Context, obj *generated.Version) (str wrapper, _ := WrapQueryTrace(ctx, "Version.link") defer wrapper.end() - link := "/v1/version/" + obj.ID + "/download" - windowsTarget := findWindowsTarget(obj) if windowsTarget != nil { - link, _ = r.VersionTarget().Link(ctx, windowsTarget) + return r.VersionTarget().Link(ctx, windowsTarget) } - return link, nil + return "/v1/version/" + obj.ID + "/download", nil } func (r *versionResolver) Mod(ctx context.Context, obj *generated.Version) (*generated.Mod, error) { From 9bd70d2c0b78b1adfdef887fa678261069089a54 Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sun, 20 Aug 2023 17:56:16 +0300 Subject: [PATCH 16/25] feat: return error message when uploading legacy mod formats --- gql/versions.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gql/versions.go b/gql/versions.go index 7fa1e8bb..fa979135 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -122,7 +122,10 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &dbVersion) } - // TODO: Should legacy plugins be supported? + if modInfo.Type != validation.MultiTargetUEPlugin { + return nil, errors.New("mods must be in the multi-target format") + } + targets := make([]*postgres.VersionTarget, 0) for _, target := range modInfo.Targets { From 7d39754c4d7940feab7a4b20d439e7325e1d78fe Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sun, 20 Aug 2023 21:40:21 +0300 Subject: [PATCH 17/25] chore: lint --- gql/resolver_versions.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index 4a775acc..4d2d216e 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -336,7 +336,8 @@ func (r *versionResolver) Link(ctx context.Context, obj *generated.Version) (str windowsTarget := findWindowsTarget(obj) if windowsTarget != nil { - return r.VersionTarget().Link(ctx, windowsTarget) + link, _ := r.VersionTarget().Link(ctx, windowsTarget) + return link, nil } return "/v1/version/" + obj.ID + "/download", nil From 6d7c4f3dc2cbaca5c0ea5758ad4976a6fc63d63f Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Sun, 20 Aug 2023 22:23:39 +0300 Subject: [PATCH 18/25] feat: migrate existing mods to use targets --- migrations/sql/000022_update_mod_targets.down.sql | 6 ++++-- migrations/sql/000022_update_mod_targets.up.sql | 2 ++ .../000023_migrate_versions_to_targets.down.sql | 15 +++++++++++++++ .../sql/000023_migrate_versions_to_targets.up.sql | 10 ++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 migrations/sql/000023_migrate_versions_to_targets.down.sql create mode 100644 migrations/sql/000023_migrate_versions_to_targets.up.sql diff --git a/migrations/sql/000022_update_mod_targets.down.sql b/migrations/sql/000022_update_mod_targets.down.sql index fb401fb8..6a196931 100644 --- a/migrations/sql/000022_update_mod_targets.down.sql +++ b/migrations/sql/000022_update_mod_targets.down.sql @@ -1,3 +1,5 @@ +-- ID generation -- +-- This is not as random as the original ID, but it should be good enough -- Create or replace function update_mod_platform_down_random_string(length integer) returns text as $$ declare @@ -15,6 +17,7 @@ begin end; $$ language plpgsql; +-- Mod version targets -- ALTER TABLE version_targets RENAME TO mod_archs; ALTER TABLE mod_archs @@ -24,7 +27,6 @@ ALTER TABLE mod_archs ALTER TABLE mod_archs ADD COLUMN id varchar(14); --- This is not as random as the original ID, but it should be good enough UPDATE mod_archs SET id = update_mod_platform_down_random_string(14) WHERE true; ALTER TABLE mod_archs @@ -37,6 +39,7 @@ ALTER TABLE mod_archs CREATE INDEX IF NOT EXISTS idx_mod_arch_id ON mod_archs (mod_version_arch_id, platform); +-- SML version targets -- ALTER TABLE sml_version_targets RENAME TO sml_archs; ALTER TABLE sml_archs @@ -46,7 +49,6 @@ ALTER TABLE sml_archs ALTER TABLE sml_archs ADD COLUMN id varchar(14); --- This is not as random as the original ID, but it should be good enough UPDATE sml_archs SET id = update_mod_platform_down_random_string(14) WHERE true; ALTER TABLE sml_archs diff --git a/migrations/sql/000022_update_mod_targets.up.sql b/migrations/sql/000022_update_mod_targets.up.sql index 23bfc8bb..687af423 100644 --- a/migrations/sql/000022_update_mod_targets.up.sql +++ b/migrations/sql/000022_update_mod_targets.up.sql @@ -1,3 +1,4 @@ +-- Mod version targets -- ALTER TABLE mod_archs RENAME TO version_targets; DROP INDEX idx_mod_arch_id; @@ -15,6 +16,7 @@ ALTER TABLE version_targets ALTER TABLE sml_archs RENAME TO sml_version_targets; +-- SML version targets -- DROP INDEX idx_sml_archs_id; ALTER TABLE sml_version_targets diff --git a/migrations/sql/000023_migrate_versions_to_targets.down.sql b/migrations/sql/000023_migrate_versions_to_targets.down.sql new file mode 100644 index 00000000..702a1db7 --- /dev/null +++ b/migrations/sql/000023_migrate_versions_to_targets.down.sql @@ -0,0 +1,15 @@ +--Mod Targets-- +DELETE FROM version_targets + USING versions + WHERE version_targets.version_id = versions.id AND + version_targets.target_name = 'Windows' AND + version_targets.key = versions.key AND + version_targets.hash = versions.hash AND + version_targets.size = versions.size; + +--SML Targets-- +DELETE FROM sml_version_targets + USING sml_versions + WHERE sml_version_targets.version_id = sml_versions.id AND + sml_version_targets.target_name = 'Windows' AND + sml_version_targets.link = replace(sml_versions.link, '/tag/', '/download/') || '/SML.zip'; diff --git a/migrations/sql/000023_migrate_versions_to_targets.up.sql b/migrations/sql/000023_migrate_versions_to_targets.up.sql new file mode 100644 index 00000000..0cfbef4c --- /dev/null +++ b/migrations/sql/000023_migrate_versions_to_targets.up.sql @@ -0,0 +1,10 @@ +-- Mod Targets -- +INSERT INTO version_targets (version_id, target_name, key, hash, size) +SELECT id, 'Windows', key, hash, size +FROM versions; + +-- SML Targets -- +INSERT INTO sml_version_targets (version_id, target_name, link) +SELECT id, 'Windows', replace(link, '/tag/', '/download/') || '/SML.zip' +FROM sml_versions +WHERE version LIKE '3%'; \ No newline at end of file From 295fa9f0edaeb211f97284a568d06e7f58f89eef Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Mon, 21 Aug 2023 15:22:23 +0300 Subject: [PATCH 19/25] fix: skip migrating versions that already have targets --- migrations/sql/000023_migrate_versions_to_targets.up.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/migrations/sql/000023_migrate_versions_to_targets.up.sql b/migrations/sql/000023_migrate_versions_to_targets.up.sql index 0cfbef4c..94e32af7 100644 --- a/migrations/sql/000023_migrate_versions_to_targets.up.sql +++ b/migrations/sql/000023_migrate_versions_to_targets.up.sql @@ -1,10 +1,13 @@ -- Mod Targets -- INSERT INTO version_targets (version_id, target_name, key, hash, size) SELECT id, 'Windows', key, hash, size -FROM versions; +FROM versions +WHERE NOT EXISTS(SELECT 1 FROM version_targets WHERE version_targets.version_id = versions.id) +ON CONFLICT DO NOTHING; -- SML Targets -- INSERT INTO sml_version_targets (version_id, target_name, link) SELECT id, 'Windows', replace(link, '/tag/', '/download/') || '/SML.zip' FROM sml_versions -WHERE version LIKE '3%'; \ No newline at end of file +WHERE version LIKE '3%' +ON CONFLICT DO NOTHING; \ No newline at end of file From 1c4ce3218a3ea39532a0a9ac848c9d15e2c6d5e4 Mon Sep 17 00:00:00 2001 From: Mircea Roata Date: Mon, 21 Aug 2023 17:26:57 +0300 Subject: [PATCH 20/25] chore: remove some debug code --- gql/versions.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gql/versions.go b/gql/versions.go index fa979135..c5eec1b8 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -6,7 +6,6 @@ import ( "io" "time" - "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -45,10 +44,8 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI modInfo, err := validation.ExtractModInfo(ctx, fileData, true, true, mod.ModReference) if err != nil { - spew.Dump(err) - l.Err(err).Msg("failed extracting mod info") storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) - return nil, err + return nil, errors.Wrap(err, "failed extracting mod info") } if modInfo.ModReference != mod.ModReference { From 4a45e6ec7e36023add0525b7e84e6955bff631bd Mon Sep 17 00:00:00 2001 From: mircearoata Date: Tue, 19 Sep 2023 23:04:50 +0200 Subject: [PATCH 21/25] chore: temporarily require single-target format --- gql/versions.go | 74 ++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/gql/versions.go b/gql/versions.go index c5eec1b8..50bde58e 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -53,6 +53,11 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI return nil, errors.New("data.json mod_reference does not match mod reference") } + if modInfo.Type != validation.UEPlugin { + storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + return nil, errors.New("only UE Plugin mods are currently supported") + } + versionMajor := int(modInfo.Semver.Major()) versionMinor := int(modInfo.Semver.Minor()) versionPatch := int(modInfo.Semver.Patch()) @@ -119,44 +124,42 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &dbVersion) } - if modInfo.Type != validation.MultiTargetUEPlugin { - return nil, errors.New("mods must be in the multi-target format") - } + if modInfo.Type == validation.MultiTargetUEPlugin { + targets := make([]*postgres.VersionTarget, 0) - targets := make([]*postgres.VersionTarget, 0) + for _, target := range modInfo.Targets { + dbVersionTarget := &postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: target, + } - for _, target := range modInfo.Targets { - dbVersionTarget := &postgres.VersionTarget{ - VersionID: dbVersion.ID, - TargetName: target, + postgres.Save(ctx, dbVersionTarget) + + targets = append(targets, dbVersionTarget) } - postgres.Save(ctx, dbVersionTarget) + separateSuccess := true + for _, target := range targets { + log.Info().Str("target", target.TargetName).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") + success, key, hash, size := storage.SeparateModTarget(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, target.TargetName) - targets = append(targets, dbVersionTarget) - } + if !success { + separateSuccess = false + break + } - separateSuccess := true - for _, target := range targets { - log.Info().Str("target", target.TargetName).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") - success, key, hash, size := storage.SeparateModTarget(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, target.TargetName) + target.Key = key + target.Hash = hash + target.Size = size - if !success { - separateSuccess = false - break + postgres.Save(ctx, target) } - target.Key = key - target.Hash = hash - target.Size = size + if !separateSuccess { + removeMod(ctx, modInfo, mod, dbVersion) - postgres.Save(ctx, target) - } - - if !separateSuccess { - removeMod(ctx, modInfo, mod, dbVersion) - - return nil, errors.New("failed to separate mod") + return nil, errors.New("failed to separate mod") + } } success, key := storage.RenameVersion(ctx, mod.ID, mod.Name, versionID, modInfo.Version) @@ -167,6 +170,18 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI return nil, errors.New("failed to upload mod") } + if modInfo.Type == validation.UEPlugin { + dbVersionTarget := &postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: "Windows", + Key: key, + Hash: *dbVersion.Hash, + Size: *dbVersion.Size, + } + + postgres.Save(ctx, dbVersionTarget) + } + dbVersion.Key = key postgres.Save(ctx, &dbVersion) postgres.Save(ctx, &mod) @@ -221,6 +236,9 @@ func removeMod(ctx context.Context, modInfo *validation.ModInfo, mod *postgres.M postgres.DeleteForced(ctx, &dbVersionTarget) } + // For UEPlugin mods, a Windows target is created. + // However, that happens after the last possible call to this function, therefore we can ignore it + postgres.DeleteForced(ctx, &dbVersion) storage.DeleteMod(ctx, mod.ID, mod.Name, dbVersion.ID) From b9ca8cff7a571245335be4945d1abadc8cb17b79 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Thu, 19 Oct 2023 21:40:10 +0300 Subject: [PATCH 22/25] feat: new metadata parser, sml engine versions, minimal mod version REST endpoint --- .envrc | 1 + README.md | 8 ++ config.sample.json | 7 +- config/config.go | 2 + db/postgres/postgres_types.go | 15 +++ db/postgres/version.go | 18 +++ docker-compose-dev.yml | 7 +- gql/gql_types.go | 1 + gql/resolver_sml_versions.go | 2 + gql/versions.go | 9 +- .../000024_add_sml_engine_version.down.sql | 2 + .../sql/000024_add_sml_engine_version.up.sql | 2 + nodes/mod.go | 27 +++++ nodes/mod_types.go | 82 ++++++++++++-- nodes/routes.go | 2 + proto/parser/.gitignore | 1 + proto/parser/parser.proto | 17 +++ schemas/sml_version.graphql | 3 + shell.nix | 11 ++ storage/b2.go | 4 + storage/s3.go | 29 +++++ storage/storage.go | 41 +++++++ storage/wasabi.go | 4 + tools.go | 1 + util/flags.go | 13 +++ validation/extractor.go | 92 +++++++++++++++ validation/validation.go | 107 +++++++++++++----- validation/virustotal.go | 3 +- 28 files changed, 465 insertions(+), 46 deletions(-) create mode 100644 .envrc create mode 100644 migrations/sql/000024_add_sml_engine_version.down.sql create mode 100644 migrations/sql/000024_add_sml_engine_version.up.sql create mode 100644 proto/parser/.gitignore create mode 100644 proto/parser/parser.proto create mode 100644 shell.nix create mode 100644 util/flags.go create mode 100644 validation/extractor.go diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..65326bb6 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix \ No newline at end of file diff --git a/README.md b/README.md index 269a5934..220b9da0 100755 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Main configuration options: The config format can be seen in `config/config.go` (each dot means a new level of nesting). +After startup requires the following minio commands to be executed: + +```shell +mc alias set local http://localhost:9000 minio minio123 +mc admin user svcacct add local minio --access-key REPLACE_ME_KEY --secret-key REPLACE_ME_SECRET +mc anonymous set public local/smr +``` + ## Contributing Before contributing, please run the [linter](https://golangci-lint.run/) to ensure the code is clean and well-formed: diff --git a/config.sample.json b/config.sample.json index 60037688..f865d27b 100644 --- a/config.sample.json +++ b/config.sample.json @@ -27,7 +27,8 @@ "key": "REPLACE_ME_KEY", "secret": "REPLACE_ME_SECRET", "endpoint": "http://localhost:9000", - "base_url": "http://localhost:9000" + "base_url": "http://localhost:9000", + "keypath": "%s/%s/%s" }, "oauth": { @@ -52,5 +53,9 @@ "frontend": { "url": "http://localhost:4200" + }, + + "feature_flags": { + "allow_multi_target_upload": false } } \ No newline at end of file diff --git a/config/config.go b/config/config.go index ad261198..8e0277c9 100644 --- a/config/config.go +++ b/config/config.go @@ -100,4 +100,6 @@ func initializeDefaults() { viper.SetDefault("frontend.url", "") viper.SetDefault("virustotal.key", "") + + viper.SetDefault("feature_flags.allow_multi_target_upload", false) } diff --git a/db/postgres/postgres_types.go b/db/postgres/postgres_types.go index 0635ffdc..89433031 100644 --- a/db/postgres/postgres_types.go +++ b/db/postgres/postgres_types.go @@ -94,6 +94,20 @@ type Version struct { Approved bool `gorm:"default:false;not null"` } +type TinyVersion struct { + Hash *string + Size *int64 + SMRModel + SMLVersion string `gorm:"type:varchar(16)"` + Version string `gorm:"type:varchar(16)"` + Arch []VersionTarget `gorm:"foreignKey:ModVersionID;preload:true"` + Dependencies []VersionDependency `gorm:"foreignKey:VersionID"` +} + +func (TinyVersion) TableName() string { + return "versions" +} + type Guide struct { SMRModel Name string `gorm:"type:varchar(50)"` @@ -120,6 +134,7 @@ type SMLVersion struct { Stability string `sql:"type:version_stability"` Link string Changelog string + EngineVersion string Targets []SMLVersionTarget `gorm:"foreignKey:VersionID"` SatisfactoryVersion int } diff --git a/db/postgres/version.go b/db/postgres/version.go index e3f99fc1..ccf95c30 100644 --- a/db/postgres/version.go +++ b/db/postgres/version.go @@ -87,6 +87,24 @@ func GetModVersions(ctx context.Context, modID string, limit int, offset int, or return versions } +func GetAllModVersionsWithDependencies(ctx context.Context, modID string) []TinyVersion { + cacheKey := "GetAllModVersionsWithDependencies_" + modID + if versions, ok := dbCache.Get(cacheKey); ok { + return versions.([]TinyVersion) + } + + var versions []TinyVersion + DBCtx(ctx).Debug(). + Preload("Dependencies"). + Preload("Arch"). + Where("approved = ? AND denied = ?", true, false). + Find(&versions, "mod_id = ?", modID) + + dbCache.Set(cacheKey, versions, cache.DefaultExpiration) + + return versions +} + func GetModVersionsNew(ctx context.Context, modID string, filter *models.VersionFilter, unapproved bool) []Version { hash, err := filter.Hash() cacheKey := "" diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3327f6b2..6b302312 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,4 +25,9 @@ services: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio123 MINIO_ACCESS_KEY: REPLACE_ME_KEY - MINIO_SECRET_KEY: REPLACE_ME_SECRET \ No newline at end of file + MINIO_SECRET_KEY: REPLACE_ME_SECRET + + pak_parser: + image: ghcr.io/vilsol/ficsit-pak-parser:v0.0.3 + ports: + - 50051:50051 \ No newline at end of file diff --git a/gql/gql_types.go b/gql/gql_types.go index 0fa9756f..6d8d3960 100644 --- a/gql/gql_types.go +++ b/gql/gql_types.go @@ -139,6 +139,7 @@ func DBSMLVersionToGenerated(smlVersion *postgres.SMLVersion) *generated.SMLVers Date: smlVersion.Date.Format(time.RFC3339Nano), UpdatedAt: smlVersion.UpdatedAt.Format(time.RFC3339Nano), CreatedAt: smlVersion.CreatedAt.Format(time.RFC3339Nano), + EngineVersion: smlVersion.EngineVersion, } } diff --git a/gql/resolver_sml_versions.go b/gql/resolver_sml_versions.go index b8102ff8..a5859f55 100644 --- a/gql/resolver_sml_versions.go +++ b/gql/resolver_sml_versions.go @@ -36,6 +36,7 @@ func (r *mutationResolver) CreateSMLVersion(ctx context.Context, smlVersion gene Link: smlVersion.Link, Changelog: smlVersion.Changelog, Date: date, + EngineVersion: smlVersion.EngineVersion, } resultSMLVersion, err := postgres.CreateSMLVersion(newCtx, dbSMLVersion) @@ -101,6 +102,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st SetStringINNOE(smlVersion.Link, &dbSMLVersion.Link) SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog) SetDateINN(smlVersion.Date, &dbSMLVersion.Date) + SetStringINNOE(smlVersion.EngineVersion, &dbSMLVersion.EngineVersion) postgres.Save(newCtx, &dbSMLVersion) diff --git a/gql/versions.go b/gql/versions.go index 50bde58e..de9a28b1 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -53,9 +53,14 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI return nil, errors.New("data.json mod_reference does not match mod reference") } - if modInfo.Type != validation.UEPlugin { + if modInfo.Type == validation.DataJSON { storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) - return nil, errors.New("only UE Plugin mods are currently supported") + return nil, errors.New("data.json mods are obsolete and not allowed") + } + + if modInfo.Type == validation.MultiTargetUEPlugin && !util.FlagEnabled(util.FeatureFlagAllowMultiTargetUpload) { + storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + return nil, errors.New("multi-target mods are not allowed") } versionMajor := int(modInfo.Semver.Major()) diff --git a/migrations/sql/000024_add_sml_engine_version.down.sql b/migrations/sql/000024_add_sml_engine_version.down.sql new file mode 100644 index 00000000..87f13a5e --- /dev/null +++ b/migrations/sql/000024_add_sml_engine_version.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE sml_versions + DROP COLUMN engine_version; \ No newline at end of file diff --git a/migrations/sql/000024_add_sml_engine_version.up.sql b/migrations/sql/000024_add_sml_engine_version.up.sql new file mode 100644 index 00000000..3a018fef --- /dev/null +++ b/migrations/sql/000024_add_sml_engine_version.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE sml_versions + ADD COLUMN IF NOT EXISTS engine_version varchar(16) default '4.26'; \ No newline at end of file diff --git a/nodes/mod.go b/nodes/mod.go index d589ebc1..4fdeeb3d 100644 --- a/nodes/mod.go +++ b/nodes/mod.go @@ -328,3 +328,30 @@ func downloadModVersionTarget(c echo.Context) error { return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key)) } + +// @Summary Retrieve all Mod Versions +// @Tags Mod +// @Description Retrieve all mod versions by mod ID +// @Accept json +// @Produce json +// @Param modId path string true "Mod ID" +// @Success 200 +// @Router /mod/{modId}/versions/all [get] +func getAllModVersions(c echo.Context) (interface{}, *ErrorResponse) { + modID := c.Param("modId") + + mod := postgres.GetModByID(c.Request().Context(), modID) + + if mod == nil { + return nil, &ErrorModNotFound + } + + versions := postgres.GetAllModVersionsWithDependencies(c.Request().Context(), mod.ID) + + converted := make([]*Version, len(versions)) + for k, v := range versions { + converted[k] = TinyVersionToVersion(&v) + } + + return converted, nil +} diff --git a/nodes/mod_types.go b/nodes/mod_types.go index 6cb453b0..bd5c2660 100644 --- a/nodes/mod_types.go +++ b/nodes/mod_types.go @@ -48,16 +48,60 @@ func ModToMod(mod *postgres.Mod, short bool) *Mod { } type Version struct { - UpdatedAt time.Time `json:"updated_at"` - CreatedAt time.Time `json:"created_at"` - ID string `json:"id"` - Version string `json:"version"` - SMLVersion string `json:"sml_version"` - Changelog string `json:"changelog"` - Stability string `json:"stability"` - ModID string `json:"mod_id"` - Downloads uint `json:"downloads"` - Approved bool `json:"approved"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + ID string `json:"id,omitempty"` + Version string `json:"version,omitempty"` + SMLVersion string `json:"sml_version,omitempty"` + Changelog string `json:"changelog,omitempty"` + Stability string `json:"stability,omitempty"` + ModID string `json:"mod_id,omitempty"` + Downloads uint `json:"downloads,omitempty"` + Approved bool `json:"approved,omitempty"` + Dependencies []VersionDependency `json:"dependencies,omitempty"` + Arch []VersionTarget `json:"arch,omitempty"` +} + +type VersionDependency struct { + ModID string `json:"mod_id"` + Condition string `json:"condition"` + Optional bool `json:"optional"` +} + +type VersionTarget struct { + VersionID string `json:"version_id"` + TargetName string `json:"target_name"` + Key string `json:"key"` + Hash string `json:"hash"` + Size int64 `json:"size"` +} + +func TinyVersionToVersion(version *postgres.TinyVersion) *Version { + var dependencies []VersionDependency + if version.Dependencies != nil { + dependencies = make([]VersionDependency, len(version.Dependencies)) + for i, v := range version.Dependencies { + dependencies[i] = VersionDependencyToVersionDependency(v) + } + } + + var archs []VersionTarget + if version.Arch != nil { + archs = make([]VersionTarget, len(version.Arch)) + for i, v := range version.Arch { + archs[i] = VersionArchToVersionArch(v) + } + } + + return &Version{ + UpdatedAt: version.UpdatedAt, + CreatedAt: version.CreatedAt, + ID: version.ID, + Version: version.Version, + SMLVersion: version.SMLVersion, + Dependencies: dependencies, + Arch: archs, + } } func VersionToVersion(version *postgres.Version) *Version { @@ -75,6 +119,24 @@ func VersionToVersion(version *postgres.Version) *Version { } } +func VersionDependencyToVersionDependency(version postgres.VersionDependency) VersionDependency { + return VersionDependency{ + ModID: version.ModID, + Condition: version.Condition, + Optional: version.Optional, + } +} + +func VersionArchToVersionArch(version postgres.VersionTarget) VersionTarget { + return VersionTarget{ + VersionID: version.VersionID, + TargetName: version.TargetName, + Key: version.Key, + Hash: version.Hash, + Size: version.Size, + } +} + type ModUser struct { UserID string `json:"user_id"` Role string `json:"role"` diff --git a/nodes/routes.go b/nodes/routes.go index e84b94fb..b6c6cf6d 100755 --- a/nodes/routes.go +++ b/nodes/routes.go @@ -13,6 +13,8 @@ func RegisterModRoutes(router *echo.Group) { router.GET("/:modId/versions", dataWrapper(getModVersions)) router.GET("/:modId/authors", dataWrapper(getModAuthors)) + router.GET("/:modId/versions/all", dataWrapper(getAllModVersions)) + router.GET("/:modId/versions/:versionId", dataWrapper(getModVersion)) router.GET("/:modId/versions/:versionId/download", downloadModVersion) router.GET("/:modId/versions/:versionId/:target/download", downloadModVersionTarget) diff --git a/proto/parser/.gitignore b/proto/parser/.gitignore new file mode 100644 index 00000000..9b0b440d --- /dev/null +++ b/proto/parser/.gitignore @@ -0,0 +1 @@ +*.pb.go \ No newline at end of file diff --git a/proto/parser/parser.proto b/proto/parser/parser.proto new file mode 100644 index 00000000..913d5f6a --- /dev/null +++ b/proto/parser/parser.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +option go_package = "github.com/satisfactorymodding/smr-api/proto/parser"; + +service Parser { + rpc Parse (ParseRequest) returns (stream AssetResponse); +} + +message ParseRequest { + bytes zip_data = 1; + string engine_version = 2; +} + +message AssetResponse { + string path = 1; + bytes data = 2; +} \ No newline at end of file diff --git a/schemas/sml_version.graphql b/schemas/sml_version.graphql index 3e300aa8..9299c3fe 100755 --- a/schemas/sml_version.graphql +++ b/schemas/sml_version.graphql @@ -12,6 +12,7 @@ type SMLVersion { changelog: String! date: Date! bootstrap_version: String + engine_version: String! updated_at: Date! created_at: Date! @@ -47,6 +48,7 @@ input NewSMLVersion { changelog: String! date: Date! bootstrap_version: String + engine_version: String! } input UpdateSMLVersion { @@ -58,6 +60,7 @@ input UpdateSMLVersion { changelog: String date: Date bootstrap_version: String + engine_version: String } input NewSMLVersionTarget { diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..216a4f80 --- /dev/null +++ b/shell.nix @@ -0,0 +1,11 @@ +{ pkgs ? import {} }: + +pkgs.mkShell { + nativeBuildInputs = with pkgs.buildPackages; [ + libwebp + go + protobuf + protoc-gen-go-grpc + minio-client + ]; +} diff --git a/storage/b2.go b/storage/b2.go index 10ec7ad2..1672ece2 100644 --- a/storage/b2.go +++ b/storage/b2.go @@ -222,3 +222,7 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) { ContentType: data.ContentType, }, nil } + +func (b2o *B2) List(key string) ([]Object, error) { + return nil, errors.New("Unsupported") +} diff --git a/storage/s3.go b/storage/s3.go index d8c09c71..b9d97522 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -203,6 +203,15 @@ func (s3o *S3) Delete(key string) error { } if len(objects) == 0 { + _, err = s3o.S3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(viper.GetString("storage.bucket")), + Key: aws.String(cleanedKey), + }) + + if err != nil { + return errors.Wrap(err, "failed to delete objects") + } + return nil } @@ -237,3 +246,23 @@ func (s3o *S3) Meta(key string) (*ObjectMeta, error) { ContentType: data.ContentType, }, nil } + +func (s3o *S3) List(prefix string) ([]Object, error) { + objects, err := s3o.S3Client.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(viper.GetString("storage.bucket")), + Prefix: aws.String(prefix), + }) + if err != nil { + return nil, errors.Wrap(err, "failed to list objects") + } + + out := make([]Object, len(objects.Contents)) + for i, obj := range objects.Contents { + out[i] = Object{ + Key: obj.Key, + LastModified: obj.LastModified, + } + } + + return out, nil +} diff --git a/storage/storage.go b/storage/storage.go index 180cf37f..5597aaa7 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "strings" + "time" "github.com/avast/retry-go/v3" "github.com/pkg/errors" @@ -27,6 +28,7 @@ type Storage interface { Rename(from string, to string) error Delete(key string) error Meta(key string) (*ObjectMeta, error) + List(key string) ([]Object, error) } type ObjectMeta struct { @@ -34,6 +36,11 @@ type ObjectMeta struct { ContentType *string } +type Object struct { + Key *string + LastModified *time.Time +} + type Config struct { Type string `json:"type"` Bucket string `json:"bucket"` @@ -436,3 +443,37 @@ func copyModFileToArchZip(file *zip.File, zipWriter *zip.Writer, newName string) return nil } + +func DeleteOldModAssets(modReference string, before time.Time) { + list, err := storage.List(fmt.Sprintf("/assets/mods/%s", modReference)) + if err != nil { + log.Err(err).Msg("failed to list assets") + return + } + + for _, object := range list { + if object.Key == nil { + continue + } + + if object.LastModified == nil || object.LastModified.Before(before) { + if err := storage.Delete(*object.Key); err != nil { + log.Err(err).Str("key", *object.Key).Msg("failed deleting old asset") + return + } + } + } +} + +func UploadModAsset(ctx context.Context, modReference string, path string, data []byte) { + if storage == nil { + return + } + + key := fmt.Sprintf("/assets/mods/%s/%s", modReference, strings.TrimPrefix(path, "/")) + + _, err := storage.Put(ctx, key, bytes.NewReader(data)) + if err != nil { + log.Err(err).Str("path", path).Msg("failed to upload mod asset") + } +} diff --git a/storage/wasabi.go b/storage/wasabi.go index 3a0a015a..c35b864e 100644 --- a/storage/wasabi.go +++ b/storage/wasabi.go @@ -118,3 +118,7 @@ func (wasabi *Wasabi) Delete(key string) error { func (wasabi *Wasabi) Meta(key string) (*ObjectMeta, error) { return nil, errors.New("Unsupported") } + +func (wasabi *Wasabi) List(key string) ([]Object, error) { + return nil, errors.New("Unsupported") +} diff --git a/tools.go b/tools.go index 0a80bbc3..fd71acc8 100644 --- a/tools.go +++ b/tools.go @@ -8,3 +8,4 @@ import _ "github.com/swaggo/swag/cmd/swag" //go:generate go run github.com/99designs/gqlgen generate //go:generate go run github.com/swaggo/swag/cmd/swag init --generalInfo cmd/api/serve.go +//go:generate protoc -I./proto --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative proto/parser/parser.proto diff --git a/util/flags.go b/util/flags.go new file mode 100644 index 00000000..196968c2 --- /dev/null +++ b/util/flags.go @@ -0,0 +1,13 @@ +package util + +import "github.com/spf13/viper" + +type FeatureFlag string + +const ( + FeatureFlagAllowMultiTargetUpload = "allow_multi_target_upload" +) + +func FlagEnabled(flag FeatureFlag) bool { + return viper.GetBool("feature_flags." + string(flag)) +} diff --git a/validation/extractor.go b/validation/extractor.go new file mode 100644 index 00000000..9526e7b3 --- /dev/null +++ b/validation/extractor.go @@ -0,0 +1,92 @@ +package validation + +import ( + "encoding/json" + "fmt" + "regexp" +) + +func ExtractMetadata(raw []byte) (map[string]map[string][]interface{}, error) { + meta := make(map[string][]map[string]interface{}) + + if err := json.Unmarshal(raw, &meta); err != nil { + return nil, fmt.Errorf("failed extracting meta: %w", err) + } + + out := make(map[string]map[string][]interface{}) + + for fileName, data := range meta { + bpTypes := make(map[string]string) + for i, obj := range data { + if i == 0 && obj["Type"] != "BlueprintGeneratedClass" { + break + } + + if obj["Type"] == "BlueprintGeneratedClass" { + superName := obj["SuperStruct"].(map[string]interface{})["ObjectName"].(string) + _, objName := splitName(superName) + bpTypes[obj["Name"].(string)] = objName + continue + } + + if obj["Properties"] != nil { + classType := obj["Type"].(string) + if _, ok := ignoredClasses[classType]; ok { + continue + } + + if _, ok := out[fileName]; !ok { + out[fileName] = make(map[string][]interface{}) + } + + typ := bpTypes[classType] + if typ == "" { + typ = classType + } + + out[fileName][typ] = append(out[fileName][typ], rewriteRecursive(obj["Properties"])) + } + } + } + + return out, nil +} + +var objNameRegex = regexp.MustCompile(`^(.+?)'(.+?)'$`) + +func splitName(n string) (string, string) { + matches := objNameRegex.FindStringSubmatch(n) + return matches[1], matches[2] +} + +func rewriteRecursive(obj interface{}) interface{} { + switch b := obj.(type) { + case map[string]interface{}: + if mapHas("CultureInvariantString", b) { + return b["CultureInvariantString"] + } else if mapHas("ObjectName", b) && mapHas("ObjectPath", b) { + _, val := splitName(b["ObjectName"].(string)) + return val + } else if mapHas("AssetPathName", b) && mapHas("SubPathString", b) { + return b["AssetPathName"] + } else { + newOut := make(map[string]interface{}) + for k, v := range b { + newOut[k] = rewriteRecursive(v) + } + return newOut + } + case []interface{}: + newOut := make([]interface{}, len(b)) + for i, v := range b { + newOut[i] = rewriteRecursive(v) + } + return newOut + } + return obj +} + +func mapHas(key string, mp map[string]interface{}) bool { + _, ok := mp[key] + return ok +} diff --git a/validation/validation.go b/validation/validation.go index d5b955d9..e02fa199 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -11,14 +11,21 @@ import ( "io" "path" "path/filepath" + "slices" "strconv" "strings" + "time" "github.com/Masterminds/semver/v3" - "github.com/Vilsol/ue4pak/parser" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/xeipuuv/gojsonschema" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/satisfactorymodding/smr-api/db/postgres" + "github.com/satisfactorymodding/smr-api/proto/parser" + "github.com/satisfactorymodding/smr-api/storage" ) var AllowedTargets = []string{"Windows", "WindowsServer", "LinuxServer"} @@ -127,39 +134,77 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal if withMetadata { // Extract all possible metadata - modInfo.Metadata = make([]map[string]map[string][]interface{}, 0) - for _, obj := range modInfo.Objects { - if strings.ToLower(obj.Type) == "pak" { - for _, archiveFile := range archive.File { - if obj.Path == archiveFile.Name { - data, err := archiveFile.Open() - if err != nil { - log.Err(err).Msg("failed opening archive file") - break - } - - pakData, err := io.ReadAll(data) - if err != nil { - log.Err(err).Msg("failed reading archive file") - break - } - - reader := &parser.PakByteReader{ - Bytes: pakData, - } - - pak, err := AttemptExtractDataFromPak(ctx, reader) - if err != nil { - log.Err(err).Msg("failed parsing archive file") - break - } - - modInfo.Metadata = append(modInfo.Metadata, pak) - break - } + conn, err := grpc.Dial("localhost:50051", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, errors.Wrap(err, "failed to connect to metadata server") + } + defer conn.Close() + + engineVersion := "4.26" + + //nolint + if postgres.DBCtx(nil) != nil { + smlVersions := postgres.GetSMLVersions(ctx, nil) + + // Sort decrementing by version + slices.SortFunc(smlVersions, func(a, b postgres.SMLVersion) int { + return semver.MustParse(b.Version).Compare(semver.MustParse(a.Version)) + }) + + for _, version := range smlVersions { + constraint, err := semver.NewConstraint(modInfo.SMLVersion) + if err != nil { + return nil, errors.Wrap(err, "failed to create semver constraint") + } + + if constraint.Check(semver.MustParse(version.Version)) { + engineVersion = version.EngineVersion + break + } + } + } + + parserClient := parser.NewParserClient(conn) + stream, err := parserClient.Parse(ctx, &parser.ParseRequest{ + ZipData: body, + EngineVersion: engineVersion, + }, + grpc.MaxCallSendMsgSize(1024*1024*1024), // 1GB + grpc.MaxCallRecvMsgSize(1024*1024*1024), // 1GB + ) + if err != nil { + return nil, errors.Wrap(err, "failed to parse mod") + } + + defer func(stream parser.Parser_ParseClient) { + err := stream.CloseSend() + if err != nil { + log.Ctx(ctx).Err(err).Msg("failed closing parser stream") + } + }(stream) + + beforeUpload := time.Now().Add(-time.Minute) + for { + asset, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + break } + return nil, errors.Wrap(err, "failed reading parser stream") } + + if asset.Path == "metadata.json" { + out, err := ExtractMetadata(asset.Data) + if err != nil { + return nil, err + } + modInfo.Metadata = append(modInfo.Metadata, out) + } + + storage.UploadModAsset(ctx, modInfo.ModReference, asset.GetPath(), asset.GetData()) } + + storage.DeleteOldModAssets(modInfo.ModReference, beforeUpload) } modInfo.Size = int64(len(body)) diff --git a/validation/virustotal.go b/validation/virustotal.go index e12f4eb6..bbfe8dc5 100644 --- a/validation/virustotal.go +++ b/validation/virustotal.go @@ -103,7 +103,8 @@ func scanFile(ctx context.Context, file io.Reader, name string) (bool, error) { return false, nil } - if *target.Attributes.Stats.Malicious > 0 || *target.Attributes.Stats.Suspicious > 0 { + // Why 1? Well because some company made a shitty AI and it flags random mods. + if *target.Attributes.Stats.Malicious > 1 || *target.Attributes.Stats.Suspicious > 1 { log.Error().Msgf("suspicious or malicious file found: %s", name) return false, nil } From c13a52bfa52a4c1aec3a5488e2a37d53e68c84f4 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Thu, 19 Oct 2023 21:43:47 +0300 Subject: [PATCH 23/25] chore: reorder go generate steps --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index fd71acc8..bf720453 100644 --- a/tools.go +++ b/tools.go @@ -6,6 +6,6 @@ package smr import _ "github.com/99designs/gqlgen" import _ "github.com/swaggo/swag/cmd/swag" +//go:generate protoc -I./proto --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative proto/parser/parser.proto //go:generate go run github.com/99designs/gqlgen generate //go:generate go run github.com/swaggo/swag/cmd/swag init --generalInfo cmd/api/serve.go -//go:generate protoc -I./proto --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative proto/parser/parser.proto From 550506750f568f2d0c03de37a55c8bab1124ea99 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Thu, 19 Oct 2023 22:15:26 +0300 Subject: [PATCH 24/25] chore: download protoc dependencies --- .github/workflows/build.yml | 15 ++++++++++++--- .github/workflows/release.yml | 5 ++++- Dockerfile | 6 ++++-- db/postgres/postgres_types.go | 2 +- db/postgres/version.go | 2 +- nodes/mod_types.go | 16 ++++++++-------- storage/b2.go | 2 +- validation/validation.go | 6 +++--- 8 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f06bb0ac..332f1398 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... @@ -39,7 +42,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... @@ -65,7 +71,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e548779..fc5a0eff 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,10 @@ jobs: fetch-depth: 0 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 diff --git a/Dockerfile b/Dockerfile index d1d66b82..83c9552d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ -FROM golang:1.18-alpine AS builder +FROM golang:1.19-alpine3.18 AS builder -RUN apk add --no-cache git build-base libpng-dev +RUN apk add --no-cache git build-base libpng-dev protoc +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 WORKDIR $GOPATH/src/github.com/satisfactorymodding/smr-api/ diff --git a/db/postgres/postgres_types.go b/db/postgres/postgres_types.go index 89433031..c09660ed 100644 --- a/db/postgres/postgres_types.go +++ b/db/postgres/postgres_types.go @@ -100,7 +100,7 @@ type TinyVersion struct { SMRModel SMLVersion string `gorm:"type:varchar(16)"` Version string `gorm:"type:varchar(16)"` - Arch []VersionTarget `gorm:"foreignKey:ModVersionID;preload:true"` + Targets []VersionTarget `gorm:"foreignKey:VersionID;preload:true"` Dependencies []VersionDependency `gorm:"foreignKey:VersionID"` } diff --git a/db/postgres/version.go b/db/postgres/version.go index ccf95c30..d70aee89 100644 --- a/db/postgres/version.go +++ b/db/postgres/version.go @@ -96,7 +96,7 @@ func GetAllModVersionsWithDependencies(ctx context.Context, modID string) []Tiny var versions []TinyVersion DBCtx(ctx).Debug(). Preload("Dependencies"). - Preload("Arch"). + Preload("Targets"). Where("approved = ? AND denied = ?", true, false). Find(&versions, "mod_id = ?", modID) diff --git a/nodes/mod_types.go b/nodes/mod_types.go index bd5c2660..88f7b82f 100644 --- a/nodes/mod_types.go +++ b/nodes/mod_types.go @@ -59,7 +59,7 @@ type Version struct { Downloads uint `json:"downloads,omitempty"` Approved bool `json:"approved,omitempty"` Dependencies []VersionDependency `json:"dependencies,omitempty"` - Arch []VersionTarget `json:"arch,omitempty"` + Targets []VersionTarget `json:"targets,omitempty"` } type VersionDependency struct { @@ -85,11 +85,11 @@ func TinyVersionToVersion(version *postgres.TinyVersion) *Version { } } - var archs []VersionTarget - if version.Arch != nil { - archs = make([]VersionTarget, len(version.Arch)) - for i, v := range version.Arch { - archs[i] = VersionArchToVersionArch(v) + var targets []VersionTarget + if version.Targets != nil { + targets = make([]VersionTarget, len(version.Targets)) + for i, v := range version.Targets { + targets[i] = VersionTargetToVersionTarget(v) } } @@ -100,7 +100,7 @@ func TinyVersionToVersion(version *postgres.TinyVersion) *Version { Version: version.Version, SMLVersion: version.SMLVersion, Dependencies: dependencies, - Arch: archs, + Targets: targets, } } @@ -127,7 +127,7 @@ func VersionDependencyToVersionDependency(version postgres.VersionDependency) Ve } } -func VersionArchToVersionArch(version postgres.VersionTarget) VersionTarget { +func VersionTargetToVersionTarget(version postgres.VersionTarget) VersionTarget { return VersionTarget{ VersionID: version.VersionID, TargetName: version.TargetName, diff --git a/storage/b2.go b/storage/b2.go index 1672ece2..a616608d 100644 --- a/storage/b2.go +++ b/storage/b2.go @@ -224,5 +224,5 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) { } func (b2o *B2) List(key string) ([]Object, error) { - return nil, errors.New("Unsupported") + return nil, nil // no-op } diff --git a/validation/validation.go b/validation/validation.go index e02fa199..813eb35f 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -11,7 +11,7 @@ import ( "io" "path" "path/filepath" - "slices" + "sort" "strconv" "strings" "time" @@ -147,8 +147,8 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal smlVersions := postgres.GetSMLVersions(ctx, nil) // Sort decrementing by version - slices.SortFunc(smlVersions, func(a, b postgres.SMLVersion) int { - return semver.MustParse(b.Version).Compare(semver.MustParse(a.Version)) + sort.Slice(smlVersions, func(a, b int) bool { + return semver.MustParse(smlVersions[a].Version).Compare(semver.MustParse(smlVersions[b].Version)) > 0 }) for _, version := range smlVersions { From 86bc1f84e97b7e343863f24d473376e26bff6e25 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Thu, 19 Oct 2023 22:22:41 +0300 Subject: [PATCH 25/25] chore: field alignment --- nodes/mod_types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes/mod_types.go b/nodes/mod_types.go index 88f7b82f..25cab078 100644 --- a/nodes/mod_types.go +++ b/nodes/mod_types.go @@ -56,10 +56,10 @@ type Version struct { Changelog string `json:"changelog,omitempty"` Stability string `json:"stability,omitempty"` ModID string `json:"mod_id,omitempty"` - Downloads uint `json:"downloads,omitempty"` - Approved bool `json:"approved,omitempty"` Dependencies []VersionDependency `json:"dependencies,omitempty"` Targets []VersionTarget `json:"targets,omitempty"` + Downloads uint `json:"downloads,omitempty"` + Approved bool `json:"approved,omitempty"` } type VersionDependency struct {