diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..65326bb6 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe345398..332f1398 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... @@ -39,7 +42,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... @@ -47,6 +53,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: + version: v1.49.0 skip-pkg-cache: true skip-build-cache: true args: --timeout 5m @@ -64,7 +71,10 @@ jobs: uses: actions/checkout@v3 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Go Generate run: go generate -tags tools -x ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e548779..fc5a0eff 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,10 @@ jobs: fetch-depth: 0 - name: Download dependencies - run: sudo apt update && sudo apt install -y build-essential libpng-dev + run: | + sudo apt update && sudo apt install -y build-essential libpng-dev protobuf-compiler + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 diff --git a/Dockerfile b/Dockerfile index d1d66b82..83c9552d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ -FROM golang:1.18-alpine AS builder +FROM golang:1.19-alpine3.18 AS builder -RUN apk add --no-cache git build-base libpng-dev +RUN apk add --no-cache git build-base libpng-dev protoc +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 WORKDIR $GOPATH/src/github.com/satisfactorymodding/smr-api/ diff --git a/README.md b/README.md index 269a5934..220b9da0 100755 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Main configuration options: The config format can be seen in `config/config.go` (each dot means a new level of nesting). +After startup requires the following minio commands to be executed: + +```shell +mc alias set local http://localhost:9000 minio minio123 +mc admin user svcacct add local minio --access-key REPLACE_ME_KEY --secret-key REPLACE_ME_SECRET +mc anonymous set public local/smr +``` + ## Contributing Before contributing, please run the [linter](https://golangci-lint.run/) to ensure the code is clean and well-formed: diff --git a/config.sample.json b/config.sample.json index 60037688..f865d27b 100644 --- a/config.sample.json +++ b/config.sample.json @@ -27,7 +27,8 @@ "key": "REPLACE_ME_KEY", "secret": "REPLACE_ME_SECRET", "endpoint": "http://localhost:9000", - "base_url": "http://localhost:9000" + "base_url": "http://localhost:9000", + "keypath": "%s/%s/%s" }, "oauth": { @@ -52,5 +53,9 @@ "frontend": { "url": "http://localhost:4200" + }, + + "feature_flags": { + "allow_multi_target_upload": false } } \ No newline at end of file diff --git a/config/config.go b/config/config.go index ad261198..8e0277c9 100644 --- a/config/config.go +++ b/config/config.go @@ -100,4 +100,6 @@ func initializeDefaults() { viper.SetDefault("frontend.url", "") viper.SetDefault("virustotal.key", "") + + viper.SetDefault("feature_flags.allow_multi_target_upload", false) } diff --git a/dataloader/loaders.go b/dataloader/loaders.go index 8fc03032..861ed84c 100644 --- a/dataloader/loaders.go +++ b/dataloader/loaders.go @@ -109,7 +109,7 @@ func Middleware() func(handlerFunc echo.HandlerFunc) echo.HandlerFunc { var entities []postgres.Version reqCtx := c.Request().Context() - postgres.DBCtx(reqCtx).Preload("Arch").Where("approved = ? AND denied = ? AND mod_id IN ?", true, false, fetchIds).Order("created_at desc").Find(&entities) + postgres.DBCtx(reqCtx).Preload("Targets").Where("approved = ? AND denied = ? AND mod_id IN ?", true, false, fetchIds).Order("created_at desc").Find(&entities) for _, entity := range entities { byID[entity.ModID] = append(byID[entity.ModID], entity) @@ -145,7 +145,7 @@ func Middleware() func(handlerFunc echo.HandlerFunc) echo.HandlerFunc { var entities []postgres.Version reqCtx := c.Request().Context() - postgres.DBCtx(reqCtx).Preload("Arch").Select( + postgres.DBCtx(reqCtx).Preload("Targets").Select( "id", "created_at", "updated_at", diff --git a/db/postgres/mod.go b/db/postgres/mod.go index 1c92c555..438ada7d 100644 --- a/db/postgres/mod.go +++ b/db/postgres/mod.go @@ -26,7 +26,7 @@ func GetModByID(ctx context.Context, modID string) *Mod { func GetModByIDNoCache(ctx context.Context, modID string) *Mod { var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "id = ?", modID) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "id = ?", modID) if mod.ID == "" { return nil @@ -44,7 +44,7 @@ func GetModByReference(ctx context.Context, modReference string) *Mod { } var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "mod_reference = ?", modReference) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "mod_reference = ?", modReference) if mod.ID == "" { return nil @@ -213,7 +213,7 @@ func NewModQuery(ctx context.Context, filter *models.ModFilter, unapproved bool, } query = query.Where("approved = ? AND denied = ?", !unapproved, false) - query = query.Preload("Tags").Preload("Versions.Arch") + query = query.Preload("Tags").Preload("Versions.Targets") if filter != nil { if filter.Search != nil && *filter.Search != "" { cleanSearch := strings.ReplaceAll(strings.TrimSpace(*filter.Search), " ", " & ") @@ -270,7 +270,7 @@ func GetModByIDOrReference(ctx context.Context, modIDOrReference string) *Mod { } var mod Mod - DBCtx(ctx).Preload("Tags").Preload("Versions.Arch").Find(&mod, "mod_reference = ? OR id = ?", modIDOrReference, modIDOrReference) + DBCtx(ctx).Preload("Tags").Preload("Versions.Targets").Find(&mod, "mod_reference = ? OR id = ?", modIDOrReference, modIDOrReference) if mod.ID == "" { return nil diff --git a/db/postgres/mod_archs.go b/db/postgres/mod_archs.go deleted file mode 100644 index aa6bde95..00000000 --- a/db/postgres/mod_archs.go +++ /dev/null @@ -1,122 +0,0 @@ -package postgres - -import ( - "context" - "strings" - - "github.com/patrickmn/go-cache" - - "github.com/satisfactorymodding/smr-api/models" - "github.com/satisfactorymodding/smr-api/util" -) - -func CreateModArch(ctx context.Context, modArch *ModArch) (*ModArch, error) { - modArch.ID = util.GenerateUniqueID() - DBCtx(ctx).Create(&modArch) - return modArch, nil -} - -func GetModArch(ctx context.Context, modArchID string) *ModArch { - cacheKey := "GetModArch_" + modArchID - - if modArch, ok := dbCache.Get(cacheKey); ok { - return modArch.(*ModArch) - } - - var modArch ModArch - DBCtx(ctx).Find(&modArch, "id = ?", modArchID) - - if modArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &modArch, cache.DefaultExpiration) - - return &modArch -} - -func GetModArchs(ctx context.Context, filter *models.ModArchFilter) []ModArch { - var modArchs []ModArch - query := DBCtx(ctx) - - if filter != nil { - query = query.Limit(*filter.Limit). - Offset(*filter.Offset). - Order(string(*filter.OrderBy) + " " + string(*filter.Order)) - - if filter.Search != nil && *filter.Search != "" { - query = query.Where("to_tsvector(name) @@ to_tsquery(?)", strings.ReplaceAll(*filter.Search, " ", " & ")) - } - } - - query.Find(&modArchs) - return modArchs -} - -func GetVersionModArchs(ctx context.Context, versionID string) []ModArch { - var modArchs []ModArch - query := DBCtx(ctx).Find(&modArchs, "mod_version_arch_id = ?", versionID) - - query.Find(&modArchs) - return modArchs -} - -func GetModArchByID(ctx context.Context, modArchID string) *ModArch { - cacheKey := "GetModArch_" + modArchID - - if modArch, ok := dbCache.Get(cacheKey); ok { - return modArch.(*ModArch) - } - - var modArch ModArch - DBCtx(ctx).Find(&modArch, "id = ?", modArchID) - - if modArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &modArch, cache.DefaultExpiration) - - return &modArch -} - -func GetModArchsByID(ctx context.Context, modArchIds []string) []ModArch { - var modArchs []ModArch - - DBCtx(ctx).Find(&modArchs, "id in (?)", modArchIds) - - if len(modArchIds) != len(modArchs) { - return nil - } - - return modArchs -} - -func GetModArchByPlatform(ctx context.Context, versionID string, platform string) *ModArch { - cacheKey := "GetModArch_" + versionID + "_" + platform - if modplatform, ok := dbCache.Get(cacheKey); ok { - return modplatform.(*ModArch) - } - - var modplatform ModArch - DBCtx(ctx).First(&modplatform, "mod_version_arch_id = ? AND platform = ?", versionID, platform) - - if modplatform.ModVersionID == "" { - return nil - } - - dbCache.Set(cacheKey, &modplatform, cache.DefaultExpiration) - - return &modplatform -} - -func GetModArchDownload(ctx context.Context, versionID string, platform string) string { - var modPlatform ModArch - DBCtx(ctx).First(&modPlatform, "mod_version_arch_id = ? AND platform = ?", versionID, platform) - - if modPlatform.ModVersionID == "" { - return "" - } - - return modPlatform.Key -} diff --git a/db/postgres/postgres_types.go b/db/postgres/postgres_types.go index 84e0622f..c09660ed 100644 --- a/db/postgres/postgres_types.go +++ b/db/postgres/postgres_types.go @@ -87,13 +87,27 @@ type Version struct { SMLVersion string `gorm:"type:varchar(16)"` Version string `gorm:"type:varchar(16)"` ModID string - Arch []ModArch `gorm:"foreignKey:ModVersionID;preload:true"` + Targets []VersionTarget `gorm:"foreignKey:VersionID"` Hotness uint Downloads uint Denied bool `gorm:"default:false;not null"` Approved bool `gorm:"default:false;not null"` } +type TinyVersion struct { + Hash *string + Size *int64 + SMRModel + SMLVersion string `gorm:"type:varchar(16)"` + Version string `gorm:"type:varchar(16)"` + Targets []VersionTarget `gorm:"foreignKey:VersionID;preload:true"` + Dependencies []VersionDependency `gorm:"foreignKey:VersionID"` +} + +func (TinyVersion) TableName() string { + return "versions" +} + type Guide struct { SMRModel Name string `gorm:"type:varchar(50)"` @@ -120,7 +134,8 @@ type SMLVersion struct { Stability string `sql:"type:version_stability"` Link string Changelog string - Arch []SMLArch `gorm:"foreignKey:SMLVersionID;preload:true"` + EngineVersion string + Targets []SMLVersionTarget `gorm:"foreignKey:VersionID"` SatisfactoryVersion int } @@ -179,26 +194,16 @@ type Compatibility struct { Note string } -type ModArch struct { - ID string `gorm:"primary_key;type:varchar(16)"` - ModVersionID string `gorm:"column:mod_version_arch_id"` - Platform string - Key string - Hash string - Size int64 -} - -func (ModArch) TableName() string { - return "mod_archs" -} - -type SMLArch struct { - ID string `gorm:"primary_key;type:varchar(14)"` - SMLVersionID string `gorm:"column:sml_version_arch_id"` - Platform string - Link string +type VersionTarget struct { + VersionID string `gorm:"primary_key;type:varchar(14)"` + TargetName string `gorm:"primary_key;type:varchar(16)"` + Key string + Hash string + Size int64 } -func (SMLArch) TableName() string { - return "sml_archs" +type SMLVersionTarget struct { + VersionID string `gorm:"primary_key;type:varchar(14)"` + TargetName string `gorm:"primary_key;type:varchar(16)"` + Link string } diff --git a/db/postgres/sml_archs.go b/db/postgres/sml_archs.go deleted file mode 100644 index f8b0108c..00000000 --- a/db/postgres/sml_archs.go +++ /dev/null @@ -1,99 +0,0 @@ -package postgres - -import ( - "context" - "strings" - - "github.com/patrickmn/go-cache" - - "github.com/satisfactorymodding/smr-api/models" - "github.com/satisfactorymodding/smr-api/util" -) - -func CreateSMLArch(ctx context.Context, smlArch *SMLArch) (*SMLArch, error) { - smlArch.ID = util.GenerateUniqueID() - - DBCtx(ctx).Create(&smlArch) - - return smlArch, nil -} - -func GetSMLArch(ctx context.Context, smlLinkID string) *SMLArch { - cacheKey := "GetSMLArch_" + smlLinkID - - if smlArch, ok := dbCache.Get(cacheKey); ok { - return smlArch.(*SMLArch) - } - - var smlArch SMLArch - DBCtx(ctx).Find(&smlArch, "id = ?", smlLinkID) - - if smlArch.ID == "" { - return nil - } - - dbCache.Set(cacheKey, &smlArch, cache.DefaultExpiration) - - return &smlArch -} - -func GetSMLArchs(ctx context.Context, filter *models.SMLArchFilter) []SMLArch { - var smlLinks []SMLArch - query := DBCtx(ctx) - - if filter != nil { - query = query.Limit(*filter.Limit). - Offset(*filter.Offset). - Order(string(*filter.OrderBy) + " " + string(*filter.Order)) - - if filter.Search != nil && *filter.Search != "" { - query = query.Where("to_tsvector(name) @@ to_tsquery(?)", strings.ReplaceAll(*filter.Search, " ", " & ")) - } - } - - query.Find(&smlLinks) - return smlLinks -} - -func GetSMLArchByID(ctx context.Context, smlLinkID string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "id in ?", smlLinkID) - - if len(smlArchs) != 0 { - return nil - } - - return smlArchs -} - -func GetSMLArchsByID(ctx context.Context, smlArchIds []string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "id in (?)", smlArchIds) - - if len(smlArchIds) != len(smlArchs) { - return nil - } - - return smlArchs -} - -func GetSMLArchBySMLID(ctx context.Context, smlVersionID string) []SMLArch { - var smlArchs []SMLArch - - DBCtx(ctx).Find(&smlArchs, "sml_version_arch_id = ?", smlVersionID) - - return smlArchs -} - -func GetSMLArchDownload(ctx context.Context, smlVersionID string, platform string) string { - var smlPlatform SMLArch - DBCtx(ctx).First(&smlPlatform, "sml_version_arch_id = ? AND platform = ?", smlVersionID, platform) - - if smlPlatform.ID == "" { - return "" - } - - return smlPlatform.Link -} diff --git a/db/postgres/sml_version.go b/db/postgres/sml_version.go index 762e923d..46f6e213 100644 --- a/db/postgres/sml_version.go +++ b/db/postgres/sml_version.go @@ -13,21 +13,12 @@ func CreateSMLVersion(ctx context.Context, smlVersion *SMLVersion) (*SMLVersion, DBCtx(ctx).Create(&smlVersion) - for _, link := range smlVersion.Arch { - DBCtx(ctx).Create(&SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: smlVersion.ID, - Platform: link.Platform, - Link: link.Link, - }) - } - return smlVersion, nil } func GetSMLVersionByID(ctx context.Context, smlVersionID string) *SMLVersion { var smlVersion SMLVersion - DBCtx(ctx).Preload("Arch").Find(&smlVersion, "id in (?)", smlVersionID) + DBCtx(ctx).Preload("Targets").Find(&smlVersion, "id in (?)", smlVersionID) if smlVersion.ID == "" { return nil @@ -50,14 +41,14 @@ func GetSMLVersions(ctx context.Context, filter *models.SMLVersionFilter) []SMLV } } - query.Preload("Arch").Find(&smlVersions) + query.Preload("Targets").Find(&smlVersions) return smlVersions } func GetSMLVersionsByID(ctx context.Context, smlVersionIds []string) []SMLVersion { var smlVersions []SMLVersion - DBCtx(ctx).Preload("Arch").Find(&smlVersions, "id in (?)", smlVersionIds) + DBCtx(ctx).Preload("Targets").Find(&smlVersions, "id in (?)", smlVersionIds) if len(smlVersionIds) != len(smlVersions) { return nil @@ -83,9 +74,17 @@ func GetSMLVersionCount(ctx context.Context, filter *models.SMLVersionFilter) in func GetSMLLatestVersions(ctx context.Context) *[]SMLVersion { var smlVersions []SMLVersion - DBCtx(ctx).Preload("Arch").Select("distinct on (stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (stability) *"). Order("stability, created_at desc"). Find(&smlVersions) return &smlVersions } + +func GetSMLVersionTargets(ctx context.Context, smlVersionID string) []SMLVersionTarget { + var smlVersionTargets []SMLVersionTarget + + DBCtx(ctx).Find(&smlVersionTargets, "version_id = ?", smlVersionID) + + return smlVersionTargets +} diff --git a/db/postgres/version.go b/db/postgres/version.go index 54400553..d70aee89 100644 --- a/db/postgres/version.go +++ b/db/postgres/version.go @@ -24,7 +24,7 @@ func GetVersionsByID(ctx context.Context, versionIds []string) []Version { } var versions []Version - DBCtx(ctx).Preload("Arch").Find(&versions, "id in (?)", versionIds) + DBCtx(ctx).Preload("Targets").Find(&versions, "id in (?)", versionIds) if len(versionIds) != len(versions) { return nil @@ -43,7 +43,7 @@ func GetModLatestVersions(ctx context.Context, modID string, unapproved bool) *[ var versions []Version - DBCtx(ctx).Preload("Arch").Select("distinct on (mod_id, stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (mod_id, stability) *"). Where("mod_id = ?", modID). Where("approved = ? AND denied = ?", !unapproved, false). Order("mod_id, stability, created_at desc"). @@ -62,7 +62,7 @@ func GetModsLatestVersions(ctx context.Context, modIds []string, unapproved bool var versions []Version - DBCtx(ctx).Preload("Arch").Select("distinct on (mod_id, stability) *"). + DBCtx(ctx).Preload("Targets").Select("distinct on (mod_id, stability) *"). Where("mod_id in (?)", modIds). Where("approved = ? AND denied = ?", !unapproved, false). Order("mod_id, stability, created_at desc"). @@ -80,7 +80,25 @@ func GetModVersions(ctx context.Context, modID string, limit int, offset int, or } var versions []Version - DBCtx(ctx).Preload("Arch").Limit(limit).Offset(offset).Order(orderBy+" "+order).Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) + DBCtx(ctx).Preload("Targets").Limit(limit).Offset(offset).Order(orderBy+" "+order).Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) + + dbCache.Set(cacheKey, versions, cache.DefaultExpiration) + + return versions +} + +func GetAllModVersionsWithDependencies(ctx context.Context, modID string) []TinyVersion { + cacheKey := "GetAllModVersionsWithDependencies_" + modID + if versions, ok := dbCache.Get(cacheKey); ok { + return versions.([]TinyVersion) + } + + var versions []TinyVersion + DBCtx(ctx).Debug(). + Preload("Dependencies"). + Preload("Targets"). + Where("approved = ? AND denied = ?", true, false). + Find(&versions, "mod_id = ?", modID) dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -98,7 +116,7 @@ func GetModVersionsNew(ctx context.Context, modID string, filter *models.Version } var versions []Version - query := DBCtx(ctx).Preload("Arch") + query := DBCtx(ctx).Preload("Targets") if filter != nil { query = query.Limit(*filter.Limit). @@ -106,7 +124,7 @@ func GetModVersionsNew(ctx context.Context, modID string, filter *models.Version Order(string(*filter.OrderBy) + " " + string(*filter.Order)) } - query.Preload("Arch").Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) + query.Preload("Targets").Where("approved = ? AND denied = ?", !unapproved, false).Find(&versions, "mod_id = ?", modID) if cacheKey != "" { dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -122,7 +140,7 @@ func GetModVersion(ctx context.Context, modID string, versionID string) *Version } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "mod_id = ? AND id = ?", modID, versionID) + DBCtx(ctx).Preload("Targets").First(&version, "mod_id = ? AND id = ?", modID, versionID) if version.ID == "" { return nil @@ -140,7 +158,7 @@ func GetModVersionByName(ctx context.Context, modID string, versionName string) } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "mod_id = ? AND version = ?", modID, versionName) + DBCtx(ctx).Preload("Targets").First(&version, "mod_id = ? AND version = ?", modID, versionName) if version.ID == "" { return nil @@ -186,7 +204,7 @@ func GetVersion(ctx context.Context, versionID string) *Version { } var version Version - DBCtx(ctx).Preload("Arch").First(&version, "id = ?", versionID) + DBCtx(ctx).Preload("Targets").First(&version, "id = ?", versionID) if version.ID == "" { return nil @@ -208,7 +226,7 @@ func GetVersionsNew(ctx context.Context, filter *models.VersionFilter, unapprove } var versions []Version - query := DBCtx(ctx).Preload("Arch").Where("approved = ? AND denied = ?", !unapproved, false) + query := DBCtx(ctx).Preload("Targets").Where("approved = ? AND denied = ?", !unapproved, false) if filter != nil { query = query.Limit(*filter.Limit). @@ -224,7 +242,7 @@ func GetVersionsNew(ctx context.Context, filter *models.VersionFilter, unapprove } } - query.Preload("Arch").Find(&versions) + query.Preload("Targets").Find(&versions) if cacheKey != "" { dbCache.Set(cacheKey, versions, cache.DefaultExpiration) @@ -261,6 +279,24 @@ func GetVersionCountNew(ctx context.Context, filter *models.VersionFilter, unapp return versionCount } +func GetVersionTarget(ctx context.Context, versionID string, target string) *VersionTarget { + cacheKey := "GetVersionTarget_" + versionID + "_" + target + if versionTarget, ok := dbCache.Get(cacheKey); ok { + return versionTarget.(*VersionTarget) + } + + var versionTarget VersionTarget + DBCtx(ctx).First(&versionTarget, "version_id = ? AND target_name = ?", versionID, target) + + if versionTarget.VersionID == "" { + return nil + } + + dbCache.Set(cacheKey, &versionTarget, cache.DefaultExpiration) + + return &versionTarget +} + func GetVersionDependencies(ctx context.Context, versionID string) []VersionDependency { var versionDependencies []VersionDependency DBCtx(ctx).Where("version_id = ?", versionID).Find(&versionDependencies) @@ -288,7 +324,7 @@ func GetModVersionsConstraint(ctx context.Context, modID string, constraint stri return nil } - query := DBCtx(ctx).Preload("Arch").Where("mod_id", modID) + query := DBCtx(ctx).Preload("Targets").Where("mod_id", modID) /* <=1.2.3 @@ -357,6 +393,6 @@ func GetModVersionsConstraint(ctx context.Context, modID string, constraint stri } var versions []Version - query.Preload("Arch").Find(&versions) + query.Preload("Targets").Find(&versions) return versions } diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3327f6b2..6b302312 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,4 +25,9 @@ services: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio123 MINIO_ACCESS_KEY: REPLACE_ME_KEY - MINIO_SECRET_KEY: REPLACE_ME_SECRET \ No newline at end of file + MINIO_SECRET_KEY: REPLACE_ME_SECRET + + pak_parser: + image: ghcr.io/vilsol/ficsit-pak-parser:v0.0.3 + ports: + - 50051:50051 \ No newline at end of file diff --git a/go.mod b/go.mod index 2a6ed37f..43a278e6 100755 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa golang.org/x/net v0.0.0-20220728030405-41545e8bf201 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/go-playground/validator.v9 v9.31.0 gorm.io/driver/postgres v1.3.5 gorm.io/gorm v1.23.5 diff --git a/go.sum b/go.sum index 42c970ce..7828d761 100755 --- a/go.sum +++ b/go.sum @@ -1677,6 +1677,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/gql/gql_types.go b/gql/gql_types.go index 7a7f8778..6d8d3960 100644 --- a/gql/gql_types.go +++ b/gql/gql_types.go @@ -85,7 +85,7 @@ func DBVersionToGenerated(version *postgres.Version) *generated.Version { Changelog: version.Changelog, Downloads: int(version.Downloads), Stability: generated.VersionStabilities(version.Stability), - Arch: DBModArchsToGeneratedSlice(version.Arch), + Targets: DBVersionTargetsToGeneratedSlice(version.Targets), Approved: version.Approved, UpdatedAt: version.UpdatedAt.Format(time.RFC3339Nano), CreatedAt: version.CreatedAt.Format(time.RFC3339Nano), @@ -134,11 +134,12 @@ func DBSMLVersionToGenerated(smlVersion *postgres.SMLVersion) *generated.SMLVers BootstrapVersion: smlVersion.BootstrapVersion, Stability: generated.VersionStabilities(smlVersion.Stability), Link: smlVersion.Link, - Arch: DBSMLArchsToGeneratedSlice(smlVersion.Arch), + Targets: DBSMLVersionTargetToGeneratedSlice(smlVersion.Targets), Changelog: smlVersion.Changelog, Date: smlVersion.Date.Format(time.RFC3339Nano), UpdatedAt: smlVersion.UpdatedAt.Format(time.RFC3339Nano), CreatedAt: smlVersion.CreatedAt.Format(time.RFC3339Nano), + EngineVersion: smlVersion.EngineVersion, } } @@ -211,47 +212,46 @@ func DBTagsToGeneratedSlice(tags []postgres.Tag) []*generated.Tag { return converted } -func DBModArchToGenerated(modArch *postgres.ModArch) *generated.ModArch { - if modArch == nil { +func DBVersionTargetToGenerated(versionTarget *postgres.VersionTarget) *generated.VersionTarget { + if versionTarget == nil { return nil } - size := int(modArch.Size) + hash := versionTarget.Hash + size := int(versionTarget.Size) - return &generated.ModArch{ - ID: modArch.ID, - ModVersionID: modArch.ModVersionID, - Platform: modArch.Platform, - Hash: &modArch.Hash, - Size: &size, + return &generated.VersionTarget{ + VersionID: versionTarget.VersionID, + TargetName: generated.TargetName(versionTarget.TargetName), + Hash: &hash, + Size: &size, } } -func DBModArchsToGeneratedSlice(modArchs []postgres.ModArch) []*generated.ModArch { - converted := make([]*generated.ModArch, len(modArchs)) - for i, modArch := range modArchs { - converted[i] = DBModArchToGenerated(&modArch) +func DBVersionTargetsToGeneratedSlice(versionTargets []postgres.VersionTarget) []*generated.VersionTarget { + converted := make([]*generated.VersionTarget, len(versionTargets)) + for i, versionTarget := range versionTargets { + converted[i] = DBVersionTargetToGenerated(&versionTarget) } return converted } -func DBSMLArchToGenerated(smlArch *postgres.SMLArch) *generated.SMLArch { - if smlArch == nil { +func DBSMLVersionTargetToGenerated(smlVersionTarget *postgres.SMLVersionTarget) *generated.SMLVersionTarget { + if smlVersionTarget == nil { return nil } - return &generated.SMLArch{ - ID: smlArch.ID, - SMLVersionID: smlArch.SMLVersionID, - Platform: smlArch.Platform, - Link: smlArch.Link, + return &generated.SMLVersionTarget{ + VersionID: smlVersionTarget.VersionID, + TargetName: generated.TargetName(smlVersionTarget.TargetName), + Link: smlVersionTarget.Link, } } -func DBSMLArchsToGeneratedSlice(smlLinks []postgres.SMLArch) []*generated.SMLArch { - converted := make([]*generated.SMLArch, len(smlLinks)) - for i, smlArch := range smlLinks { - converted[i] = DBSMLArchToGenerated(&smlArch) +func DBSMLVersionTargetToGeneratedSlice(smlVersionTargets []postgres.SMLVersionTarget) []*generated.SMLVersionTarget { + converted := make([]*generated.SMLVersionTarget, len(smlVersionTargets)) + for i, smlVersionTarget := range smlVersionTargets { + converted[i] = DBSMLVersionTargetToGenerated(&smlVersionTarget) } return converted } diff --git a/gql/resolver.go b/gql/resolver.go index 51c0c370..43547809 100755 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -26,8 +26,8 @@ func (r *Resolver) UserMod() generated.UserModResolver { return &userModResolver{r} } -func (r *Resolver) ModArch() generated.ModArchResolver { - return &modlinkResolver{r} +func (r *Resolver) VersionTarget() generated.VersionTargetResolver { + return &versionTargetResolver{r} } func (r *Resolver) Version() generated.VersionResolver { diff --git a/gql/resolver_mod_archs.go b/gql/resolver_mod_archs.go deleted file mode 100644 index 0181328e..00000000 --- a/gql/resolver_mod_archs.go +++ /dev/null @@ -1,34 +0,0 @@ -package gql - -import ( - "context" - - "github.com/satisfactorymodding/smr-api/db/postgres" - "github.com/satisfactorymodding/smr-api/generated" -) - -type modlinkResolver struct{ *Resolver } - -func (r *modlinkResolver) Asset(_ context.Context, obj *generated.ModArch) (string, error) { - return "/v1/version/" + obj.ModVersionID + "/" + obj.Platform + "/download", nil -} - -func (r *queryResolver) GetModArch(ctx context.Context, modArchID string) (*generated.ModArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getModArch") - defer wrapper.end() - modArch := postgres.GetModArch(newCtx, modArchID) - return DBModArchToGenerated(modArch), nil -} - -func (r *queryResolver) GetModArchs(ctx context.Context, filter map[string]interface{}) (*generated.GetModArchs, error) { - wrapper, _ := WrapQueryTrace(ctx, "getModArchs") - defer wrapper.end() - return &generated.GetModArchs{}, nil -} - -func (r *queryResolver) GetModArchByID(ctx context.Context, modArchID string) (*generated.ModArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getModArchByID") - defer wrapper.end() - modArch := postgres.GetModArchByID(newCtx, modArchID) - return DBModArchToGenerated(modArch), nil -} diff --git a/gql/resolver_sml_archs.go b/gql/resolver_sml_archs.go deleted file mode 100644 index be4e7763..00000000 --- a/gql/resolver_sml_archs.go +++ /dev/null @@ -1,101 +0,0 @@ -package gql - -import ( - "context" - - "github.com/pkg/errors" - "gopkg.in/go-playground/validator.v9" - - "github.com/satisfactorymodding/smr-api/db/postgres" - "github.com/satisfactorymodding/smr-api/generated" - "github.com/satisfactorymodding/smr-api/util" -) - -func (r *mutationResolver) CreateSMLArch(ctx context.Context, smlArch generated.NewSMLArch) (*generated.SMLArch, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "createSMLArch") - defer wrapper.end() - - val := ctx.Value(util.ContextValidator{}).(*validator.Validate) - if err := val.Struct(&smlArch); err != nil { - return nil, errors.Wrap(err, "validation failed") - } - - dbSMLArchs := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArchs) - if err != nil { - return nil, err - } - - return DBSMLArchToGenerated(resultSMLArch), nil -} - -func (r *mutationResolver) DeleteSMLArch(ctx context.Context, linksID string) (bool, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "deleteSMLArch") - defer wrapper.end() - - dbSMLArch := postgres.GetSMLArchByID(newCtx, linksID) - - if dbSMLArch == nil { - return false, errors.New("SML Link not found") - } - - postgres.Delete(newCtx, &dbSMLArch) - - return true, nil -} - -func (r *mutationResolver) UpdateSMLArch(ctx context.Context, smlLinkID string, smlArch generated.UpdateSMLArch) (*generated.SMLArch, error) { - wrapper, newCtx := WrapMutationTrace(ctx, "updateSMLArch") - defer wrapper.end() - val := ctx.Value(util.ContextValidator{}).(*validator.Validate) - if err := val.Struct(&smlArch); err != nil { - return nil, errors.Wrap(err, "validation failed") - } - - dbSMLArch := postgres.GetSMLArch(newCtx, smlLinkID) - - if dbSMLArch == nil { - return nil, errors.New("sml link not found") - } - - SetStringINNOE(&smlArch.Platform, &dbSMLArch.Platform) - SetStringINNOE(&smlArch.Link, &dbSMLArch.Link) - - postgres.Save(newCtx, &dbSMLArch) - - return DBSMLArchToGenerated(dbSMLArch), nil -} - -func (r *queryResolver) GetSMLArch(ctx context.Context, smlLinkID string) (*generated.SMLArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getSMLArch") - defer wrapper.end() - - smlArch := postgres.GetSMLArch(newCtx, smlLinkID) - - return DBSMLArchToGenerated(smlArch), nil -} - -func (r *queryResolver) GetSMLArchs(ctx context.Context, filter map[string]interface{}) (*generated.GetSMLArchs, error) { - wrapper, _ := WrapQueryTrace(ctx, "getSMLArchs") - defer wrapper.end() - return &generated.GetSMLArchs{}, nil -} - -func (r *queryResolver) GetSMLArchBySMLID(ctx context.Context, smlVersionID string) ([]postgres.SMLArch, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "GetSMLArchBySMLID") - defer wrapper.end() - smlArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) - return smlArch, nil -} - -func (r *queryResolver) GetSMLDownload(ctx context.Context, smlVersionID string, platform string) (string, error) { - wrapper, newCtx := WrapQueryTrace(ctx, "getSMLDownload") - defer wrapper.end() - smlArch := postgres.GetSMLArchDownload(newCtx, smlVersionID, platform) - return smlArch, nil -} diff --git a/gql/resolver_sml_versions.go b/gql/resolver_sml_versions.go index c04c6c1c..a5859f55 100644 --- a/gql/resolver_sml_versions.go +++ b/gql/resolver_sml_versions.go @@ -36,24 +36,17 @@ func (r *mutationResolver) CreateSMLVersion(ctx context.Context, smlVersion gene Link: smlVersion.Link, Changelog: smlVersion.Changelog, Date: date, + EngineVersion: smlVersion.EngineVersion, } resultSMLVersion, err := postgres.CreateSMLVersion(newCtx, dbSMLVersion) - for _, smlArch := range smlVersion.Arch { - dbSMLArchs := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: resultSMLVersion.ID, - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArchs) - if err != nil { - return nil, err - } - - DBSMLArchToGenerated(resultSMLArch) + for _, smlVersionTarget := range smlVersion.Targets { + postgres.Save(newCtx, &postgres.SMLVersionTarget{ + VersionID: resultSMLVersion.ID, + TargetName: string(smlVersionTarget.TargetName), + Link: smlVersionTarget.Link, + }) } if err != nil { @@ -72,6 +65,30 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st return nil, errors.Wrap(err, "validation failed") } + dbSMLTargets := postgres.GetSMLVersionTargets(newCtx, smlVersionID) + + for _, dbSMLTarget := range dbSMLTargets { + found := false + + for _, smlTarget := range smlVersion.Targets { + if dbSMLTarget.TargetName == string(smlTarget.TargetName) { + found = true + } + } + + if !found { + postgres.Delete(newCtx, &dbSMLTarget) + } + } + + for _, smlTarget := range smlVersion.Targets { + postgres.Save(newCtx, &postgres.SMLVersionTarget{ + VersionID: smlVersionID, + TargetName: string(smlTarget.TargetName), + Link: smlTarget.Link, + }) + } + dbSMLVersion := postgres.GetSMLVersionByID(newCtx, smlVersionID) if dbSMLVersion == nil { @@ -85,43 +102,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st SetStringINNOE(smlVersion.Link, &dbSMLVersion.Link) SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog) SetDateINN(smlVersion.Date, &dbSMLVersion.Date) - - dbSMLArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) - - if len(dbSMLArch) == len(smlVersion.Arch) { - for i, smlArch := range smlVersion.Arch { - SetStringINNOE(&smlArch.Platform, &dbSMLArch[i].Platform) - SetStringINNOE(&smlArch.Link, &dbSMLArch[i].Link) - - postgres.Save(newCtx, dbSMLArch) - } - } else { - for _, smlArch := range dbSMLVersion.Arch { - dbSMLArch := postgres.GetSMLArchBySMLID(newCtx, smlVersionID) - - if dbSMLVersion == nil { - return nil, errors.New("smlArch not found" + smlArch.Platform) - } - - postgres.Delete(newCtx, &dbSMLArch) - } - - for _, smlArch := range smlVersion.Arch { - dbSMLArch := &postgres.SMLArch{ - ID: util.GenerateUniqueID(), - SMLVersionID: smlVersionID, - Platform: smlArch.Platform, - Link: smlArch.Link, - } - - resultSMLArch, err := postgres.CreateSMLArch(newCtx, dbSMLArch) - if err != nil { - return nil, err - } - - DBSMLArchToGenerated(resultSMLArch) - } - } + SetStringINNOE(smlVersion.EngineVersion, &dbSMLVersion.EngineVersion) postgres.Save(newCtx, &dbSMLVersion) @@ -138,14 +119,10 @@ func (r *mutationResolver) DeleteSMLVersion(ctx context.Context, smlVersionID st return false, errors.New("smlVersion not found") } - for _, smlArch := range dbSMLVersion.Arch { - dbSMLArch := postgres.GetSMLArch(newCtx, smlArch.ID) - - if dbSMLVersion == nil { - return false, errors.New("smlArch not found") - } + dbSMLVersionTargets := postgres.GetSMLVersionTargets(newCtx, smlVersionID) - postgres.Delete(newCtx, &dbSMLArch) + for _, dbSMLVersionTarget := range dbSMLVersionTargets { + postgres.Delete(newCtx, &dbSMLVersionTarget) } postgres.Delete(newCtx, &dbSMLVersion) diff --git a/gql/resolver_versions.go b/gql/resolver_versions.go index f551b381..4d2d216e 100644 --- a/gql/resolver_versions.go +++ b/gql/resolver_versions.go @@ -184,8 +184,6 @@ func (r *mutationResolver) ApproveVersion(ctx context.Context, versionID string) postgres.Save(newCtx, &mod) go integrations.NewVersion(util.ReWrapCtx(ctx), dbVersion) - go storage.DeleteModArch(ctx, dbVersion.ModID, mod.Name, versionID, "Combined") - go storage.DeleteModArch(ctx, dbVersion.ModID, mod.Name, dbVersion.Version, "Combined") return true, nil } @@ -321,7 +319,27 @@ func (r *getVersionsResolver) Count(ctx context.Context, _ *generated.GetVersion type versionResolver struct{ *Resolver } -func (r *versionResolver) Link(_ context.Context, obj *generated.Version) (string, error) { +func findWindowsTarget(obj *generated.Version) *generated.VersionTarget { + var windowsTarget *generated.VersionTarget + for _, target := range obj.Targets { + if target.TargetName == "Windows" { + windowsTarget = target + break + } + } + return windowsTarget +} + +func (r *versionResolver) Link(ctx context.Context, obj *generated.Version) (string, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.link") + defer wrapper.end() + + windowsTarget := findWindowsTarget(obj) + if windowsTarget != nil { + link, _ := r.VersionTarget().Link(ctx, windowsTarget) + return link, nil + } + return "/v1/version/" + obj.ID + "/download", nil } @@ -332,6 +350,50 @@ func (r *versionResolver) Mod(ctx context.Context, obj *generated.Version) (*gen return DBModToGenerated(postgres.GetModByID(newCtx, obj.ModID)), nil } +func (r *versionResolver) Hash(ctx context.Context, obj *generated.Version) (*string, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.hash") + defer wrapper.end() + + hash := "" + + windowsTarget := findWindowsTarget(obj) + if windowsTarget == nil { + if obj.Hash == nil { + return nil, nil + } + hash = *obj.Hash + } else { + if windowsTarget.Hash == nil { + return nil, nil + } + hash = *windowsTarget.Hash + } + + return &hash, nil +} + +func (r *versionResolver) Size(ctx context.Context, obj *generated.Version) (*int, error) { + wrapper, _ := WrapQueryTrace(ctx, "Version.size") + defer wrapper.end() + + size := 0 + + windowsTarget := findWindowsTarget(obj) + if windowsTarget == nil { + if obj.Size == nil { + return nil, nil + } + size = *obj.Size + } else { + if windowsTarget.Size == nil { + return nil, nil + } + size = *windowsTarget.Size + } + + return &size, nil +} + var versionDependencyCache, _ = ristretto.NewCache(&ristretto.Config{ NumCounters: 1e6, // number of keys to track frequency of (1M). MaxCost: 1e6, // maximum cost of cache (1M). @@ -369,6 +431,12 @@ func (r *versionResolver) Dependencies(ctx context.Context, obj *generated.Versi return converted, nil } +type versionTargetResolver struct{ *Resolver } + +func (r *versionTargetResolver) Link(_ context.Context, obj *generated.VersionTarget) (string, error) { + return "/v1/version/" + obj.VersionID + "/" + string(obj.TargetName) + "/download", nil +} + type getMyVersionsResolver struct{ *Resolver } func (r *getMyVersionsResolver) Versions(ctx context.Context, _ *generated.GetMyVersions) ([]*generated.Version, error) { diff --git a/gql/versions.go b/gql/versions.go index 97e2c0ae..de9a28b1 100644 --- a/gql/versions.go +++ b/gql/versions.go @@ -6,7 +6,6 @@ import ( "io" "time" - "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -45,10 +44,8 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI modInfo, err := validation.ExtractModInfo(ctx, fileData, true, true, mod.ModReference) if err != nil { - spew.Dump(err) - l.Err(err).Msg("failed extracting mod info") storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) - return nil, err + return nil, errors.Wrap(err, "failed extracting mod info") } if modInfo.ModReference != mod.ModReference { @@ -56,6 +53,16 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI return nil, errors.New("data.json mod_reference does not match mod reference") } + if modInfo.Type == validation.DataJSON { + storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + return nil, errors.New("data.json mods are obsolete and not allowed") + } + + if modInfo.Type == validation.MultiTargetUEPlugin && !util.FlagEnabled(util.FeatureFlagAllowMultiTargetUpload) { + storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + return nil, errors.New("multi-target mods are not allowed") + } + versionMajor := int(modInfo.Semver.Major()) versionMinor := int(modInfo.Semver.Minor()) versionPatch := int(modInfo.Semver.Patch()) @@ -67,6 +74,8 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI ModID: mod.ID, Stability: string(version.Stability), ModReference: &modInfo.ModReference, + Size: &modInfo.Size, + Hash: &modInfo.Hash, VersionMajor: &versionMajor, VersionMinor: &versionMinor, VersionPatch: &versionPatch, @@ -120,50 +129,68 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &dbVersion) } - separated := storage.SeparateMod(ctx, fileData, mod.ID, mod.Name, dbVersion.ID, modInfo.Version) + if modInfo.Type == validation.MultiTargetUEPlugin { + targets := make([]*postgres.VersionTarget, 0) - if !separated { - for modID, condition := range modInfo.Dependencies { - dependency := postgres.VersionDependency{ - VersionID: dbVersion.ID, - ModID: modID, - Condition: condition, - Optional: false, + for _, target := range modInfo.Targets { + dbVersionTarget := &postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: target, } - postgres.DeleteForced(ctx, &dependency) + postgres.Save(ctx, dbVersionTarget) + + targets = append(targets, dbVersionTarget) } - for modID, condition := range modInfo.OptionalDependencies { - dependency := postgres.VersionDependency{ - VersionID: dbVersion.ID, - ModID: modID, - Condition: condition, - Optional: true, + separateSuccess := true + for _, target := range targets { + log.Info().Str("target", target.TargetName).Str("mod", mod.Name).Str("version", dbVersion.Version).Msg("separating mod") + success, key, hash, size := storage.SeparateModTarget(ctx, fileData, mod.ID, mod.Name, dbVersion.Version, target.TargetName) + + if !success { + separateSuccess = false + break } - postgres.DeleteForced(ctx, &dependency) + target.Key = key + target.Hash = hash + target.Size = size + + postgres.Save(ctx, target) } - postgres.DeleteForced(ctx, &dbVersion) - storage.DeleteMod(ctx, mod.ID, mod.Name, versionID) + if !separateSuccess { + removeMod(ctx, modInfo, mod, dbVersion) - for _, dbModArch := range dbVersion.Arch { - postgres.DeleteForced(ctx, &dbModArch) + return nil, errors.New("failed to separate mod") } + } + + success, key := storage.RenameVersion(ctx, mod.ID, mod.Name, versionID, modInfo.Version) + + if !success { + removeMod(ctx, modInfo, mod, dbVersion) + return nil, errors.New("failed to upload mod") } - dbModArch := postgres.GetModArchByPlatform(ctx, dbVersion.ID, "WindowsNoEditor") + if modInfo.Type == validation.UEPlugin { + dbVersionTarget := &postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: "Windows", + Key: key, + Hash: *dbVersion.Hash, + Size: *dbVersion.Size, + } + + postgres.Save(ctx, dbVersionTarget) + } - dbVersion.Key = dbModArch.Key - dbVersion.Hash = &dbModArch.Hash - dbVersion.Size = &dbModArch.Size + dbVersion.Key = key postgres.Save(ctx, &dbVersion) postgres.Save(ctx, &mod) - storage.DeleteVersion(ctx, mod.ID, mod.Name, versionID) - if autoApproved { mod := postgres.GetModByID(ctx, dbVersion.ModID) now := time.Now() @@ -171,8 +198,6 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI postgres.Save(ctx, &mod) go integrations.NewVersion(util.ReWrapCtx(ctx), dbVersion) - storage.DeleteModArch(ctx, mod.ID, mod.Name, versionID, "Combined") - storage.DeleteModArch(ctx, mod.ID, mod.Name, dbVersion.Version, "Combined") } else { l.Info().Msg("Submitting version job for virus scan") jobs.SubmitJobScanModOnVirusTotalTask(ctx, mod.ID, dbVersion.ID, true) @@ -183,3 +208,46 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI Version: DBVersionToGenerated(dbVersion), }, nil } + +func removeMod(ctx context.Context, modInfo *validation.ModInfo, mod *postgres.Mod, dbVersion *postgres.Version) { + for modID, condition := range modInfo.Dependencies { + dependency := postgres.VersionDependency{ + VersionID: dbVersion.ID, + ModID: modID, + Condition: condition, + Optional: false, + } + + postgres.DeleteForced(ctx, &dependency) + } + + for modID, condition := range modInfo.OptionalDependencies { + dependency := postgres.VersionDependency{ + VersionID: dbVersion.ID, + ModID: modID, + Condition: condition, + Optional: true, + } + + postgres.DeleteForced(ctx, &dependency) + } + + for _, target := range modInfo.Targets { + dbVersionTarget := postgres.VersionTarget{ + VersionID: dbVersion.ID, + TargetName: target, + } + + postgres.DeleteForced(ctx, &dbVersionTarget) + } + + // For UEPlugin mods, a Windows target is created. + // However, that happens after the last possible call to this function, therefore we can ignore it + + postgres.DeleteForced(ctx, &dbVersion) + + storage.DeleteMod(ctx, mod.ID, mod.Name, dbVersion.ID) + for _, target := range modInfo.Targets { + storage.DeleteModTarget(ctx, mod.ID, mod.Name, dbVersion.ID, target) + } +} diff --git a/gqlgen.yml b/gqlgen.yml index ecb11180..7c2148e4 100755 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -20,10 +20,6 @@ models: UpdateMod: model: github.com/satisfactorymodding/smr-api/generated.UpdateMod - ModArchFilter: - model: "map[string]interface{}" - SMLArchFilter: - model: "map[string]interface{}" VersionFilter: model: "map[string]interface{}" ModFilter: @@ -57,11 +53,6 @@ models: latestVersions: resolver: true - ModArch: - fields: - asset: - resolver: true - UserMod: fields: user: @@ -77,6 +68,15 @@ models: resolver: true dependencies: resolver: true + size: + resolver: true + hash: + resolver: true + + VersionTarget: + fields: + link: + resolver: true GetMods: fields: diff --git a/migrations/sql/000022_update_mod_targets.down.sql b/migrations/sql/000022_update_mod_targets.down.sql new file mode 100644 index 00000000..6a196931 --- /dev/null +++ b/migrations/sql/000022_update_mod_targets.down.sql @@ -0,0 +1,64 @@ +-- ID generation -- +-- This is not as random as the original ID, but it should be good enough -- +Create or replace function update_mod_platform_down_random_string(length integer) returns text as +$$ +declare + chars text[] := '{0,1,2,3,4,5,6,7,8,9,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}'; + result text := ''; + i integer := 0; +begin + if length < 0 then + raise exception 'Given length cannot be less than 0'; + end if; + for i in 1..length loop + result := result || chars[1+random()*(array_length(chars, 1)-1)]; + end loop; + return result; +end; +$$ language plpgsql; + +-- Mod version targets -- +ALTER TABLE version_targets RENAME TO mod_archs; + +ALTER TABLE mod_archs + RENAME COLUMN version_id TO mod_version_arch_id; +ALTER TABLE mod_archs + RENAME COLUMN target_name TO platform; +ALTER TABLE mod_archs + ADD COLUMN id varchar(14); + +UPDATE mod_archs SET id = update_mod_platform_down_random_string(14) WHERE true; + +ALTER TABLE mod_archs + ALTER COLUMN id SET NOT NULL; + +ALTER TABLE mod_archs + DROP CONSTRAINT version_targets_version_id_fkey, + DROP CONSTRAINT version_targets_pkey, + ADD CONSTRAINT mod_archs_pkey PRIMARY KEY (id); + +CREATE INDEX IF NOT EXISTS idx_mod_arch_id ON mod_archs (mod_version_arch_id, platform); + +-- SML version targets -- +ALTER TABLE sml_version_targets RENAME TO sml_archs; + +ALTER TABLE sml_archs + RENAME COLUMN version_id TO sml_version_arch_id; +ALTER TABLE sml_archs + RENAME COLUMN target_name TO platform; +ALTER TABLE sml_archs + ADD COLUMN id varchar(14); + +UPDATE sml_archs SET id = update_mod_platform_down_random_string(14) WHERE true; + +ALTER TABLE sml_archs + ALTER COLUMN id SET NOT NULL; + +ALTER TABLE sml_archs + DROP CONSTRAINT sml_version_targets_version_id_fkey, + DROP CONSTRAINT sml_version_targets_pkey, + ADD CONSTRAINT sml_archs_pkey PRIMARY KEY (id); + +CREATE INDEX IF NOT EXISTS idx_sml_archs_id ON sml_archs (sml_version_arch_id, platform); + +DROP FUNCTION update_mod_platform_down_random_string(length integer); \ No newline at end of file diff --git a/migrations/sql/000022_update_mod_targets.up.sql b/migrations/sql/000022_update_mod_targets.up.sql new file mode 100644 index 00000000..687af423 --- /dev/null +++ b/migrations/sql/000022_update_mod_targets.up.sql @@ -0,0 +1,31 @@ +-- Mod version targets -- +ALTER TABLE mod_archs RENAME TO version_targets; + +DROP INDEX idx_mod_arch_id; + +ALTER TABLE version_targets + RENAME COLUMN mod_version_arch_id TO version_id; +ALTER TABLE version_targets + RENAME COLUMN platform TO target_name; +ALTER TABLE version_targets + DROP COLUMN id; + +ALTER TABLE version_targets + ADD CONSTRAINT version_targets_version_id_fkey FOREIGN KEY (version_id) REFERENCES versions (id), + ADD CONSTRAINT version_targets_pkey PRIMARY KEY (version_id, target_name); + +ALTER TABLE sml_archs RENAME TO sml_version_targets; + +-- SML version targets -- +DROP INDEX idx_sml_archs_id; + +ALTER TABLE sml_version_targets + RENAME COLUMN sml_version_arch_id TO version_id; +ALTER TABLE sml_version_targets + RENAME COLUMN platform TO target_name; +ALTER TABLE sml_version_targets + DROP COLUMN id; + +ALTER TABLE sml_version_targets + ADD CONSTRAINT sml_version_targets_version_id_fkey FOREIGN KEY (version_id) REFERENCES sml_versions (id), + ADD CONSTRAINT sml_version_targets_pkey PRIMARY KEY (version_id, target_name); \ No newline at end of file diff --git a/migrations/sql/000023_migrate_versions_to_targets.down.sql b/migrations/sql/000023_migrate_versions_to_targets.down.sql new file mode 100644 index 00000000..702a1db7 --- /dev/null +++ b/migrations/sql/000023_migrate_versions_to_targets.down.sql @@ -0,0 +1,15 @@ +--Mod Targets-- +DELETE FROM version_targets + USING versions + WHERE version_targets.version_id = versions.id AND + version_targets.target_name = 'Windows' AND + version_targets.key = versions.key AND + version_targets.hash = versions.hash AND + version_targets.size = versions.size; + +--SML Targets-- +DELETE FROM sml_version_targets + USING sml_versions + WHERE sml_version_targets.version_id = sml_versions.id AND + sml_version_targets.target_name = 'Windows' AND + sml_version_targets.link = replace(sml_versions.link, '/tag/', '/download/') || '/SML.zip'; diff --git a/migrations/sql/000023_migrate_versions_to_targets.up.sql b/migrations/sql/000023_migrate_versions_to_targets.up.sql new file mode 100644 index 00000000..94e32af7 --- /dev/null +++ b/migrations/sql/000023_migrate_versions_to_targets.up.sql @@ -0,0 +1,13 @@ +-- Mod Targets -- +INSERT INTO version_targets (version_id, target_name, key, hash, size) +SELECT id, 'Windows', key, hash, size +FROM versions +WHERE NOT EXISTS(SELECT 1 FROM version_targets WHERE version_targets.version_id = versions.id) +ON CONFLICT DO NOTHING; + +-- SML Targets -- +INSERT INTO sml_version_targets (version_id, target_name, link) +SELECT id, 'Windows', replace(link, '/tag/', '/download/') || '/SML.zip' +FROM sml_versions +WHERE version LIKE '3%' +ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/migrations/sql/000024_add_sml_engine_version.down.sql b/migrations/sql/000024_add_sml_engine_version.down.sql new file mode 100644 index 00000000..87f13a5e --- /dev/null +++ b/migrations/sql/000024_add_sml_engine_version.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE sml_versions + DROP COLUMN engine_version; \ No newline at end of file diff --git a/migrations/sql/000024_add_sml_engine_version.up.sql b/migrations/sql/000024_add_sml_engine_version.up.sql new file mode 100644 index 00000000..3a018fef --- /dev/null +++ b/migrations/sql/000024_add_sml_engine_version.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE sml_versions + ADD COLUMN IF NOT EXISTS engine_version varchar(16) default '4.26'; \ No newline at end of file diff --git a/models/filters.go b/models/filters.go index 885e9ce1..c7fd5d61 100644 --- a/models/filters.go +++ b/models/filters.go @@ -328,85 +328,3 @@ func ProcessBootstrapVersionFilter(filter map[string]interface{}) (*BootstrapVer return base, nil } - -type SMLArchFilter struct { - Limit *int `json:"limit" validate:"omitempty,min=1,max=100"` - Offset *int `json:"offset" validate:"omitempty,min=0"` - OrderBy *generated.SMLArchFields `json:"order_by"` - Order *generated.Order `json:"order"` - Search *string `json:"search" validate:"omitempty,min=3"` - Ids []string `json:"ids" validate:"omitempty,max=100"` -} - -func DefaultSMLArchFilter() *SMLArchFilter { - limit := 10 - offset := 0 - order := generated.OrderDesc - orderBy := generated.SMLArchFieldsPlatform - return &SMLArchFilter{ - Limit: &limit, - Offset: &offset, - Ids: nil, - Order: &order, - OrderBy: &orderBy, - } -} - -func ProcessSMLArchFilter(filter map[string]interface{}) (*SMLArchFilter, error) { - base := DefaultSMLArchFilter() - - if filter == nil { - return base, nil - } - - if err := ApplyChanges(filter, base); err != nil { - return nil, err - } - - if err := dataValidator.Struct(base); err != nil { - return nil, errors.Wrap(err, "failed to validate SMLArchFilter") - } - - return base, nil -} - -type ModArchFilter struct { - Limit *int `json:"limit" validate:"omitempty,min=1,max=100"` - Offset *int `json:"offset" validate:"omitempty,min=0"` - OrderBy *generated.ModArchFields `json:"order_by"` - Order *generated.Order `json:"order"` - Search *string `json:"search" validate:"omitempty,min=3"` - Ids []string `json:"ids" validate:"omitempty,max=100"` -} - -func DefaultModArchFilter() *ModArchFilter { - limit := 10 - offset := 0 - order := generated.OrderDesc - orderBy := generated.ModArchFieldsPlatform - return &ModArchFilter{ - Limit: &limit, - Offset: &offset, - Ids: nil, - Order: &order, - OrderBy: &orderBy, - } -} - -func ProcessModArchFilter(filter map[string]interface{}) (*ModArchFilter, error) { - base := DefaultModArchFilter() - - if filter == nil { - return base, nil - } - - if err := ApplyChanges(filter, base); err != nil { - return nil, err - } - - if err := dataValidator.Struct(base); err != nil { - return nil, errors.Wrap(err, "failed to validate ModArchFilter") - } - - return base, nil -} diff --git a/nodes/mod.go b/nodes/mod.go index 725fb109..4fdeeb3d 100644 --- a/nodes/mod.go +++ b/nodes/mod.go @@ -289,20 +289,20 @@ func downloadModVersion(c echo.Context) error { return c.Redirect(302, storage.GenerateDownloadLink(version.Key)) } -// @Summary Download a Mod Version by Platform +// @Summary Download a Mod Version by TargetName // @Tags Mod -// @Description Download a mod version by mod ID and version ID and Platform +// @Description Download a mod version by mod ID and version ID and TargetName // @Accept json // @Produce json // @Param modId path string true "Mod ID" // @Param versionId path string true "Version ID" -// @Param versionId path string true "Platform" +// @Param target path string true "TargetName" // @Success 200 -// @Router /mod/{modId}/versions/{versionId}/{platform}/download [get] -func downloadModVersionArch(c echo.Context) error { +// @Router /mod/{modId}/versions/{versionId}/{target}/download [get] +func downloadModVersionTarget(c echo.Context) error { modID := c.Param("modId") versionID := c.Param("versionId") - platform := c.Param("platform") + target := c.Param("target") mod := postgres.GetModByID(c.Request().Context(), modID) @@ -316,15 +316,42 @@ func downloadModVersionArch(c echo.Context) error { return c.String(404, "version not found, modID:"+modID+" versionID:"+versionID) } - arch := postgres.GetModArchByPlatform(c.Request().Context(), versionID, platform) + versionTarget := postgres.GetVersionTarget(c.Request().Context(), versionID, target) - if arch == nil { - return c.String(404, "platform not found, modID:"+modID+" versionID:"+versionID+" platform:"+platform) + if versionTarget == nil { + return c.String(404, "target not found, modID:"+modID+" versionID:"+versionID+" target:"+target) } if redis.CanIncrement(c.RealIP(), "download", "version:"+versionID, time.Hour*4) { postgres.IncrementVersionDownloads(c.Request().Context(), version) } - return c.Redirect(302, storage.GenerateDownloadLink(arch.Key)) + return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key)) +} + +// @Summary Retrieve all Mod Versions +// @Tags Mod +// @Description Retrieve all mod versions by mod ID +// @Accept json +// @Produce json +// @Param modId path string true "Mod ID" +// @Success 200 +// @Router /mod/{modId}/versions/all [get] +func getAllModVersions(c echo.Context) (interface{}, *ErrorResponse) { + modID := c.Param("modId") + + mod := postgres.GetModByID(c.Request().Context(), modID) + + if mod == nil { + return nil, &ErrorModNotFound + } + + versions := postgres.GetAllModVersionsWithDependencies(c.Request().Context(), mod.ID) + + converted := make([]*Version, len(versions)) + for k, v := range versions { + converted[k] = TinyVersionToVersion(&v) + } + + return converted, nil } diff --git a/nodes/mod_types.go b/nodes/mod_types.go index 6cb453b0..25cab078 100644 --- a/nodes/mod_types.go +++ b/nodes/mod_types.go @@ -48,16 +48,60 @@ func ModToMod(mod *postgres.Mod, short bool) *Mod { } type Version struct { - UpdatedAt time.Time `json:"updated_at"` - CreatedAt time.Time `json:"created_at"` - ID string `json:"id"` - Version string `json:"version"` - SMLVersion string `json:"sml_version"` - Changelog string `json:"changelog"` - Stability string `json:"stability"` - ModID string `json:"mod_id"` - Downloads uint `json:"downloads"` - Approved bool `json:"approved"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + ID string `json:"id,omitempty"` + Version string `json:"version,omitempty"` + SMLVersion string `json:"sml_version,omitempty"` + Changelog string `json:"changelog,omitempty"` + Stability string `json:"stability,omitempty"` + ModID string `json:"mod_id,omitempty"` + Dependencies []VersionDependency `json:"dependencies,omitempty"` + Targets []VersionTarget `json:"targets,omitempty"` + Downloads uint `json:"downloads,omitempty"` + Approved bool `json:"approved,omitempty"` +} + +type VersionDependency struct { + ModID string `json:"mod_id"` + Condition string `json:"condition"` + Optional bool `json:"optional"` +} + +type VersionTarget struct { + VersionID string `json:"version_id"` + TargetName string `json:"target_name"` + Key string `json:"key"` + Hash string `json:"hash"` + Size int64 `json:"size"` +} + +func TinyVersionToVersion(version *postgres.TinyVersion) *Version { + var dependencies []VersionDependency + if version.Dependencies != nil { + dependencies = make([]VersionDependency, len(version.Dependencies)) + for i, v := range version.Dependencies { + dependencies[i] = VersionDependencyToVersionDependency(v) + } + } + + var targets []VersionTarget + if version.Targets != nil { + targets = make([]VersionTarget, len(version.Targets)) + for i, v := range version.Targets { + targets[i] = VersionTargetToVersionTarget(v) + } + } + + return &Version{ + UpdatedAt: version.UpdatedAt, + CreatedAt: version.CreatedAt, + ID: version.ID, + Version: version.Version, + SMLVersion: version.SMLVersion, + Dependencies: dependencies, + Targets: targets, + } } func VersionToVersion(version *postgres.Version) *Version { @@ -75,6 +119,24 @@ func VersionToVersion(version *postgres.Version) *Version { } } +func VersionDependencyToVersionDependency(version postgres.VersionDependency) VersionDependency { + return VersionDependency{ + ModID: version.ModID, + Condition: version.Condition, + Optional: version.Optional, + } +} + +func VersionTargetToVersionTarget(version postgres.VersionTarget) VersionTarget { + return VersionTarget{ + VersionID: version.VersionID, + TargetName: version.TargetName, + Key: version.Key, + Hash: version.Hash, + Size: version.Size, + } +} + type ModUser struct { UserID string `json:"user_id"` Role string `json:"role"` diff --git a/nodes/routes.go b/nodes/routes.go index 8cd75b83..b6c6cf6d 100755 --- a/nodes/routes.go +++ b/nodes/routes.go @@ -13,9 +13,11 @@ func RegisterModRoutes(router *echo.Group) { router.GET("/:modId/versions", dataWrapper(getModVersions)) router.GET("/:modId/authors", dataWrapper(getModAuthors)) + router.GET("/:modId/versions/all", dataWrapper(getAllModVersions)) + router.GET("/:modId/versions/:versionId", dataWrapper(getModVersion)) router.GET("/:modId/versions/:versionId/download", downloadModVersion) - router.GET("/:modId/versions/:versionId/:platform/download", downloadModVersionArch) + router.GET("/:modId/versions/:versionId/:target/download", downloadModVersionTarget) } func RegisterModsRoutes(router *echo.Group) { @@ -48,7 +50,7 @@ func RegisterUsersRoutes(router *echo.Group) { func RegisterVersionRoutes(router *echo.Group) { router.GET("/:versionId", dataWrapper(getVersion)) router.GET("/:versionId/download", downloadVersion) - router.GET("/:versionId/:platform/download", downloadModArch) + router.GET("/:versionId/:target/download", downloadModTarget) } func RegisterSMLRoutes(router *echo.Group) { diff --git a/nodes/version.go b/nodes/version.go index 178064f0..1da2eeef 100644 --- a/nodes/version.go +++ b/nodes/version.go @@ -54,19 +54,19 @@ func downloadVersion(c echo.Context) error { return c.Redirect(302, storage.GenerateDownloadLink(version.Key)) } -// @Summary Download a Platform +// @Summary Download a TargetName // @Tags Version -// @Tags Platform -// @Description Download a mod version by version ID and Platform +// @Tags TargetName +// @Description Download a mod version by version ID and TargetName // @Accept json // @Produce json // @Param versionId path string true "Version ID" -// @Param versionId path string true "Version ID" +// @Param target path string true "TargetName" // @Success 200 -// @Router /versions/{versionId}/{platform}/download [get] -func downloadModArch(c echo.Context) error { +// @Router /versions/{versionId}/{target}/download [get] +func downloadModTarget(c echo.Context) error { versionID := c.Param("versionId") - platform := c.Param("platform") + target := c.Param("target") version := postgres.GetVersion(c.Request().Context(), versionID) @@ -74,15 +74,15 @@ func downloadModArch(c echo.Context) error { return c.String(404, "version not found, versionID:"+versionID) } - arch := postgres.GetModArchByPlatform(c.Request().Context(), versionID, platform) + versionTarget := postgres.GetVersionTarget(c.Request().Context(), versionID, target) - if arch == nil { - return c.String(404, "platform not found, versionID:"+versionID+" platform:"+platform) + if versionTarget == nil { + return c.String(404, "target not found, versionID:"+versionID+" target:"+target) } if redis.CanIncrement(c.RealIP(), "download", "version:"+versionID, time.Hour*4) { postgres.IncrementVersionDownloads(c.Request().Context(), version) } - return c.Redirect(302, storage.GenerateDownloadLink(arch.Key)) + return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key)) } diff --git a/proto/parser/.gitignore b/proto/parser/.gitignore new file mode 100644 index 00000000..9b0b440d --- /dev/null +++ b/proto/parser/.gitignore @@ -0,0 +1 @@ +*.pb.go \ No newline at end of file diff --git a/proto/parser/parser.proto b/proto/parser/parser.proto new file mode 100644 index 00000000..913d5f6a --- /dev/null +++ b/proto/parser/parser.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +option go_package = "github.com/satisfactorymodding/smr-api/proto/parser"; + +service Parser { + rpc Parse (ParseRequest) returns (stream AssetResponse); +} + +message ParseRequest { + bytes zip_data = 1; + string engine_version = 2; +} + +message AssetResponse { + string path = 1; + bytes data = 2; +} \ No newline at end of file diff --git a/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go b/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go index b3d7a76c..32ca8213 100644 --- a/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go +++ b/redis/jobs/consumers/consumer_scan_mod_on_virus_total.go @@ -38,10 +38,7 @@ func ScanModOnVirusTotalConsumer(ctx context.Context, payload []byte) error { log.Info().Msgf("starting virus scan of mod %s version %s", task.ModID, task.VersionID) version := postgres.GetVersion(ctx, task.VersionID) - mod := postgres.GetModByID(ctx, version.ModID) - - modArch := postgres.GetModArchByPlatform(ctx, task.VersionID, "Combined") - link := storage.GenerateDownloadLink(modArch.Key) + link := storage.GenerateDownloadLink(version.Key) response, _ := http.Get(link) @@ -92,9 +89,5 @@ func ScanModOnVirusTotalConsumer(ctx context.Context, payload []byte) error { go integrations.NewVersion(util.ReWrapCtx(ctx), version) } - go storage.DeleteModArch(ctx, mod.ID, mod.Name, version.ID, "Combined") - go storage.DeleteModArch(ctx, mod.ID, mod.Name, version.Version, "Combined") - go postgres.Delete(ctx, modArch) - return nil } diff --git a/schemas/mod_archs.graphql b/schemas/mod_archs.graphql deleted file mode 100644 index 648fc180..00000000 --- a/schemas/mod_archs.graphql +++ /dev/null @@ -1,38 +0,0 @@ -### Types - -scalar ModArchID - -type ModArch { - id: ModArchID! - ModVersionID: String! - platform: String! - asset: String! - size: Int - hash: String -} - -enum ModArchFields { - platform -} - -type GetModArchs { - arch: [ModArch!]! -} - -### Inputs - -input ModArchFilter { - limit: Int - offset: Int - order_by: ModArchFields - order: Order - search: String - ids: [String!] -} - -### Queries - -extend type Query { - getModArch(id: ModArchID!): ModArch - getModArchs(filter: ModArchFilter): GetModArchs! -} \ No newline at end of file diff --git a/schemas/sml_archs.graphql b/schemas/sml_archs.graphql deleted file mode 100644 index e3cdb327..00000000 --- a/schemas/sml_archs.graphql +++ /dev/null @@ -1,46 +0,0 @@ -### Types - -scalar SMLArchID - -type SMLArch { - id: SMLArchID! - SMLVersionID: String! - platform: String! - link: String! -} - -enum SMLArchFields { - platform -} - -type GetSMLArchs { - arch: [SMLArch!]! -} - -### Inputs - -input SMLArchFilter { - limit: Int - offset: Int - order_by: SMLArchFields - order: Order - search: String - ids: [String!] -} - -input NewSMLArch { - platform: String! - link: String! -} - -input UpdateSMLArch { - platform: String! - link: String! -} - -### Queries - -extend type Query { - getSMLArch(smlVersionId: SMLVersionID!): SMLArch - getSMLArchs(filter: SMLArchFilter): GetSMLArchs! -} \ No newline at end of file diff --git a/schemas/sml_version.graphql b/schemas/sml_version.graphql index ef207ca2..9299c3fe 100755 --- a/schemas/sml_version.graphql +++ b/schemas/sml_version.graphql @@ -8,15 +8,22 @@ type SMLVersion { satisfactory_version: Int! stability: VersionStabilities! link: String! - arch: [SMLArch]! + targets: [SMLVersionTarget]! changelog: String! date: Date! bootstrap_version: String + engine_version: String! updated_at: Date! created_at: Date! } +type SMLVersionTarget { + VersionID: SMLVersionID! + targetName: TargetName! + link: String! +} + type GetSMLVersions { sml_versions: [SMLVersion!]! count: Int! @@ -37,10 +44,11 @@ input NewSMLVersion { satisfactory_version: Int! stability: VersionStabilities! link: String! - arch: [NewSMLArch!]! + targets: [NewSMLVersionTarget!]! changelog: String! date: Date! bootstrap_version: String + engine_version: String! } input UpdateSMLVersion { @@ -48,10 +56,21 @@ input UpdateSMLVersion { satisfactory_version: Int stability: VersionStabilities link: String - arch: [UpdateSMLArch]! + targets: [UpdateSMLVersionTarget]! changelog: String date: Date bootstrap_version: String + engine_version: String +} + +input NewSMLVersionTarget { + targetName: TargetName! + link: String! +} + +input UpdateSMLVersionTarget { + targetName: TargetName! + link: String! } input SMLVersionFilter { diff --git a/schemas/version.graphql b/schemas/version.graphql index 2513639f..168d0de8 100755 --- a/schemas/version.graphql +++ b/schemas/version.graphql @@ -32,7 +32,7 @@ type Version { updated_at: Date! created_at: Date! link: String! - arch: [ModArch]! + targets: [VersionTarget]! metadata: String size: Int hash: String @@ -41,6 +41,14 @@ type Version { dependencies: [VersionDependency!]! } +type VersionTarget { + VersionID: VersionID! + targetName: TargetName! + link: String! + size: Int + hash: String +} + type CreateVersionResponse { auto_approved: Boolean! version: Version diff --git a/schemas/version_target.graphql b/schemas/version_target.graphql new file mode 100644 index 00000000..2d77e3d1 --- /dev/null +++ b/schemas/version_target.graphql @@ -0,0 +1,5 @@ +enum TargetName { + Windows, + WindowsServer, + LinuxServer +} \ No newline at end of file diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..216a4f80 --- /dev/null +++ b/shell.nix @@ -0,0 +1,11 @@ +{ pkgs ? import {} }: + +pkgs.mkShell { + nativeBuildInputs = with pkgs.buildPackages; [ + libwebp + go + protobuf + protoc-gen-go-grpc + minio-client + ]; +} diff --git a/storage/b2.go b/storage/b2.go index 10ec7ad2..a616608d 100644 --- a/storage/b2.go +++ b/storage/b2.go @@ -222,3 +222,7 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) { ContentType: data.ContentType, }, nil } + +func (b2o *B2) List(key string) ([]Object, error) { + return nil, nil // no-op +} diff --git a/storage/s3.go b/storage/s3.go index d8c09c71..b9d97522 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -203,6 +203,15 @@ func (s3o *S3) Delete(key string) error { } if len(objects) == 0 { + _, err = s3o.S3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(viper.GetString("storage.bucket")), + Key: aws.String(cleanedKey), + }) + + if err != nil { + return errors.Wrap(err, "failed to delete objects") + } + return nil } @@ -237,3 +246,23 @@ func (s3o *S3) Meta(key string) (*ObjectMeta, error) { ContentType: data.ContentType, }, nil } + +func (s3o *S3) List(prefix string) ([]Object, error) { + objects, err := s3o.S3Client.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(viper.GetString("storage.bucket")), + Prefix: aws.String(prefix), + }) + if err != nil { + return nil, errors.Wrap(err, "failed to list objects") + } + + out := make([]Object, len(objects.Contents)) + for i, obj := range objects.Contents { + out[i] = Object{ + Key: obj.Key, + LastModified: obj.LastModified, + } + } + + return out, nil +} diff --git a/storage/storage.go b/storage/storage.go index af214dd5..5597aaa7 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -9,13 +9,12 @@ import ( "fmt" "io" "strings" + "time" "github.com/avast/retry-go/v3" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" - - "github.com/satisfactorymodding/smr-api/db/postgres" ) type Storage interface { @@ -29,6 +28,7 @@ type Storage interface { Rename(from string, to string) error Delete(key string) error Meta(key string) (*ObjectMeta, error) + List(key string) ([]Object, error) } type ObjectMeta struct { @@ -36,6 +36,11 @@ type ObjectMeta struct { ContentType *string } +type Object struct { + Key *string + LastModified *time.Time +} + type Config struct { Type string `json:"type"` Bucket string `json:"bucket"` @@ -273,7 +278,7 @@ func RenameVersion(ctx context.Context, modID string, name string, versionID str return true, fmt.Sprintf("/mods/%s/%s.smod", modID, EncodeName(cleanName)+"-"+version) } -func DeleteVersion(ctx context.Context, modID string, name string, versionID string) bool { +func DeleteMod(ctx context.Context, modID string, name string, versionID string) bool { if storage == nil { return false } @@ -291,45 +296,17 @@ func DeleteVersion(ctx context.Context, modID string, name string, versionID str return true } -func DeleteMod(ctx context.Context, modID string, name string, versionID string) bool { +func DeleteModTarget(ctx context.Context, modID string, name string, versionID string, target string) bool { if storage == nil { return false } cleanName := cleanModName(name) + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+target+"-"+versionID) - query := postgres.GetModVersion(ctx, modID, versionID) - - if query != nil && len(query.Arch) != 0 { - for _, link := range query.Arch { - if success := DeleteModArch(ctx, modID, cleanName, versionID, link.Platform); !success { - return false - } - } - } else { - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+versionID) - - log.Info().Str("key", key).Msg("deleting mod") - if err := storage.Delete(key); err != nil { - log.Ctx(ctx).Err(err).Msg("failed to delete version") - return false - } - } - - return true -} - -func DeleteModArch(ctx context.Context, modID string, name string, versionID string, platform string) bool { - if storage == nil { - return false - } - - cleanName := cleanModName(name) - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+platform+"-"+versionID) - - log.Info().Str("key", key).Msg("deleting mod arch") + log.Info().Str("key", key).Msg("deleting mod target") if err := storage.Delete(key); err != nil { - log.Ctx(ctx).Err(err).Msg("failed to delete version link") + log.Err(err).Msg("failed to delete version target") return false } @@ -396,105 +373,107 @@ func EncodeName(name string) string { return result } -func SeparateMod(ctx context.Context, body []byte, modID, name string, versionID string, modVersion string) bool { +func SeparateModTarget(ctx context.Context, body []byte, modID, name, modVersion, target string) (bool, string, string, int64) { zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { - return false + return false, "", "", 0 } - ModPlatforms := []string{"Combined", "WindowsNoEditor", "WindowsServer", "LinuxServer"} cleanName := cleanModName(name) - bufPlatform := bytes.NewBuffer(body) - - for _, ModPlatform := range ModPlatforms { - if ModPlatform != "Combined" { - bufPlatform = new(bytes.Buffer) - zipWriter := zip.NewWriter(bufPlatform) - for _, file := range zipReader.File { - if strings.HasPrefix(file.Name, ".pdb") || strings.HasPrefix(file.Name, ".debug") || !strings.Contains(file.Name, ModPlatform) { - continue - } - - err = WriteZipFile(ctx, file, ModPlatform, zipWriter) - - if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to write zip to " + ModPlatform + " smod") - return false - } - } + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) - zipWriter.Close() + for _, file := range zipReader.File { + if !strings.HasPrefix(file.Name, target+"/") && file.Name != target+"/" { + continue } - key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+ModPlatform+"-"+modVersion) + err = copyModFileToArchZip(file, zipWriter, strings.TrimPrefix(file.Name, target+"/")) - err = WriteModArch(ctx, key, versionID, ModPlatform, bufPlatform) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to save " + ModPlatform + " smod") - return false + log.Err(err).Msg("failed to add file to " + target + " archive") + return false, "", "", 0 } } - return true + zipWriter.Close() + + key := fmt.Sprintf("/mods/%s/%s.smod", modID, cleanName+"-"+target+"-"+modVersion) + + _, err = storage.Put(ctx, key, bytes.NewReader(buf.Bytes())) + if err != nil { + log.Err(err).Msg("failed to save " + target + " archive") + return false, "", "", 0 + } + + hash := sha256.New() + hash.Write(buf.Bytes()) + + return true, key, hex.EncodeToString(hash.Sum(nil)), int64(buf.Len()) } -func WriteZipFile(ctx context.Context, file *zip.File, platform string, zipWriter *zip.Writer) error { - fileName := strings.ReplaceAll(file.Name, platform+"/", "") - zipFile, err := zipWriter.Create(fileName) +func copyModFileToArchZip(file *zip.File, zipWriter *zip.Writer, newName string) error { + fileHeader := file.FileHeader + fileHeader.Name = newName + + zipFile, err := zipWriter.CreateHeader(&fileHeader) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to create smod file for " + platform) - return errors.Wrap(err, "Failed to open smod file for "+platform) + return errors.Wrap(err, "failed to create file") } rawFile, err := file.Open() if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to open smod file for " + platform) - return errors.Wrap(err, "Failed to open smod file for "+platform) + return errors.Wrap(err, "failed to open file") } + defer rawFile.Close() buf := new(bytes.Buffer) _, err = buf.ReadFrom(rawFile) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to read from buffer for " + platform) - return errors.Wrap(err, "Failed to read from buffer for "+platform) + return errors.Wrap(err, "failed to read file") } _, err = zipFile.Write(buf.Bytes()) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to write to smod file: " + platform) - return errors.Wrap(err, "Failed to write smod file for "+platform) + return errors.Wrap(err, "failed to write file") } return nil } -func WriteModArch(ctx context.Context, key string, versionID string, platform string, buffer *bytes.Buffer) error { - _, err := storage.Put(ctx, key, bytes.NewReader(buffer.Bytes())) +func DeleteOldModAssets(modReference string, before time.Time) { + list, err := storage.List(fmt.Sprintf("/assets/mods/%s", modReference)) if err != nil { - log.Ctx(ctx).Err(err).Msg("failed to write smod: " + key) - return errors.Wrap(err, "Failed to load smod:"+key) + log.Err(err).Msg("failed to list assets") + return } - hash := sha256.New() - hash.Write(buffer.Bytes()) + for _, object := range list { + if object.Key == nil { + continue + } - dbModArch := &postgres.ModArch{ - ModVersionID: versionID, - Platform: platform, - Key: key, - Hash: hex.EncodeToString(hash.Sum(nil)), - Size: int64(len(buffer.Bytes())), + if object.LastModified == nil || object.LastModified.Before(before) { + if err := storage.Delete(*object.Key); err != nil { + log.Err(err).Str("key", *object.Key).Msg("failed deleting old asset") + return + } + } } +} - _, err = postgres.CreateModArch(ctx, dbModArch) +func UploadModAsset(ctx context.Context, modReference string, path string, data []byte) { + if storage == nil { + return + } + + key := fmt.Sprintf("/assets/mods/%s/%s", modReference, strings.TrimPrefix(path, "/")) + _, err := storage.Put(ctx, key, bytes.NewReader(data)) if err != nil { - log.Ctx(ctx).Err(err).Msg("Failed to create ModArch: " + versionID + "-" + platform) - return errors.Wrap(err, "Failed to create ModArch: "+versionID+"-"+platform) + log.Err(err).Str("path", path).Msg("failed to upload mod asset") } - - return nil } diff --git a/storage/wasabi.go b/storage/wasabi.go index 3a0a015a..c35b864e 100644 --- a/storage/wasabi.go +++ b/storage/wasabi.go @@ -118,3 +118,7 @@ func (wasabi *Wasabi) Delete(key string) error { func (wasabi *Wasabi) Meta(key string) (*ObjectMeta, error) { return nil, errors.New("Unsupported") } + +func (wasabi *Wasabi) List(key string) ([]Object, error) { + return nil, errors.New("Unsupported") +} diff --git a/tools.go b/tools.go index 0a80bbc3..bf720453 100644 --- a/tools.go +++ b/tools.go @@ -6,5 +6,6 @@ package smr import _ "github.com/99designs/gqlgen" import _ "github.com/swaggo/swag/cmd/swag" +//go:generate protoc -I./proto --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative proto/parser/parser.proto //go:generate go run github.com/99designs/gqlgen generate //go:generate go run github.com/swaggo/swag/cmd/swag init --generalInfo cmd/api/serve.go diff --git a/util/converter/converter_windows.go b/util/converter/converter_windows.go index 02a77a7b..9ab1eb71 100755 --- a/util/converter/converter_windows.go +++ b/util/converter/converter_windows.go @@ -2,28 +2,39 @@ package converter import ( "bytes" + "context" + "image" + "github.com/chai2010/webp" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "image" + + // GIF Support _ "image/gif" + // JPEG Support _ "image/jpeg" + // PNG Support _ "image/png" ) -func ConvertAnyImageToWebp(imageAsBytes []byte) ([]byte, error) { - imageData, _, err := image.Decode(bytes.NewReader(imageAsBytes)) - +func ConvertAnyImageToWebp(ctx context.Context, imageAsBytes []byte) ([]byte, error) { + imageData, imageType, err := image.Decode(bytes.NewReader(imageAsBytes)) if err != nil { - err := errors.Wrap(err, "error converting image to webp") - log.Error(err) - return nil, err + message := "error converting image to webp" + log.Err(err).Msg(message) + return nil, errors.Wrap(err, message) } result := bytes.NewBuffer(make([]byte, 0)) + if imageType == "gif" { + message := "converting gif to webp not supported on windows" + log.Err(err).Msg(message) + return nil, errors.Wrap(err, message) + } + if err := webp.Encode(result, imageData, nil); err != nil { - return nil, err + return nil, errors.Wrap(err, "error converting image to webp") } return result.Bytes(), nil diff --git a/util/flags.go b/util/flags.go new file mode 100644 index 00000000..196968c2 --- /dev/null +++ b/util/flags.go @@ -0,0 +1,13 @@ +package util + +import "github.com/spf13/viper" + +type FeatureFlag string + +const ( + FeatureFlagAllowMultiTargetUpload = "allow_multi_target_upload" +) + +func FlagEnabled(flag FeatureFlag) bool { + return viper.GetBool("feature_flags." + string(flag)) +} diff --git a/validation/extractor.go b/validation/extractor.go new file mode 100644 index 00000000..9526e7b3 --- /dev/null +++ b/validation/extractor.go @@ -0,0 +1,92 @@ +package validation + +import ( + "encoding/json" + "fmt" + "regexp" +) + +func ExtractMetadata(raw []byte) (map[string]map[string][]interface{}, error) { + meta := make(map[string][]map[string]interface{}) + + if err := json.Unmarshal(raw, &meta); err != nil { + return nil, fmt.Errorf("failed extracting meta: %w", err) + } + + out := make(map[string]map[string][]interface{}) + + for fileName, data := range meta { + bpTypes := make(map[string]string) + for i, obj := range data { + if i == 0 && obj["Type"] != "BlueprintGeneratedClass" { + break + } + + if obj["Type"] == "BlueprintGeneratedClass" { + superName := obj["SuperStruct"].(map[string]interface{})["ObjectName"].(string) + _, objName := splitName(superName) + bpTypes[obj["Name"].(string)] = objName + continue + } + + if obj["Properties"] != nil { + classType := obj["Type"].(string) + if _, ok := ignoredClasses[classType]; ok { + continue + } + + if _, ok := out[fileName]; !ok { + out[fileName] = make(map[string][]interface{}) + } + + typ := bpTypes[classType] + if typ == "" { + typ = classType + } + + out[fileName][typ] = append(out[fileName][typ], rewriteRecursive(obj["Properties"])) + } + } + } + + return out, nil +} + +var objNameRegex = regexp.MustCompile(`^(.+?)'(.+?)'$`) + +func splitName(n string) (string, string) { + matches := objNameRegex.FindStringSubmatch(n) + return matches[1], matches[2] +} + +func rewriteRecursive(obj interface{}) interface{} { + switch b := obj.(type) { + case map[string]interface{}: + if mapHas("CultureInvariantString", b) { + return b["CultureInvariantString"] + } else if mapHas("ObjectName", b) && mapHas("ObjectPath", b) { + _, val := splitName(b["ObjectName"].(string)) + return val + } else if mapHas("AssetPathName", b) && mapHas("SubPathString", b) { + return b["AssetPathName"] + } else { + newOut := make(map[string]interface{}) + for k, v := range b { + newOut[k] = rewriteRecursive(v) + } + return newOut + } + case []interface{}: + newOut := make([]interface{}, len(b)) + for i, v := range b { + newOut[i] = rewriteRecursive(v) + } + return newOut + } + return obj +} + +func mapHas(key string, mp map[string]interface{}) bool { + _, ok := mp[key] + return ok +} diff --git a/validation/validation.go b/validation/validation.go index 839317bb..813eb35f 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -9,22 +9,40 @@ import ( "encoding/json" "fmt" "io" + "path" "path/filepath" + "sort" "strconv" "strings" + "time" "github.com/Masterminds/semver/v3" - "github.com/Vilsol/ue4pak/parser" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/xeipuuv/gojsonschema" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/satisfactorymodding/smr-api/db/postgres" + "github.com/satisfactorymodding/smr-api/proto/parser" + "github.com/satisfactorymodding/smr-api/storage" ) +var AllowedTargets = []string{"Windows", "WindowsServer", "LinuxServer"} + type ModObject struct { Path string `json:"path"` Type string `json:"type"` } +type ModType int + +const ( + DataJSON ModType = iota + UEPlugin = 1 + MultiTargetUEPlugin = 2 +) + type ModInfo struct { Dependencies map[string]string `json:"dependencies"` OptionalDependencies map[string]string `json:"optional_dependencies"` @@ -35,7 +53,9 @@ type ModInfo struct { SMLVersion string `json:"sml_version"` Objects []ModObject `json:"objects"` Metadata []map[string]map[string][]interface{} `json:"-"` + Targets []string `json:"-"` Size int64 `json:"-"` + Type ModType `json:"-"` } var ( @@ -78,7 +98,7 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal dataFile = v break } - if v.Name == "WindowsNoEditor/"+modReference+".uplugin" { + if v.Name == modReference+".uplugin" { uPlugin = v break } @@ -101,44 +121,90 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal } if modInfo == nil { - return nil, errors.New("missing WindowsNoEditor/" + modReference + ".uplugin or data.json") + // Neither data.json nor .uplugin found, try multi-target .uplugin + modInfo, err = validateMultiTargetPlugin(archive, withValidation, modReference) + if err != nil { + return nil, err + } + } + + if modInfo == nil { + return nil, errors.New("missing " + modReference + ".uplugin or data.json") } if withMetadata { // Extract all possible metadata - modInfo.Metadata = make([]map[string]map[string][]interface{}, 0) - for _, obj := range modInfo.Objects { - if strings.ToLower(obj.Type) == "pak" { - for _, archiveFile := range archive.File { - if obj.Path == archiveFile.Name { - data, err := archiveFile.Open() - if err != nil { - log.Err(err).Msg("failed opening archive file") - break - } - - pakData, err := io.ReadAll(data) - if err != nil { - log.Err(err).Msg("failed reading archive file") - break - } - - reader := &parser.PakByteReader{ - Bytes: pakData, - } - - pak, err := AttemptExtractDataFromPak(ctx, reader) - if err != nil { - log.Err(err).Msg("failed parsing archive file") - break - } - - modInfo.Metadata = append(modInfo.Metadata, pak) - break - } + conn, err := grpc.Dial("localhost:50051", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, errors.Wrap(err, "failed to connect to metadata server") + } + defer conn.Close() + + engineVersion := "4.26" + + //nolint + if postgres.DBCtx(nil) != nil { + smlVersions := postgres.GetSMLVersions(ctx, nil) + + // Sort decrementing by version + sort.Slice(smlVersions, func(a, b int) bool { + return semver.MustParse(smlVersions[a].Version).Compare(semver.MustParse(smlVersions[b].Version)) > 0 + }) + + for _, version := range smlVersions { + constraint, err := semver.NewConstraint(modInfo.SMLVersion) + if err != nil { + return nil, errors.Wrap(err, "failed to create semver constraint") + } + + if constraint.Check(semver.MustParse(version.Version)) { + engineVersion = version.EngineVersion + break } } } + + parserClient := parser.NewParserClient(conn) + stream, err := parserClient.Parse(ctx, &parser.ParseRequest{ + ZipData: body, + EngineVersion: engineVersion, + }, + grpc.MaxCallSendMsgSize(1024*1024*1024), // 1GB + grpc.MaxCallRecvMsgSize(1024*1024*1024), // 1GB + ) + if err != nil { + return nil, errors.Wrap(err, "failed to parse mod") + } + + defer func(stream parser.Parser_ParseClient) { + err := stream.CloseSend() + if err != nil { + log.Ctx(ctx).Err(err).Msg("failed closing parser stream") + } + }(stream) + + beforeUpload := time.Now().Add(-time.Minute) + for { + asset, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, errors.Wrap(err, "failed reading parser stream") + } + + if asset.Path == "metadata.json" { + out, err := ExtractMetadata(asset.Data) + if err != nil { + return nil, err + } + modInfo.Metadata = append(modInfo.Metadata, out) + } + + storage.UploadModAsset(ctx, modInfo.ModReference, asset.GetPath(), asset.GetData()) + } + + storage.DeleteOldModAssets(modInfo.ModReference, beforeUpload) } modInfo.Size = int64(len(body)) @@ -244,6 +310,8 @@ func validateDataJSON(archive *zip.Reader, dataFile *zip.File, withValidation bo } } + modInfo.Type = DataJSON + return &modInfo, nil } @@ -357,5 +425,81 @@ func validateUPluginJSON(archive *zip.Reader, uPluginFile *zip.File, withValidat return nil, errors.New(uPluginFile.Name + " doesn't contain SML as a dependency.") } + modInfo.Type = UEPlugin + return &modInfo, nil } + +func validateMultiTargetPlugin(archive *zip.Reader, withValidation bool, modReference string) (*ModInfo, error) { + var targets []string + var uPluginFiles []*zip.File + for _, file := range archive.File { + if path.Base(file.Name) == modReference+".uplugin" && path.Dir(file.Name) != "." { + targets = append(targets, path.Dir(file.Name)) + uPluginFiles = append(uPluginFiles, file) + } + } + + if withValidation { + for _, target := range targets { + found := false + for _, allowedTarget := range AllowedTargets { + if target == allowedTarget { + found = true + break + } + } + if !found { + return nil, errors.New("multi-target plugin contains invalid target: " + target) + } + } + + for _, file := range archive.File { + found := false + for _, target := range targets { + if strings.HasPrefix(file.Name, target+"/") { + found = true + break + } + } + if !found { + return nil, errors.New("multi-target plugin contains file outside of target directories: " + file.Name) + } + } + } + + if len(uPluginFiles) == 0 { + return nil, errors.New("multi-target plugin doesn't contain any .uplugin files") + } + + if withValidation { + var lastData []byte + for _, uPluginFile := range uPluginFiles { + file, err := uPluginFile.Open() + if err != nil { + return nil, errors.Wrap(err, "failed to open .uplugin file") + } + data, err := io.ReadAll(file) + file.Close() + if err != nil { + return nil, errors.Wrap(err, "failed to read .uplugin file") + } + + if lastData != nil && !bytes.Equal(lastData, data) { + return nil, errors.New("multi-target plugin contains different .uplugin files") + } + lastData = data + } + } + + // All the .uplugin files should be the same at this point (assuming validation is enabled) + modInfo, err := validateUPluginJSON(archive, uPluginFiles[0], withValidation, modReference) + if err != nil { + return nil, errors.Wrap(err, "failed to validate multi-target plugin") + } + + modInfo.Targets = targets + modInfo.Type = MultiTargetUEPlugin + + return modInfo, nil +} diff --git a/validation/virustotal.go b/validation/virustotal.go index f9104ed7..bbfe8dc5 100644 --- a/validation/virustotal.go +++ b/validation/virustotal.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" ) var client *vt.Client @@ -32,44 +33,83 @@ type AnalysisResults struct { } func ScanFiles(ctx context.Context, files []io.Reader, names []string) (bool, error) { - for i, file := range files { - scan, err := client.NewFileScanner().Scan(file, names[i], nil) - if err != nil { - return false, errors.Wrap(err, "failed to scan file") + errs, gctx := errgroup.WithContext(context.Background()) + fileCount := len(files) + + c := make(chan bool) + + for i := 0; i < fileCount; i++ { + count := i + errs.Go(func() error { + ok, err := scanFile(gctx, files[count], names[count]) + if err != nil { + return errors.Wrap(err, "failed to scan file") + } + c <- ok + return nil + }) + } + go func() { + _ = errs.Wait() + close(c) + }() + + success := true + for res := range c { + if !res { + success = false + break } + } - analysisID := scan.ID() + if err := errs.Wait(); err != nil { + return false, errors.Wrap(err, "failed to scan file") + } - log.Info().Msgf("uploaded virus scan for file %s and analysis ID: %s", names[i], analysisID) + return success, nil +} - for { - time.Sleep(time.Second * 15) +func scanFile(ctx context.Context, file io.Reader, name string) (bool, error) { + scan, err := client.NewFileScanner().Scan(file, name, nil) + if err != nil { + return false, errors.Wrap(err, "failed to scan file") + } - var target AnalysisResults - _, err = client.GetData(vt.URL("analyses/%s", analysisID), &target) + analysisID := scan.ID() - if err != nil { - return false, errors.Wrap(err, "failed to get analysis results") - } + log.Info().Msgf("uploaded virus scan for file %s and analysis ID: %s", name, analysisID) - if target.Attributes.Status != "completed" { - continue - } + for { + time.Sleep(time.Second * 15) - if target.Attributes.Stats == nil { - return false, nil - } + var target AnalysisResults + _, err = client.GetData(vt.URL("analyses/%s", analysisID), &target) - if target.Attributes.Stats.Malicious == nil || target.Attributes.Stats.Suspicious == nil { - return false, nil - } + if err != nil { + return false, errors.Wrap(err, "failed to get analysis results") + } - if *target.Attributes.Stats.Malicious > 0 || *target.Attributes.Stats.Suspicious > 0 { - return false, nil - } + if target.Attributes.Status != "completed" { + continue + } - break + if target.Attributes.Stats == nil { + log.Error().Msgf("no stats available. failing file: %s", name) + return false, nil + } + + if target.Attributes.Stats.Malicious == nil || target.Attributes.Stats.Suspicious == nil { + log.Error().Msgf("unable to determine malicious or suspicious File: %s", name) + return false, nil } + + // Why 1? Well because some company made a shitty AI and it flags random mods. + if *target.Attributes.Stats.Malicious > 1 || *target.Attributes.Stats.Suspicious > 1 { + log.Error().Msgf("suspicious or malicious file found: %s", name) + return false, nil + } + + break } return true, nil