feat: new metadata parser, sml engine versions, minimal mod version R…

…EST endpoint

.envrc

@@ -0,0 +1 @@
+use nix

README.md

@@ -49,6 +49,14 @@ Main configuration options:
 
 The config format can be seen in `config/config.go` (each dot means a new level of nesting).
 
+After startup requires the following minio commands to be executed:
+
+```shell
+mc alias set local http://localhost:9000 minio minio123
+mc admin user svcacct add local minio --access-key REPLACE_ME_KEY --secret-key REPLACE_ME_SECRET
+mc anonymous set public local/smr
+```
+
 ## Contributing
 
 Before contributing, please run the [linter](https://golangci-lint.run/) to ensure the code is clean and well-formed:

config.sample.json

@@ -27,7 +27,8 @@
     "key": "REPLACE_ME_KEY",
     "secret": "REPLACE_ME_SECRET",
     "endpoint": "http://localhost:9000",
-    "base_url": "http://localhost:9000"
+    "base_url": "http://localhost:9000",
+    "keypath": "%s/%s/%s"
   },
 
   "oauth": {
@@ -52,5 +53,9 @@
 
   "frontend": {
     "url": "http://localhost:4200"
+  },
+
+  "feature_flags": {
+    "allow_multi_target_upload": false
   }
 }

config/config.go

@@ -100,4 +100,6 @@ func initializeDefaults() {
 	viper.SetDefault("frontend.url", "")
 
 	viper.SetDefault("virustotal.key", "")
+
+	viper.SetDefault("feature_flags.allow_multi_target_upload", false)
 }

db/postgres/postgres_types.go

@@ -94,6 +94,20 @@ type Version struct {
 	Approved   bool `gorm:"default:false;not null"`
 }
 
+type TinyVersion struct {
+	Hash *string
+	Size *int64
+	SMRModel
+	SMLVersion   string              `gorm:"type:varchar(16)"`
+	Version      string              `gorm:"type:varchar(16)"`
+	Arch         []VersionTarget     `gorm:"foreignKey:ModVersionID;preload:true"`
+	Dependencies []VersionDependency `gorm:"foreignKey:VersionID"`
+}
+
+func (TinyVersion) TableName() string {
+	return "versions"
+}
+
 type Guide struct {
 	SMRModel
 	Name             string `gorm:"type:varchar(50)"`
@@ -120,6 +134,7 @@ type SMLVersion struct {
 	Stability           string `sql:"type:version_stability"`
 	Link                string
 	Changelog           string
+	EngineVersion       string
 	Targets             []SMLVersionTarget `gorm:"foreignKey:VersionID"`
 	SatisfactoryVersion int
 }

db/postgres/version.go

@@ -87,6 +87,24 @@ func GetModVersions(ctx context.Context, modID string, limit int, offset int, or
 	return versions
 }
 
+func GetAllModVersionsWithDependencies(ctx context.Context, modID string) []TinyVersion {
+	cacheKey := "GetAllModVersionsWithDependencies_" + modID
+	if versions, ok := dbCache.Get(cacheKey); ok {
+		return versions.([]TinyVersion)
+	}
+
+	var versions []TinyVersion
+	DBCtx(ctx).Debug().
+		Preload("Dependencies").
+		Preload("Arch").
+		Where("approved = ? AND denied = ?", true, false).
+		Find(&versions, "mod_id = ?", modID)
+
+	dbCache.Set(cacheKey, versions, cache.DefaultExpiration)
+
+	return versions
+}
+
 func GetModVersionsNew(ctx context.Context, modID string, filter *models.VersionFilter, unapproved bool) []Version {
 	hash, err := filter.Hash()
 	cacheKey := ""

docker-compose-dev.yml

@@ -25,4 +25,9 @@ services:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
       MINIO_ACCESS_KEY: REPLACE_ME_KEY
-      MINIO_SECRET_KEY: REPLACE_ME_SECRET
+      MINIO_SECRET_KEY: REPLACE_ME_SECRET
+
+  pak_parser:
+    image: ghcr.io/vilsol/ficsit-pak-parser:v0.0.3
+    ports:
+      - 50051:50051

gql/gql_types.go

@@ -139,6 +139,7 @@ func DBSMLVersionToGenerated(smlVersion *postgres.SMLVersion) *generated.SMLVers
 		Date:                smlVersion.Date.Format(time.RFC3339Nano),
 		UpdatedAt:           smlVersion.UpdatedAt.Format(time.RFC3339Nano),
 		CreatedAt:           smlVersion.CreatedAt.Format(time.RFC3339Nano),
+		EngineVersion:       smlVersion.EngineVersion,
 	}
 }
 

gql/resolver_sml_versions.go

@@ -36,6 +36,7 @@ func (r *mutationResolver) CreateSMLVersion(ctx context.Context, smlVersion gene
 		Link:                smlVersion.Link,
 		Changelog:           smlVersion.Changelog,
 		Date:                date,
+		EngineVersion:       smlVersion.EngineVersion,
 	}
 
 	resultSMLVersion, err := postgres.CreateSMLVersion(newCtx, dbSMLVersion)
@@ -101,6 +102,7 @@ func (r *mutationResolver) UpdateSMLVersion(ctx context.Context, smlVersionID st
 	SetStringINNOE(smlVersion.Link, &dbSMLVersion.Link)
 	SetStringINNOE(smlVersion.Changelog, &dbSMLVersion.Changelog)
 	SetDateINN(smlVersion.Date, &dbSMLVersion.Date)
+	SetStringINNOE(smlVersion.EngineVersion, &dbSMLVersion.EngineVersion)
 
 	postgres.Save(newCtx, &dbSMLVersion)
 

gql/versions.go

@@ -53,9 +53,14 @@ func FinalizeVersionUploadAsync(ctx context.Context, mod *postgres.Mod, versionI
 		return nil, errors.New("data.json mod_reference does not match mod reference")
 	}
 
-	if modInfo.Type != validation.UEPlugin {
+	if modInfo.Type == validation.DataJSON {
 		storage.DeleteMod(ctx, mod.ID, mod.Name, versionID)
-		return nil, errors.New("only UE Plugin mods are currently supported")
+		return nil, errors.New("data.json mods are obsolete and not allowed")
+	}
+
+	if modInfo.Type == validation.MultiTargetUEPlugin && !util.FlagEnabled(util.FeatureFlagAllowMultiTargetUpload) {
+		storage.DeleteMod(ctx, mod.ID, mod.Name, versionID)
+		return nil, errors.New("multi-target mods are not allowed")
 	}
 
 	versionMajor := int(modInfo.Semver.Major())

migrations/sql/000024_add_sml_engine_version.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE sml_versions
+    DROP COLUMN engine_version;

migrations/sql/000024_add_sml_engine_version.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE sml_versions
+    ADD COLUMN IF NOT EXISTS engine_version varchar(16) default '4.26';

nodes/mod.go

@@ -328,3 +328,30 @@ func downloadModVersionTarget(c echo.Context) error {
 
 	return c.Redirect(302, storage.GenerateDownloadLink(versionTarget.Key))
 }
+
+// @Summary Retrieve all Mod Versions
+// @Tags Mod
+// @Description Retrieve all mod versions by mod ID
+// @Accept  json
+// @Produce  json
+// @Param modId path string true "Mod ID"
+// @Success 200
+// @Router /mod/{modId}/versions/all [get]
+func getAllModVersions(c echo.Context) (interface{}, *ErrorResponse) {
+	modID := c.Param("modId")
+
+	mod := postgres.GetModByID(c.Request().Context(), modID)
+
+	if mod == nil {
+		return nil, &ErrorModNotFound
+	}
+
+	versions := postgres.GetAllModVersionsWithDependencies(c.Request().Context(), mod.ID)
+
+	converted := make([]*Version, len(versions))
+	for k, v := range versions {
+		converted[k] = TinyVersionToVersion(&v)
+	}
+
+	return converted, nil
+}

nodes/mod_types.go

@@ -48,16 +48,60 @@ func ModToMod(mod *postgres.Mod, short bool) *Mod {
 }
 
 type Version struct {
-	UpdatedAt  time.Time `json:"updated_at"`
-	CreatedAt  time.Time `json:"created_at"`
-	ID         string    `json:"id"`
-	Version    string    `json:"version"`
-	SMLVersion string    `json:"sml_version"`
-	Changelog  string    `json:"changelog"`
-	Stability  string    `json:"stability"`
-	ModID      string    `json:"mod_id"`
-	Downloads  uint      `json:"downloads"`
-	Approved   bool      `json:"approved"`
+	UpdatedAt    time.Time           `json:"updated_at,omitempty"`
+	CreatedAt    time.Time           `json:"created_at,omitempty"`
+	ID           string              `json:"id,omitempty"`
+	Version      string              `json:"version,omitempty"`
+	SMLVersion   string              `json:"sml_version,omitempty"`
+	Changelog    string              `json:"changelog,omitempty"`
+	Stability    string              `json:"stability,omitempty"`
+	ModID        string              `json:"mod_id,omitempty"`
+	Downloads    uint                `json:"downloads,omitempty"`
+	Approved     bool                `json:"approved,omitempty"`
+	Dependencies []VersionDependency `json:"dependencies,omitempty"`
+	Arch         []VersionTarget     `json:"arch,omitempty"`
+}
+
+type VersionDependency struct {
+	ModID     string `json:"mod_id"`
+	Condition string `json:"condition"`
+	Optional  bool   `json:"optional"`
+}
+
+type VersionTarget struct {
+	VersionID  string `json:"version_id"`
+	TargetName string `json:"target_name"`
+	Key        string `json:"key"`
+	Hash       string `json:"hash"`
+	Size       int64  `json:"size"`
+}
+
+func TinyVersionToVersion(version *postgres.TinyVersion) *Version {
+	var dependencies []VersionDependency
+	if version.Dependencies != nil {
+		dependencies = make([]VersionDependency, len(version.Dependencies))
+		for i, v := range version.Dependencies {
+			dependencies[i] = VersionDependencyToVersionDependency(v)
+		}
+	}
+
+	var archs []VersionTarget
+	if version.Arch != nil {
+		archs = make([]VersionTarget, len(version.Arch))
+		for i, v := range version.Arch {
+			archs[i] = VersionArchToVersionArch(v)
+		}
+	}
+
+	return &Version{
+		UpdatedAt:    version.UpdatedAt,
+		CreatedAt:    version.CreatedAt,
+		ID:           version.ID,
+		Version:      version.Version,
+		SMLVersion:   version.SMLVersion,
+		Dependencies: dependencies,
+		Arch:         archs,
+	}
 }
 
 func VersionToVersion(version *postgres.Version) *Version {
@@ -75,6 +119,24 @@ func VersionToVersion(version *postgres.Version) *Version {
 	}
 }
 
+func VersionDependencyToVersionDependency(version postgres.VersionDependency) VersionDependency {
+	return VersionDependency{
+		ModID:     version.ModID,
+		Condition: version.Condition,
+		Optional:  version.Optional,
+	}
+}
+
+func VersionArchToVersionArch(version postgres.VersionTarget) VersionTarget {
+	return VersionTarget{
+		VersionID:  version.VersionID,
+		TargetName: version.TargetName,
+		Key:        version.Key,
+		Hash:       version.Hash,
+		Size:       version.Size,
+	}
+}
+
 type ModUser struct {
 	UserID string `json:"user_id"`
 	Role   string `json:"role"`

nodes/routes.go

@@ -13,6 +13,8 @@ func RegisterModRoutes(router *echo.Group) {
 	router.GET("/:modId/versions", dataWrapper(getModVersions))
 	router.GET("/:modId/authors", dataWrapper(getModAuthors))
 
+	router.GET("/:modId/versions/all", dataWrapper(getAllModVersions))
+
 	router.GET("/:modId/versions/:versionId", dataWrapper(getModVersion))
 	router.GET("/:modId/versions/:versionId/download", downloadModVersion)
 	router.GET("/:modId/versions/:versionId/:target/download", downloadModVersionTarget)

proto/parser/.gitignore

@@ -0,0 +1 @@
+*.pb.go

proto/parser/parser.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+option go_package = "github.com/satisfactorymodding/smr-api/proto/parser";
+
+service Parser {
+  rpc Parse (ParseRequest) returns (stream AssetResponse);
+}
+
+message ParseRequest {
+  bytes zip_data = 1;
+  string engine_version = 2;
+}
+
+message AssetResponse {
+  string path = 1;
+  bytes data = 2;
+}

schemas/sml_version.graphql

@@ -12,6 +12,7 @@ type SMLVersion {
     changelog: String!
     date: Date!
     bootstrap_version: String
+    engine_version: String!
 
     updated_at: Date!
     created_at: Date!
@@ -47,6 +48,7 @@ input NewSMLVersion {
     changelog: String!
     date: Date!
     bootstrap_version: String
+    engine_version: String!
 }
 
 input UpdateSMLVersion {
@@ -58,6 +60,7 @@ input UpdateSMLVersion {
     changelog: String
     date: Date
     bootstrap_version: String
+    engine_version: String
 }
 
 input NewSMLVersionTarget {

shell.nix

@@ -0,0 +1,11 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+pkgs.mkShell {
+  nativeBuildInputs = with pkgs.buildPackages; [
+    libwebp
+    go
+    protobuf
+    protoc-gen-go-grpc
+    minio-client
+  ];
+}

storage/b2.go

@@ -222,3 +222,7 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) {
 		ContentType:   data.ContentType,
 	}, nil
 }
+
+func (b2o *B2) List(key string) ([]Object, error) {
+	return nil, errors.New("Unsupported")
+}