From 2c169a4f1882ac562105f17b1469cffb89000044 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Mon, 8 Jan 2024 09:04:56 +0200 Subject: [PATCH] feat: mod asset listing fix: reindex latest mods --- db/postgres/mod.go | 2 +- gql/gql_utils.go | 25 ++++++++------ gql/resolver_mods.go | 19 +++++++++++ migrations/code/20240108075200_new_parser.go | 25 ++++++++++++++ redis/redis.go | 36 ++++++++++++++++++++ schemas/mod.graphql | 2 ++ storage/b2.go | 22 ++++++++++-- storage/s3.go | 20 ++++++----- storage/storage.go | 27 ++++++++++++++- 9 files changed, 154 insertions(+), 24 deletions(-) create mode 100644 migrations/code/20240108075200_new_parser.go diff --git a/db/postgres/mod.go b/db/postgres/mod.go index 5b5fe9ee..00f29422 100644 --- a/db/postgres/mod.go +++ b/db/postgres/mod.go @@ -121,7 +121,7 @@ func GetModCountNew(ctx context.Context, filter *models.ModFilter, unapproved bo func IncrementModViews(ctx context.Context, mod *Mod) { // TODO unignore - //DBCtx(ctx).Model(mod).Update("views", mod.Views+1) + // DBCtx(ctx).Model(mod).Update("views", mod.Views+1) } func GetMods(ctx context.Context, limit int, offset int, orderBy string, order string, search string, unapproved bool) []Mod { diff --git a/gql/gql_utils.go b/gql/gql_utils.go index 5968c62a..5dbe22cf 100644 --- a/gql/gql_utils.go +++ b/gql/gql_utils.go @@ -2,11 +2,14 @@ package gql import ( "context" + "fmt" "net" "net/http" "strings" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/satisfactorymodding/smr-api/db/postgres" @@ -27,22 +30,22 @@ func WrapMutationTrace(ctx context.Context, action string) (TraceWrapper, contex } func wrapTrace(ctx context.Context, action string, actionType string) (TraceWrapper, context.Context) { - // spanCtx, span := otel.Tracer("gql").Start(ctx, "GraphQL "+action, trace.WithAttributes( - // attribute.String("action_type", "API.graphql."+actionType), - //)) + spanCtx, span := otel.Tracer("gql").Start(ctx, "GraphQL "+action, trace.WithAttributes( + attribute.String("action_type", "API.graphql."+actionType), + )) return TraceWrapper{ - // Span: span, - }, ctx + Span: span, + }, spanCtx } func (wrapper TraceWrapper) end() { - // defer wrapper.Span.End() - // - //if err := recover(); err != nil { - // wrapper.Span.RecordError(fmt.Errorf("panic: %v", err)) - // panic(err) - //} + defer wrapper.Span.End() + + if err := recover(); err != nil { + wrapper.Span.RecordError(fmt.Errorf("panic: %v", err)) + panic(err) + } } // SetStringINNOE sets target if value not nil or empty diff --git a/gql/resolver_mods.go b/gql/resolver_mods.go index 58586c73..83ae6275 100644 --- a/gql/resolver_mods.go +++ b/gql/resolver_mods.go @@ -619,3 +619,22 @@ func (r *queryResolver) ResolveModVersions(ctx context.Context, filter []*genera return modVersions, nil } + +func (r *queryResolver) GetModAssetList(ctx context.Context, modReference string) ([]string, error) { + wrapper, ctx := WrapQueryTrace(ctx, "getModAssetList") + defer wrapper.end() + + list := redis.GetModAssetList(modReference) + if list != nil { + return list, nil + } + + assets, err := storage.ListModAssets(modReference) + if err != nil { + return nil, err + } + + redis.StoreModAssetList(modReference, assets) + + return assets, nil +} diff --git a/migrations/code/20240108075200_new_parser.go b/migrations/code/20240108075200_new_parser.go new file mode 100644 index 00000000..021f9aa0 --- /dev/null +++ b/migrations/code/20240108075200_new_parser.go @@ -0,0 +1,25 @@ +package code + +import ( + "context" + "strings" + + "github.com/lab259/go-migration" + "github.com/rs/zerolog/log" + + "github.com/satisfactorymodding/smr-api/db/postgres" + "github.com/satisfactorymodding/smr-api/migrations/utils" +) + +func init() { + migration.NewCodeMigration( + func(executionContext interface{}) error { + ctx := log.Logger.WithContext(context.TODO()) + utils.ReindexAllModFiles(ctx, true, nil, func(version postgres.Version) bool { + smlVersion := version.SMLVersion + return strings.Contains(smlVersion, "3.6.1") || strings.Contains(smlVersion, "3.7.0") + }) + return nil + }, + ) +} diff --git a/redis/redis.go b/redis/redis.go index d5b829c3..51411c85 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -1,11 +1,14 @@ package redis import ( + "bytes" + "compress/gzip" "context" "encoding/base64" "encoding/binary" "encoding/json" "fmt" + "io" "strconv" "time" @@ -140,3 +143,36 @@ func GetVersionUploadState(versionID string) (*generated.CreateVersionResponse, func FlushRedis() { client.FlushDB() } + +func StoreModAssetList(modReference string, assets []string) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + b, _ := json.Marshal(assets) + _, _ = zw.Write(b) + _ = zw.Close() + + client.Set(fmt.Sprintf("assets:%s", modReference), buf.Bytes(), time.Hour*24) +} + +func GetModAssetList(modReference string) []string { + result, err := client.Get(fmt.Sprintf("assets:%s", modReference)).Result() + if err != nil { + return nil + } + + reader, err := gzip.NewReader(bytes.NewReader([]byte(result))) + if err != nil { + return nil + } + + all, err := io.ReadAll(reader) + if err != nil { + return nil + } + + out := make([]string, 0) + _ = json.Unmarshal(all, &out) + + return out +} diff --git a/schemas/mod.graphql b/schemas/mod.graphql index 0c6cac6f..e4819c30 100755 --- a/schemas/mod.graphql +++ b/schemas/mod.graphql @@ -119,6 +119,8 @@ extend type Query { getMyUnapprovedMods(filter: ModFilter): GetMyMods! @isLoggedIn resolveModVersions(filter: [ModVersionConstraint!]!): [ModVersion!]! + + getModAssetList(modReference: ModID!): [String!]! } ### Mutations diff --git a/storage/b2.go b/storage/b2.go index a616608d..3c0b79e2 100644 --- a/storage/b2.go +++ b/storage/b2.go @@ -223,6 +223,24 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) { }, nil } -func (b2o *B2) List(key string) ([]Object, error) { - return nil, nil // no-op +func (b2o *B2) List(prefix string) ([]Object, error) { + out := make([]Object, 0) + + err := b2o.S3Client.ListObjectsPages(&s3.ListObjectsInput{ + Bucket: aws.String(b2o.Config.Bucket), + Prefix: aws.String(prefix), + }, func(output *s3.ListObjectsOutput, b bool) bool { + for _, obj := range output.Contents { + out = append(out, Object{ + Key: obj.Key, + LastModified: obj.LastModified, + }) + } + return true + }) + if err != nil { + return nil, errors.Wrap(err, "failed to list objects") + } + + return out, nil } diff --git a/storage/s3.go b/storage/s3.go index b9d97522..f2335f4e 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -248,21 +248,23 @@ func (s3o *S3) Meta(key string) (*ObjectMeta, error) { } func (s3o *S3) List(prefix string) ([]Object, error) { - objects, err := s3o.S3Client.ListObjects(&s3.ListObjectsInput{ + out := make([]Object, 0) + + err := s3o.S3Client.ListObjectsPages(&s3.ListObjectsInput{ Bucket: aws.String(viper.GetString("storage.bucket")), Prefix: aws.String(prefix), + }, func(output *s3.ListObjectsOutput, b bool) bool { + for _, obj := range output.Contents { + out = append(out, Object{ + Key: obj.Key, + LastModified: obj.LastModified, + }) + } + return true }) if err != nil { return nil, errors.Wrap(err, "failed to list objects") } - out := make([]Object, len(objects.Contents)) - for i, obj := range objects.Contents { - out[i] = Object{ - Key: obj.Key, - LastModified: obj.LastModified, - } - } - return out, nil } diff --git a/storage/storage.go b/storage/storage.go index 5597aaa7..da94729e 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "io" + "sort" "strings" "time" @@ -445,7 +446,7 @@ func copyModFileToArchZip(file *zip.File, zipWriter *zip.Writer, newName string) } func DeleteOldModAssets(modReference string, before time.Time) { - list, err := storage.List(fmt.Sprintf("/assets/mods/%s", modReference)) + list, err := storage.List(fmt.Sprintf("assets/mods/%s", modReference)) if err != nil { log.Err(err).Msg("failed to list assets") return @@ -477,3 +478,27 @@ func UploadModAsset(ctx context.Context, modReference string, path string, data log.Err(err).Str("path", path).Msg("failed to upload mod asset") } } + +func ListModAssets(modReference string) ([]string, error) { + if storage == nil { + return nil, errors.New("no storage defined") + } + + list, err := storage.List(fmt.Sprintf("assets/mods/%s", modReference)) + if err != nil { + return nil, errors.New("failed to list assets") + } + + out := make([]string, len(list)) + for i, object := range list { + if object.Key == nil { + continue + } + + out[i] = *object.Key + } + + sort.Strings(out) + + return out, nil +}