using assets repo for glossary

cmd/core/scheduler/scheduler.go

@@ -37,8 +37,8 @@ func RegisterDefaultTasks(s *scheduler, coreClient core.Client) {
 	s.Add("0 5 * * *", CreateCleanupTaskWorker(coreClient)) // delete expired documents
 
 	// Glossary - Do it around the same time WG releases game updates
-	s.Add("0 10 * * *", UpdateGlossaryWorker(coreClient))
-	s.Add("0 12 * * *", UpdateGlossaryWorker(coreClient))
+	s.Add("10 12 * * *", UpdateGlossaryWorker(coreClient))
+	s.Add("10 10 * * *", UpdateGlossaryWorker(coreClient))
 	// c.AddFunc("40 9 * * 0", updateAchievementsWorker)
 
 	// Averages - Update averages once daily

cmd/core/scheduler/workers.go

@@ -1,15 +1,19 @@
 package scheduler
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
+	"encoding/json"
+	"io"
 	"time"
 
 	"github.com/cufee/aftermath/cmd/core"
 	"github.com/cufee/aftermath/cmd/core/tasks"
-	"golang.org/x/text/language"
 
 	"github.com/cufee/aftermath/internal/database"
 	"github.com/cufee/aftermath/internal/database/models"
+	"github.com/cufee/aftermath/internal/external/github"
 	"github.com/cufee/aftermath/internal/log"
 )
 
@@ -121,38 +125,49 @@ func UpdateGlossaryWorker(client core.Client) func() {
 	return func() {
 		log.Info().Msg("updating glossary cache")
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+		dctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
 		defer cancel()
 
-		glossary, err := client.Wargaming().CompleteVehicleGlossary(ctx, "eu", "en")
+		rc, size, err := github.GetLatestGameAssets(dctx)
 		if err != nil {
-			log.Err(err).Msg("failed to get vehicle glossary")
+			log.Err(err).Msg("failed to download latest game assets")
 			return
 		}
 
-		vehicles := make(map[string]models.Vehicle)
-		for id, data := range glossary {
-			vehicles[id] = models.Vehicle{
-				ID:             id,
-				Tier:           data.Tier,
-				LocalizedNames: map[string]string{language.English.String(): data.Name},
-			}
+		data, err := io.ReadAll(rc)
+		rc.Close()
+		if err != nil {
+			log.Err(err).Msg("failed to read assets data")
+			return
 		}
 
-		vErr, err := client.Database().UpsertVehicles(ctx, vehicles)
+		zr, err := zip.NewReader(bytes.NewReader(data), size)
 		if err != nil {
-			log.Err(err).Msg("failed to save vehicle glossary")
+			log.Err(err).Msg("failed to read assets as zip")
 			return
 		}
-		if len(vErr) > 0 {
-			event := log.Error()
-			for id, err := range vErr {
-				event.Str(id, err.Error())
-			}
-			event.Msg("failed to save some vehicle glossaries")
+
+		vf, err := zr.Open("assets/vehicles.json")
+		if err != nil {
+			log.Err(err).Msg("failed to open vehicles.json in assets.zip")
 			return
-		}
+		} else {
+			var vehicles map[string]models.Vehicle
+			err := json.NewDecoder(vf).Decode(&vehicles)
+			if err != nil {
+				log.Err(err).Msg("failed to decode vehicles.json")
+				return
+			}
 
+			vctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+			defer cancel()
+
+			_, err = client.Database().UpsertVehicles(vctx, vehicles)
+			if err != nil {
+				log.Err(err).Msg("failed to save vehicle glossary")
+				return
+			}
+		}
 		log.Info().Msg("glossary cache updated")
 	}
 }

go.mod

@@ -8,14 +8,17 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/a-h/templ v0.2.731
 	github.com/bwmarrin/discordgo v0.28.1
+	github.com/cufee/aftermath-assets v0.0.1
 	github.com/cufee/am-wg-proxy-next/v2 v2.1.7
 	github.com/disintegration/imaging v1.6.2
 	github.com/fogleman/gg v1.3.0
 	github.com/go-co-op/gocron v1.37.0
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
+	github.com/google/go-github/v63 v63.0.0
 	github.com/joho/godotenv v1.5.1
 	github.com/justinas/alice v1.2.0
 	github.com/lucsky/cuid v1.2.1
+	github.com/matryer/is v1.4.1
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/nlpodyssey/gopickle v0.3.0
 	github.com/pierrec/lz4/v4 v4.1.21
@@ -39,6 +42,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-openapi/inflect v0.21.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/hcl/v2 v2.21.0 // indirect
@@ -51,9 +55,9 @@ require (
 	github.com/tdewolff/parse/v2 v2.7.15 // indirect
 	github.com/zclconf/go-cty v1.14.4 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
 )

go.sum

@@ -20,6 +20,8 @@ github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd
 github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cufee/aftermath-assets v0.0.1 h1:4XBoGrEP/h+H5sHNDI8GF+K4VMTLcrcBaV8uwAfHtrs=
+github.com/cufee/aftermath-assets v0.0.1/go.mod h1:6yCITCiz7POJnUMn1oohvadLA4z5YrFNo3p9EKgRdGU=
 github.com/cufee/am-wg-proxy-next/v2 v2.1.7 h1:cNIQ6hLM/eqqKLuNH0K52AEjzQQqwo7qoi9VOXD6bSM=
 github.com/cufee/am-wg-proxy-next/v2 v2.1.7/go.mod h1:+VxiIdbrdhHdRThASbVmZ5Fz4H6XPCzL3S2Ix6TO/84=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,8 +40,13 @@ github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3a
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v63 v63.0.0 h1:13xwK/wk9alSokujB9lJkuzdmQuVn2QCPeck76wR3nE=
+github.com/google/go-github/v63 v63.0.0/go.mod h1:IqbcrgUmIcEaioWrGYei/09o+ge5vhffGOcxrO0AfmA=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -62,6 +69,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lucsky/cuid v1.2.1 h1:MtJrL2OFhvYufUIn48d35QGXyeTC8tn0upumW9WwTHg=
 github.com/lucsky/cuid v1.2.1/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE=
+github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
+github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -137,8 +146,8 @@ golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -152,8 +161,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
@@ -169,8 +178,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -187,6 +196,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/database/ent/schema/vehicle.go

@@ -4,6 +4,7 @@ import (
 	"entgo.io/ent"
 	"entgo.io/ent/schema/field"
 	"entgo.io/ent/schema/index"
+	"golang.org/x/text/language"
 )
 
 // Vehicle holds the schema definition for the Vehicle entity.
@@ -27,7 +28,7 @@ func (Vehicle) Fields() []ent.Field {
 		field.Int("tier").
 			Min(0). // vehicle that does not exist in official glossary has tier set to 0
 			Max(10),
-		field.JSON("localized_names", map[string]string{}),
+		field.JSON("localized_names", map[language.Tag]string{}),
 	}
 
 }

internal/database/models/vehicle.go

@@ -3,20 +3,17 @@ package models
 import (
 	"fmt"
 
+	assets "github.com/cufee/aftermath-assets/types"
 	"golang.org/x/text/language"
 )
 
-type Vehicle struct {
-	ID             string
-	Tier           int
-	LocalizedNames map[string]string
-}
+type Vehicle assets.Vehicle
 
 func (v Vehicle) Name(locale language.Tag) string {
-	if n := v.LocalizedNames[locale.String()]; n != "" {
+	if n := v.LocalizedNames[locale]; n != "" {
 		return n
 	}
-	if n := v.LocalizedNames[language.English.String()]; n != "" {
+	if n := v.LocalizedNames[language.English]; n != "" {
 		return n
 	}
 	return fmt.Sprintf("Secret Tank %s", v.ID)

internal/external/github/assets.go

@@ -0,0 +1,31 @@
+package github
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/google/go-github/v63/github"
+	"github.com/pkg/errors"
+)
+
+var client = github.NewClient(nil)
+
+func GetLatestGameAssets(ctx context.Context) (io.ReadCloser, int64, error) {
+	release, _, err := client.Repositories.GetLatestRelease(ctx, "cufee", "aftermath-assets")
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "gailed to get latest release")
+	}
+
+	for _, asset := range release.Assets {
+		if *asset.Name == "assets.zip" {
+			rc, _, err := client.Repositories.DownloadReleaseAsset(ctx, "cufee", "aftermath-assets", *asset.ID, http.DefaultClient)
+			if err != nil {
+				return nil, 0, errors.Wrap(err, "failed to download assets.zip")
+			}
+
+			return rc, int64(asset.GetSize()), nil
+		}
+	}
+	return nil, 0, errors.New("assets not found in latest release")
+}