From 320ca1bb104ad9680adea90b9301048a0c8de675 Mon Sep 17 00:00:00 2001 From: Sammers21 Date: Sun, 30 Jun 2024 17:29:15 +0300 Subject: [PATCH] releaseb strong alts connection --- .../sammers/pla/blizzard/BlizzardAPI.java | 51 ++++++++++--------- .../sammers/pla/blizzard/WowAPICharacter.java | 47 +++++++++++++---- .../github/sammers/pla/logic/CharUpdater.java | 34 ++++++++----- .../sammers/pla/logic/CharacterCache.java | 29 +++++++++-- src/io/github/sammers/pla/logic/Ladder.java | 31 +++++------ 5 files changed, 123 insertions(+), 69 deletions(-) diff --git a/src/io/github/sammers/pla/blizzard/BlizzardAPI.java b/src/io/github/sammers/pla/blizzard/BlizzardAPI.java index 3ea3b096..c31803f1 100644 --- a/src/io/github/sammers/pla/blizzard/BlizzardAPI.java +++ b/src/io/github/sammers/pla/blizzard/BlizzardAPI.java @@ -27,6 +27,7 @@ import static io.github.sammers.pla.logic.Conts.EU; import static io.github.sammers.pla.logic.Conts.US; + import java.util.concurrent.TimeUnit; /** @@ -45,9 +46,9 @@ public class BlizzardAPI { private final Map cutoffs; private final String clientId; private final AtomicReference token = new AtomicReference<>(); - private final RateLimiter rateLimiter = new RateLimiter(100, TimeUnit.SECONDS, 1000, - Optional.of(new RateLimiter(36000, TimeUnit.HOURS, 1000, Optional.empty(), Main.VTHREAD_SCHEDULER)), - Main.VTHREAD_SCHEDULER); + private final RateLimiter rateLimiter = new RateLimiter(100, TimeUnit.SECONDS, 1000, + Optional.of(new RateLimiter(36000, TimeUnit.HOURS, 1000, Optional.empty(), Main.VTHREAD_SCHEDULER)), + Main.VTHREAD_SCHEDULER); public BlizzardAPI(String clientId, String clientSecret, WebClient webClient, Refs refs, CharacterCache characterCache, Map cutoffs) { this.clientId = clientId; @@ -98,7 +99,7 @@ public Maybe character(String region, String realm, String name String absoluteURI = "https://" + realRegion + ".api.blizzard.com/profile/wow/character/" + realmSearch + "/" + nameSearch; return token().flatMapMaybe(blizzardAuthToken -> { long tick = System.nanoTime(); - return maybeResponse(realNamespace, absoluteURI) + return maybeResponse(realNamespace, absoluteURI) .flatMap(json -> { Maybe res; if (json.getInteger("code") != null && json.getInteger("code") == 404) { @@ -113,37 +114,37 @@ public Maybe character(String region, String realm, String name bracketFromJson = new JsonArray(); } Single> bracketList = Maybe.concatEager( - bracketFromJson.stream() - .map(o -> ((JsonObject) o).getString("href")) - .map(ref -> maybeResponse(realNamespace, ref) - ).toList() - ).toList(); + bracketFromJson.stream() + .map(o -> ((JsonObject) o).getString("href")) + .map(ref -> maybeResponse(realNamespace, ref) + ).toList() + ).toList(); Maybe achievementsRx = maybeResponse(realNamespace, absoluteURI + "/achievements"); Maybe mediaRx = maybeResponse(realNamespace, absoluteURI + "/character-media").onErrorReturnItem(new JsonObject()); Maybe petsRx = maybeResponse(realNamespace, absoluteURI + "/collections/pets"); Maybe specsRx = maybeResponse(realNamespace, absoluteURI + "/specializations"); return Single.zip( - bracketList, - Maybe.concatEager(List.of(achievementsRx, mediaRx, petsRx, specsRx)).toList(), - Pair::new).flatMapMaybe(pair -> { - List brackets = pair.getValue0(); - List otherStuff = pair.getValue1(); - Optional prev = Optional.ofNullable(characterCache.getByFullName(Character.fullNameByRealmAndName(name, realm))); - Optional ctfs = Optional.ofNullable(cutoffs.get(realRegion)); - return Maybe.just( - WowAPICharacter.parse(prev, refs, ctfs, json, pvp, - brackets, otherStuff.get(0), otherStuff.get(1), otherStuff.get(3), otherStuff.get(2), realRegion)); - }).doOnSuccess(wowAPICharacter -> { - long elapsed = System.nanoTime() - tick; - log.debug("Parsed character {} in {} ms", wowAPICharacter.fullName(), elapsed / 1000000); - }); + bracketList, + Maybe.concatEager(List.of(achievementsRx, mediaRx, petsRx, specsRx)).toList(), + Pair::new).flatMapMaybe(pair -> { + List brackets = pair.getValue0(); + List otherStuff = pair.getValue1(); + Optional prev = Optional.ofNullable(characterCache.getByFullName(Character.fullNameByRealmAndName(name, realm))); + Optional ctfs = Optional.ofNullable(cutoffs.get(realRegion)); + WowAPICharacter parsed = WowAPICharacter.parse(characterCache, prev, refs, ctfs, json, pvp, + brackets, otherStuff.get(0), otherStuff.get(1), otherStuff.get(3), otherStuff.get(2), realRegion); + return Maybe.just(parsed); + }).doOnSuccess(wowAPICharacter -> { + long elapsed = System.nanoTime() - tick; + log.debug("Parsed character {} in {} ms", wowAPICharacter.fullName(), elapsed / 1000000); + }); }) .doOnError(e -> log.error("Error parsing character: " + name + " on " + realm + " in " + realRegion, e)) .onErrorResumeNext(Maybe.empty()); } return res; }); - }); + }); } Maybe maybeResponse(String namespace, String url) { @@ -162,7 +163,7 @@ Maybe maybeResponse(String namespace, String url) { return Single.error(er); }) .flatMapMaybe(resp -> { - log.debug("Got response to" + url+ " " + resp.statusCode()); + log.debug("Got response to" + url + " " + resp.statusCode()); if (resp.statusCode() == 200) { return Maybe.just(resp.bodyAsJsonObject()); } else if (resp.statusCode() == 429 || resp.statusCode() / 100 == 5) { diff --git a/src/io/github/sammers/pla/blizzard/WowAPICharacter.java b/src/io/github/sammers/pla/blizzard/WowAPICharacter.java index 7a9cf217..9315a176 100644 --- a/src/io/github/sammers/pla/blizzard/WowAPICharacter.java +++ b/src/io/github/sammers/pla/blizzard/WowAPICharacter.java @@ -5,6 +5,7 @@ import io.github.sammers.pla.http.JsonConvertable; import io.github.sammers.pla.logic.Calculator; import io.github.sammers.pla.logic.CharAndDiff; +import io.github.sammers.pla.logic.CharacterCache; import io.github.sammers.pla.logic.Refs; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -13,9 +14,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -150,7 +149,8 @@ public record WowAPICharacter(long id, Achievements achievements, int petHash, CharacterMedia media, - String talents) implements JsonConvertable { + String talents, + Set alts) implements JsonConvertable { private static final Logger log = org.slf4j.LoggerFactory.getLogger(WowAPICharacter.class); @@ -165,6 +165,7 @@ public record WowAPICharacter(long id, } public static WowAPICharacter parse( + CharacterCache cache, Optional previous, Refs refs, Optional cutoffs, @@ -223,8 +224,17 @@ public static WowAPICharacter parse( : CharacterMedia.parse(characterMedia); Long lastUpdatedUTCms = Instant.now().toEpochMilli(); Achievements parsedAchievements = Achievements.parse(achievements); + Long id = entries.getLong("id"); + int petHash; + if (pets.getJsonArray("pets") == null) { + petHash = -1; + } else { + petHash = pets.getJsonArray("pets").hashCode(); + } + Set alts = cache.alts.getOrDefault(petHash, new HashSet<>(0)); + previous.ifPresent(wowAPICharacter -> alts.addAll(wowAPICharacter.alts)); return new WowAPICharacter( - entries.getInteger("id"), + id, previous.map(WowAPICharacter::hidden).orElse(false), name, realm, @@ -239,9 +249,10 @@ public static WowAPICharacter parse( pvpBrackets, lastUpdatedUTCms, parsedAchievements, - pets.getJsonArray("pets").encode().hashCode(), + petHash, media, - talents + talents, + alts ); } @@ -257,6 +268,13 @@ public static WowAPICharacter fromJson(JsonObject entries) { } else { brcktsFromJson = array.stream().map(o -> PvpBracket.fromJson((JsonObject) o)).toList(); } + Set alts; + JsonArray altsArr = entries.getJsonArray("alts"); + if (altsArr == null) { + alts = new HashSet<>(0); + } else { + alts = new HashSet<>(altsArr.stream().map(o -> Long.valueOf(o.toString())).collect(Collectors.toSet())); + } if (entries.getLong("lastUpdatedUTCms") == null) { entries.put("lastUpdatedUTCms", 0L); } @@ -284,10 +302,16 @@ public static WowAPICharacter fromJson(JsonObject entries) { Achievements.fromJson(entries.getJsonObject("achievements")), entries.getInteger("petHash"), CharacterMedia.fromJson(entries.getJsonObject("media")), - Optional.ofNullable(entries.getString("talents")).orElse("") + Optional.ofNullable(entries.getString("talents")).orElse(""), + alts ); } + public WowAPICharacter changeAlts(Set newAlts) { + return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, + region, brackets, lastUpdatedUTCms, achievements, petHash, media, talents, newAlts); + } + public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracket, List withWho) { List newBrackets = brackets.stream().map(pvpBracket -> { PvpBracket res; @@ -334,7 +358,7 @@ public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracke } return res; }).toList(); - return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents); + return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents, alts); } public byte[] toGzippedJson() { @@ -350,7 +374,7 @@ public static WowAPICharacter fromGzippedJson(byte[] gzippedJson) { @Override public int hashCode() { - return fullName().hashCode(); + return (int) id; } @Override @@ -373,7 +397,8 @@ public JsonObject toJson() { .put("achievements", achievements.toJson()) .put("petHash", petHash) .put("media", media.toJson()) - .put("talents", talents); + .put("talents", talents) + .put("alts", new JsonArray(alts.stream().toList())); } } diff --git a/src/io/github/sammers/pla/logic/CharUpdater.java b/src/io/github/sammers/pla/logic/CharUpdater.java index e8297c05..57da83fb 100644 --- a/src/io/github/sammers/pla/logic/CharUpdater.java +++ b/src/io/github/sammers/pla/logic/CharUpdater.java @@ -52,8 +52,8 @@ public CharUpdater(BlizzardAPI api, } public Completable updateCharacters(String region, - int timeWithoutUpdateMin, TimeUnit units, - int timeout, TimeUnit timeoutUnits) { + int timeWithoutUpdateMin, TimeUnit units, + int timeout, TimeUnit timeoutUnits) { return Completable.defer(() -> { log.info("Updating characters for region " + region); // Get top chars from 3v3, 2v2, RBG, shuffle @@ -152,11 +152,7 @@ public Completable updateChars(List nickNames, String region) { public Completable updateChar(String region, String nickName) { return Completable.defer(() -> { if (charsLoaded.get()) { - return api.character(region, nickName).flatMapCompletable(wowAPICharacter -> { - characterCache.upsert(wowAPICharacter); - charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter)); - return db.upsertCharacter(wowAPICharacter).ignoreElement(); - }).onErrorComplete(); + return api.character(region, nickName).flatMapCompletable(this::updateCharsDbSize).onErrorComplete(); } else { log.warn("Not allowing char updates before char load from db"); return Completable.error(new IllegalStateException("Not allowing char updates before char load from db")); @@ -165,17 +161,27 @@ public Completable updateChar(String region, String nickName) { ); } + private Completable updateCharsDbSize(WowAPICharacter wowAPICharacter) { + Set inconsistencies = characterCache.findAltsInconsistenciesAndFix(wowAPICharacter); + Completable fixInconsistencies; + if (inconsistencies.isEmpty()) { + fixInconsistencies = Completable.complete(); + } else { + fixInconsistencies = db.bulkUpdateChars(inconsistencies.stream().toList()).ignoreElement().andThen(Completable.fromAction(() -> { + log.info("Updated {} alts inconsistencies", inconsistencies.size()); + })); + } + inconsistencies.forEach(characterCache::upsert); + characterCache.upsert(wowAPICharacter); + charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter)); + return db.upsertCharacter(wowAPICharacter).ignoreElement().andThen(fixInconsistencies); + } + public Single> updateCharFast(String region, String nickName) { return Single.defer(() -> { if (charsLoaded.get()) { return api.character(region, nickName) - .doAfterSuccess(wowAPICharacter -> { - Main.VTHREAD_SCHEDULER.scheduleDirect(() -> { - characterCache.upsert(wowAPICharacter); - charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter)); - db.upsertCharacter(wowAPICharacter).subscribe(); - }); - }) + .doAfterSuccess(wowAPICharacter -> Main.VTHREAD_SCHEDULER.scheduleDirect(() -> updateCharsDbSize(wowAPICharacter).subscribe())) .map(Optional::of) .onErrorReturnItem(Optional.empty()) .toSingle(Optional.empty()); diff --git a/src/io/github/sammers/pla/logic/CharacterCache.java b/src/io/github/sammers/pla/logic/CharacterCache.java index 6f1ba7e6..f502c270 100644 --- a/src/io/github/sammers/pla/logic/CharacterCache.java +++ b/src/io/github/sammers/pla/logic/CharacterCache.java @@ -1,7 +1,6 @@ package io.github.sammers.pla.logic; import io.github.sammers.pla.blizzard.BracketType; -import io.github.sammers.pla.blizzard.PvpBracket; import io.github.sammers.pla.blizzard.WowAPICharacter; import io.github.sammers.pla.db.Character; @@ -13,7 +12,7 @@ public class CharacterCache { private final Map idCache; private final Map nameCache; - private final Map> alts; + public final Map> alts; public CharacterCache() { nameCache = new ConcurrentHashMap<>(); @@ -69,13 +68,35 @@ public List upsertGroupDiff(List groupDiff, String return res; } + public Set findAltsInconsistenciesAndFix(WowAPICharacter character) { + Set idSet = new HashSet<>(character.alts()); + Set charset = new HashSet<>(idSet.stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson).collect(Collectors.toSet())); + charset.add(character); + charset.forEach(ch -> idSet.add(ch.id())); + Set resl = charset.stream().filter(ch -> { + var specificSet = new HashSet<>(idSet); + specificSet.remove(ch.id()); + boolean eq = !specificSet.equals(ch.alts()); + return eq; + }).collect(Collectors.toSet()); + Set resm = resl.stream().map(ch -> { + Set longs = new HashSet<>(idSet); + longs.remove(ch.id()); + WowAPICharacter changedAlts = ch.changeAlts(longs); + return changedAlts; + }).collect(Collectors.toSet()); + return resm; + } + public Collection values() { return idCache.values(); } public Set altsFor(WowAPICharacter character) { - return Optional.of(alts.get(character.petHash()).stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson - ).collect(Collectors.toSet())) + Set longs = Optional.ofNullable(alts.get(character.petHash())).orElse(new HashSet<>()); + longs.addAll(character.alts()); + return Optional.of(longs.stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson) + .collect(Collectors.toSet())) .orElse(Set.of()) .stream() .filter(c -> !c.hidden()) diff --git a/src/io/github/sammers/pla/logic/Ladder.java b/src/io/github/sammers/pla/logic/Ladder.java index d1184184..6ecccb2b 100644 --- a/src/io/github/sammers/pla/logic/Ladder.java +++ b/src/io/github/sammers/pla/logic/Ladder.java @@ -511,25 +511,26 @@ private Completable loadCutoffs(String region) { .onErrorComplete(); }); } + private Completable loadCutoffsFromDb(String region) { return Completable.defer(() -> { log.info("Load cutoffs from DB for region " + region); return db.getLastCutoffs(realRegion(region)).map(cutoffs -> { - if (cutoffs.isPresent()) { - Cutoffs ctf = cutoffs.get(); - List.of(TWO_V_TWO, THREE_V_THREE, RBG, SHUFFLE).forEach(bracket -> { - Snapshot s = refs.refByBracket(bracket, region).get(); - if (s != null) { - s.predictCutoffs(bracket, ctf); - } - }); - regionCutoffFromDb.put(oldRegion(region), ctf); - regionCutoffFromDb.put(realRegion(region), ctf); - } - return cutoffs; - }).doOnSuccess(cutoffs -> log.info("Cutoffs from DB for region={} has been loaded", region)) - .ignoreElement() - .onErrorComplete(); + if (cutoffs.isPresent()) { + Cutoffs ctf = cutoffs.get(); + List.of(TWO_V_TWO, THREE_V_THREE, RBG, SHUFFLE).forEach(bracket -> { + Snapshot s = refs.refByBracket(bracket, region).get(); + if (s != null) { + s.predictCutoffs(bracket, ctf); + } + }); + regionCutoffFromDb.put(oldRegion(region), ctf); + regionCutoffFromDb.put(realRegion(region), ctf); + } + return cutoffs; + }).doOnSuccess(cutoffs -> log.info("Cutoffs from DB for region={} has been loaded", region)) + .ignoreElement() + .onErrorComplete(); }); }