Skip to content

Commit

Permalink
releaseb strong alts connection
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sammers21
Sammers21 committed Jun 30, 2024
1 parent 27a6713 commit 320ca1b
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 69 deletions.
51 changes: 26 additions & 25 deletions src/io/github/sammers/pla/blizzard/BlizzardAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import static io.github.sammers.pla.logic.Conts.EU;
import static io.github.sammers.pla.logic.Conts.US;

import java.util.concurrent.TimeUnit;

/**
Expand All @@ -45,9 +46,9 @@ public class BlizzardAPI {
private final Map<String, Cutoffs> cutoffs;
private final String clientId;
private final AtomicReference<BlizzardAuthToken> token = new AtomicReference<>();
private final RateLimiter rateLimiter = new RateLimiter(100, TimeUnit.SECONDS, 1000,
Optional.of(new RateLimiter(36000, TimeUnit.HOURS, 1000, Optional.empty(), Main.VTHREAD_SCHEDULER)),
Main.VTHREAD_SCHEDULER);
private final RateLimiter rateLimiter = new RateLimiter(100, TimeUnit.SECONDS, 1000,
Optional.of(new RateLimiter(36000, TimeUnit.HOURS, 1000, Optional.empty(), Main.VTHREAD_SCHEDULER)),
Main.VTHREAD_SCHEDULER);

public BlizzardAPI(String clientId, String clientSecret, WebClient webClient, Refs refs, CharacterCache characterCache, Map<String, Cutoffs> cutoffs) {
this.clientId = clientId;
Expand Down Expand Up @@ -98,7 +99,7 @@ public Maybe<WowAPICharacter> character(String region, String realm, String name
String absoluteURI = "https://" + realRegion + ".api.blizzard.com/profile/wow/character/" + realmSearch + "/" + nameSearch;
return token().flatMapMaybe(blizzardAuthToken -> {
long tick = System.nanoTime();
return maybeResponse(realNamespace, absoluteURI)
return maybeResponse(realNamespace, absoluteURI)
.flatMap(json -> {
Maybe<WowAPICharacter> res;
if (json.getInteger("code") != null && json.getInteger("code") == 404) {
Expand All @@ -113,37 +114,37 @@ public Maybe<WowAPICharacter> character(String region, String realm, String name
bracketFromJson = new JsonArray();
}
Single<List<JsonObject>> bracketList = Maybe.concatEager(
bracketFromJson.stream()
.map(o -> ((JsonObject) o).getString("href"))
.map(ref -> maybeResponse(realNamespace, ref)
).toList()
).toList();
bracketFromJson.stream()
.map(o -> ((JsonObject) o).getString("href"))
.map(ref -> maybeResponse(realNamespace, ref)
).toList()
).toList();
Maybe<JsonObject> achievementsRx = maybeResponse(realNamespace, absoluteURI + "/achievements");
Maybe<JsonObject> mediaRx = maybeResponse(realNamespace, absoluteURI + "/character-media").onErrorReturnItem(new JsonObject());
Maybe<JsonObject> petsRx = maybeResponse(realNamespace, absoluteURI + "/collections/pets");
Maybe<JsonObject> specsRx = maybeResponse(realNamespace, absoluteURI + "/specializations");
return Single.zip(
bracketList,
Maybe.concatEager(List.of(achievementsRx, mediaRx, petsRx, specsRx)).toList(),
Pair::new).flatMapMaybe(pair -> {
List<JsonObject> brackets = pair.getValue0();
List<JsonObject> otherStuff = pair.getValue1();
Optional<WowAPICharacter> prev = Optional.ofNullable(characterCache.getByFullName(Character.fullNameByRealmAndName(name, realm)));
Optional<Cutoffs> ctfs = Optional.ofNullable(cutoffs.get(realRegion));
return Maybe.just(
WowAPICharacter.parse(prev, refs, ctfs, json, pvp,
brackets, otherStuff.get(0), otherStuff.get(1), otherStuff.get(3), otherStuff.get(2), realRegion));
}).doOnSuccess(wowAPICharacter -> {
long elapsed = System.nanoTime() - tick;
log.debug("Parsed character {} in {} ms", wowAPICharacter.fullName(), elapsed / 1000000);
});
bracketList,
Maybe.concatEager(List.of(achievementsRx, mediaRx, petsRx, specsRx)).toList(),
Pair::new).flatMapMaybe(pair -> {
List<JsonObject> brackets = pair.getValue0();
List<JsonObject> otherStuff = pair.getValue1();
Optional<WowAPICharacter> prev = Optional.ofNullable(characterCache.getByFullName(Character.fullNameByRealmAndName(name, realm)));
Optional<Cutoffs> ctfs = Optional.ofNullable(cutoffs.get(realRegion));
WowAPICharacter parsed = WowAPICharacter.parse(characterCache, prev, refs, ctfs, json, pvp,
brackets, otherStuff.get(0), otherStuff.get(1), otherStuff.get(3), otherStuff.get(2), realRegion);
return Maybe.just(parsed);
}).doOnSuccess(wowAPICharacter -> {
long elapsed = System.nanoTime() - tick;
log.debug("Parsed character {} in {} ms", wowAPICharacter.fullName(), elapsed / 1000000);
});
})
.doOnError(e -> log.error("Error parsing character: " + name + " on " + realm + " in " + realRegion, e))
.onErrorResumeNext(Maybe.empty());
}
return res;
});
});
});
}

Maybe<JsonObject> maybeResponse(String namespace, String url) {
Expand All @@ -162,7 +163,7 @@ Maybe<JsonObject> maybeResponse(String namespace, String url) {
return Single.error(er);
})
.flatMapMaybe(resp -> {
log.debug("Got response to" + url+ " " + resp.statusCode());
log.debug("Got response to" + url + " " + resp.statusCode());
if (resp.statusCode() == 200) {
return Maybe.just(resp.bodyAsJsonObject());
} else if (resp.statusCode() == 429 || resp.statusCode() / 100 == 5) {
Expand Down
47 changes: 36 additions & 11 deletions src/io/github/sammers/pla/blizzard/WowAPICharacter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.github.sammers.pla.http.JsonConvertable;
import io.github.sammers.pla.logic.Calculator;
import io.github.sammers.pla.logic.CharAndDiff;
import io.github.sammers.pla.logic.CharacterCache;
import io.github.sammers.pla.logic.Refs;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
Expand All @@ -13,9 +14,7 @@
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -150,7 +149,8 @@ public record WowAPICharacter(long id,
Achievements achievements,
int petHash,
CharacterMedia media,
String talents) implements JsonConvertable {
String talents,
Set<Long> alts) implements JsonConvertable {

private static final Logger log = org.slf4j.LoggerFactory.getLogger(WowAPICharacter.class);

Expand All @@ -165,6 +165,7 @@ public record WowAPICharacter(long id,
}

public static WowAPICharacter parse(
CharacterCache cache,
Optional<WowAPICharacter> previous,
Refs refs,
Optional<Cutoffs> cutoffs,
Expand Down Expand Up @@ -223,8 +224,17 @@ public static WowAPICharacter parse(
: CharacterMedia.parse(characterMedia);
Long lastUpdatedUTCms = Instant.now().toEpochMilli();
Achievements parsedAchievements = Achievements.parse(achievements);
Long id = entries.getLong("id");
int petHash;
if (pets.getJsonArray("pets") == null) {
petHash = -1;
} else {
petHash = pets.getJsonArray("pets").hashCode();
}
Set<Long> alts = cache.alts.getOrDefault(petHash, new HashSet<>(0));
previous.ifPresent(wowAPICharacter -> alts.addAll(wowAPICharacter.alts));
return new WowAPICharacter(
entries.getInteger("id"),
id,
previous.map(WowAPICharacter::hidden).orElse(false),
name,
realm,
Expand All @@ -239,9 +249,10 @@ public static WowAPICharacter parse(
pvpBrackets,
lastUpdatedUTCms,
parsedAchievements,
pets.getJsonArray("pets").encode().hashCode(),
petHash,
media,
talents
talents,
alts
);
}

Expand All @@ -257,6 +268,13 @@ public static WowAPICharacter fromJson(JsonObject entries) {
} else {
brcktsFromJson = array.stream().map(o -> PvpBracket.fromJson((JsonObject) o)).toList();
}
Set<Long> alts;
JsonArray altsArr = entries.getJsonArray("alts");
if (altsArr == null) {
alts = new HashSet<>(0);
} else {
alts = new HashSet<>(altsArr.stream().map(o -> Long.valueOf(o.toString())).collect(Collectors.toSet()));
}
if (entries.getLong("lastUpdatedUTCms") == null) {
entries.put("lastUpdatedUTCms", 0L);
}
Expand Down Expand Up @@ -284,10 +302,16 @@ public static WowAPICharacter fromJson(JsonObject entries) {
Achievements.fromJson(entries.getJsonObject("achievements")),
entries.getInteger("petHash"),
CharacterMedia.fromJson(entries.getJsonObject("media")),
Optional.ofNullable(entries.getString("talents")).orElse("")
Optional.ofNullable(entries.getString("talents")).orElse(""),
alts
);
}

public WowAPICharacter changeAlts(Set<Long> newAlts) {
return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel,
region, brackets, lastUpdatedUTCms, achievements, petHash, media, talents, newAlts);
}

public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracket, List<Character> withWho) {
List<PvpBracket> newBrackets = brackets.stream().map(pvpBracket -> {
PvpBracket res;
Expand Down Expand Up @@ -334,7 +358,7 @@ public WowAPICharacter updatePvpBracketData(CharAndDiff diff, BracketType bracke
}
return res;
}).toList();
return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents);
return new WowAPICharacter(id, hidden, name, realm, gender, fraction, race, activeSpec, level, clazz, itemLevel, region, newBrackets, lastUpdatedUTCms, achievements, petHash, media, talents, alts);
}

public byte[] toGzippedJson() {
Expand All @@ -350,7 +374,7 @@ public static WowAPICharacter fromGzippedJson(byte[] gzippedJson) {

@Override
public int hashCode() {
return fullName().hashCode();
return (int) id;
}

@Override
Expand All @@ -373,7 +397,8 @@ public JsonObject toJson() {
.put("achievements", achievements.toJson())
.put("petHash", petHash)
.put("media", media.toJson())
.put("talents", talents);
.put("talents", talents)
.put("alts", new JsonArray(alts.stream().toList()));
}

}
34 changes: 20 additions & 14 deletions src/io/github/sammers/pla/logic/CharUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public CharUpdater(BlizzardAPI api,
}

public Completable updateCharacters(String region,
int timeWithoutUpdateMin, TimeUnit units,
int timeout, TimeUnit timeoutUnits) {
int timeWithoutUpdateMin, TimeUnit units,
int timeout, TimeUnit timeoutUnits) {
return Completable.defer(() -> {
log.info("Updating characters for region " + region);
// Get top chars from 3v3, 2v2, RBG, shuffle
Expand Down Expand Up @@ -152,11 +152,7 @@ public Completable updateChars(List<String> nickNames, String region) {
public Completable updateChar(String region, String nickName) {
return Completable.defer(() -> {
if (charsLoaded.get()) {
return api.character(region, nickName).flatMapCompletable(wowAPICharacter -> {
characterCache.upsert(wowAPICharacter);
charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter));
return db.upsertCharacter(wowAPICharacter).ignoreElement();
}).onErrorComplete();
return api.character(region, nickName).flatMapCompletable(this::updateCharsDbSize).onErrorComplete();
} else {
log.warn("Not allowing char updates before char load from db");
return Completable.error(new IllegalStateException("Not allowing char updates before char load from db"));
Expand All @@ -165,17 +161,27 @@ public Completable updateChar(String region, String nickName) {
);
}

private Completable updateCharsDbSize(WowAPICharacter wowAPICharacter) {
Set<WowAPICharacter> inconsistencies = characterCache.findAltsInconsistenciesAndFix(wowAPICharacter);
Completable fixInconsistencies;
if (inconsistencies.isEmpty()) {
fixInconsistencies = Completable.complete();
} else {
fixInconsistencies = db.bulkUpdateChars(inconsistencies.stream().toList()).ignoreElement().andThen(Completable.fromAction(() -> {
log.info("Updated {} alts inconsistencies", inconsistencies.size());
}));
}
inconsistencies.forEach(characterCache::upsert);
characterCache.upsert(wowAPICharacter);
charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter));
return db.upsertCharacter(wowAPICharacter).ignoreElement().andThen(fixInconsistencies);
}

public Single<Optional<WowAPICharacter>> updateCharFast(String region, String nickName) {
return Single.defer(() -> {
if (charsLoaded.get()) {
return api.character(region, nickName)
.doAfterSuccess(wowAPICharacter -> {
Main.VTHREAD_SCHEDULER.scheduleDirect(() -> {
characterCache.upsert(wowAPICharacter);
charSearchIndex.insertNickNamesWC(List.of(wowAPICharacter));
db.upsertCharacter(wowAPICharacter).subscribe();
});
})
.doAfterSuccess(wowAPICharacter -> Main.VTHREAD_SCHEDULER.scheduleDirect(() -> updateCharsDbSize(wowAPICharacter).subscribe()))
.map(Optional::of)
.onErrorReturnItem(Optional.empty())
.toSingle(Optional.empty());
Expand Down
29 changes: 25 additions & 4 deletions src/io/github/sammers/pla/logic/CharacterCache.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package io.github.sammers.pla.logic;

import io.github.sammers.pla.blizzard.BracketType;
import io.github.sammers.pla.blizzard.PvpBracket;
import io.github.sammers.pla.blizzard.WowAPICharacter;
import io.github.sammers.pla.db.Character;

Expand All @@ -13,7 +12,7 @@ public class CharacterCache {

private final Map<Long, byte[]> idCache;
private final Map<String, byte[]> nameCache;
private final Map<Integer, Set<Long>> alts;
public final Map<Integer, Set<Long>> alts;

public CharacterCache() {
nameCache = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -69,13 +68,35 @@ public List<WowAPICharacter> upsertGroupDiff(List<CharAndDiff> groupDiff, String
return res;
}

public Set<WowAPICharacter> findAltsInconsistenciesAndFix(WowAPICharacter character) {
Set<Long> idSet = new HashSet<>(character.alts());
Set<WowAPICharacter> charset = new HashSet<>(idSet.stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson).collect(Collectors.toSet()));
charset.add(character);
charset.forEach(ch -> idSet.add(ch.id()));
Set<WowAPICharacter> resl = charset.stream().filter(ch -> {
var specificSet = new HashSet<>(idSet);
specificSet.remove(ch.id());
boolean eq = !specificSet.equals(ch.alts());
return eq;
}).collect(Collectors.toSet());
Set<WowAPICharacter> resm = resl.stream().map(ch -> {
Set<Long> longs = new HashSet<>(idSet);
longs.remove(ch.id());
WowAPICharacter changedAlts = ch.changeAlts(longs);
return changedAlts;
}).collect(Collectors.toSet());
return resm;
}

public Collection<byte[]> values() {
return idCache.values();
}

public Set<WowAPICharacter> altsFor(WowAPICharacter character) {
return Optional.of(alts.get(character.petHash()).stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson
).collect(Collectors.toSet()))
Set<Long> longs = Optional.ofNullable(alts.get(character.petHash())).orElse(new HashSet<>());
longs.addAll(character.alts());
return Optional.of(longs.stream().map(idCache::get).map(WowAPICharacter::fromGzippedJson)
.collect(Collectors.toSet()))
.orElse(Set.of())
.stream()
.filter(c -> !c.hidden())
Expand Down
Loading

0 comments on commit 320ca1b

Please sign in to comment.