From 4626c2f0f4c3b0fd0cd73b33a45c68e449a06a45 Mon Sep 17 00:00:00 2001 From: "Darryl L. Pierce" Date: Sat, 15 Jun 2024 16:04:23 -0400 Subject: [PATCH] Added a delay between requests [#19] * Added support for reference ID matching; * Added loading the site detail URL. --- INSTALLATION.md | 6 ++- .../AbstractComicVineScrapingAction.java | 2 + .../actions/ComicVineGetAllIssuesAction.java | 10 +++++ .../ComicVineGetIssueDetailsAction.java | 1 + .../ComicVineGetIssueWithDetailsAction.java | 1 + .../actions/ComicVineGetVolumesAction.java | 10 +++++ .../adaptors/ComicVineMetadataAdaptor.java | 41 ++++++++++++++++++- .../ComicVineMetadataAdaptorProvider.java | 7 ++++ .../comicvine/model/ComicVineIssue.java | 4 ++ .../ComicVineGetVolumesActionTest.java | 2 + .../ComicVineMetadataAdaptorProviderTest.java | 28 +++++++++++++ .../ComicVineMetadataAdaptorTest.java | 14 +++++++ 12 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProviderTest.java diff --git a/INSTALLATION.md b/INSTALLATION.md index a84fa81..d5edbc8 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -17,9 +17,13 @@ ![properties dialog](images/metadata-properties-dialog-1.png) 1. Click on the **Add Property** button. 1. Create a new property with a property name of **comic-vine.api-key**. -1. For the property value enter your API key. +1. Set your ComicVine API key value. +1. Set the delay (in seconds) to use between volume and issue requests. ![properties dialog](images/metadata-properties-dialog-2.png) 1. **Optional** mark this as your preferred metadata source. 1. Click the **Save** button. +**NOTE:** Any delay of less than 1 second is ignored and the default of 1 +second will be used. + You should now be able to scrape comics using the ComicVine database! diff --git a/src/main/java/org/comixedproject/metadata/comicvine/actions/AbstractComicVineScrapingAction.java b/src/main/java/org/comixedproject/metadata/comicvine/actions/AbstractComicVineScrapingAction.java index 99a4106..55a34f8 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/actions/AbstractComicVineScrapingAction.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/actions/AbstractComicVineScrapingAction.java @@ -71,6 +71,8 @@ public abstract class AbstractComicVineScrapingAction extends AbstractScrapin @Getter @Setter protected String baseUrl; @Getter @Setter private String apiKey; + @Getter @Setter private long delay = 0L; + private String maskedApiKey; protected AbstractComicVineScrapingAction() { diff --git a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetAllIssuesAction.java b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetAllIssuesAction.java index 17d411a..fd09ffd 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetAllIssuesAction.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetAllIssuesAction.java @@ -112,6 +112,16 @@ public List execute() throws MetadataException { done = response.getOffset() + response.getNumberOfPageResults() >= response.getNumberOfTotalResults(); + + if (!done) { + log.trace("Sleeping for {}s", this.getDelay()); + try { + Thread.sleep(this.getDelay() * 1000L); + } catch (InterruptedException error) { + log.error("ComicVine get volumes action interrupted", error); + throw new RuntimeException(error); + } + } } return result; diff --git a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueDetailsAction.java b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueDetailsAction.java index 23dcbde..66f98fe 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueDetailsAction.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueDetailsAction.java @@ -84,6 +84,7 @@ public IssueDetailsMetadata execute() throws MetadataException { result.setStoreDate(issueDetails.getStoreDate()); result.setTitle(issueDetails.getTitle()); result.setDescription(issueDetails.getDescription()); + result.setWebAddress(issueDetails.getSiteDetailURL()); for (ComicVineCharacter character : issueDetails.getCharacters()) result.getCharacters().add(character.getName()); diff --git a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueWithDetailsAction.java b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueWithDetailsAction.java index 4a9fbeb..9a55210 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueWithDetailsAction.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetIssueWithDetailsAction.java @@ -54,6 +54,7 @@ public ComicVineIssue execute() throws MetadataException { this.addField("store_date"); this.addField("name"); this.addField("description"); + this.addField("site_detail_url"); this.addField("character_credits"); this.addField("team_credits"); this.addField("location_credits"); diff --git a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesAction.java b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesAction.java index 548aff2..c1e726e 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesAction.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesAction.java @@ -113,6 +113,16 @@ public List execute() throws MetadataException { (hitMaxRecordLimit(result)) || (response.getOffset() + response.getNumberOfPageResults()) >= response.getNumberOfTotalResults(); + + if (!done) { + log.trace("Sleeping for {}s", this.getDelay()); + try { + Thread.sleep(this.getDelay() * 1000L); + } catch (InterruptedException error) { + log.error("ComicVine get volumes action interrupted", error); + throw new RuntimeException(error); + } + } } return result; diff --git a/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptor.java b/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptor.java index 7294054..74f4bfa 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptor.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptor.java @@ -18,10 +18,12 @@ package org.comixedproject.metadata.comicvine.adaptors; -import static org.comixedproject.metadata.comicvine.adaptors.ComicVineMetadataAdaptorProvider.PROPERTY_API_KEY; -import static org.comixedproject.metadata.comicvine.adaptors.ComicVineMetadataAdaptorProvider.PROVIDER_NAME; +import static org.comixedproject.metadata.comicvine.adaptors.ComicVineMetadataAdaptorProvider.*; import java.util.List; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import lombok.extern.log4j.Log4j2; import org.comixedproject.metadata.MetadataException; import org.comixedproject.metadata.adaptors.AbstractMetadataAdaptor; @@ -43,9 +45,13 @@ */ @Log4j2 public class ComicVineMetadataAdaptor extends AbstractMetadataAdaptor { + static final String REFERENCE_ID_PATTERN = + "^http[s]?\\:\\/\\/comicvine\\.gamespot\\.com\\/.*\\/4000-([\\d]{1,6}).*"; /** The base URL for ComicVine. */ public static final String BASE_URL = "https://comicvine.gamespot.com"; + public static final long MINIMUM_DELAY_VALUE = 1L; + /** The action to fetch the list of volumes. */ protected ComicVineGetVolumesAction comicVineGetVolumesAction = new ComicVineGetVolumesAction(); @@ -73,6 +79,7 @@ public List getVolumes( this.comicVineGetVolumesAction.setBaseUrl(BASE_URL); this.comicVineGetVolumesAction.setApiKey( this.getSourcePropertyByName(metadataSource.getProperties(), PROPERTY_API_KEY, true)); + this.comicVineGetVolumesAction.setDelay(this.doGetDelayValue(metadataSource)); this.comicVineGetVolumesAction.setSeries(seriesName); this.comicVineGetVolumesAction.setMaxRecords(maxRecords); @@ -91,6 +98,7 @@ public List getAllIssues( this.comicVineGetAllIssuesAction.setBaseUrl(BASE_URL); this.comicVineGetAllIssuesAction.setApiKey( this.getSourcePropertyByName(metadataSource.getProperties(), PROPERTY_API_KEY, true)); + this.comicVineGetAllIssuesAction.setDelay(this.doGetDelayValue(metadataSource)); this.comicVineGetAllIssuesAction.setVolumeId(volume); log.debug("Executing action"); @@ -129,4 +137,33 @@ public IssueDetailsMetadata getIssueDetails( return this.comicVineGetIssueDetailsAction.execute(); } + + @Override + public String getReferenceId(final String webAddress) { + final Pattern pattern = Pattern.compile(REFERENCE_ID_PATTERN); + final Matcher matches = pattern.matcher(webAddress); + String referenceId = null; + if (matches.matches()) { + referenceId = matches.group(1); + } + return referenceId; + } + + private long doGetDelayValue(final MetadataSource metadataSource) { + long result = MINIMUM_DELAY_VALUE; + try { + final String defined = + this.getSourcePropertyByName(metadataSource.getProperties(), PROPERTY_DELAY, false); + if (!Objects.isNull(defined)) { + result = Long.parseLong(defined); + } + } catch (MetadataException | NumberFormatException error) { + log.error("Failed to load property: " + PROPERTY_DELAY, error); + } + if (result < MINIMUM_DELAY_VALUE) { + result = MINIMUM_DELAY_VALUE; + } + log.trace("Returning delay value: {}", result); + return result; + } } diff --git a/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProvider.java b/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProvider.java index 0fc5d62..1ebd207 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProvider.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProvider.java @@ -39,12 +39,14 @@ public class ComicVineMetadataAdaptorProvider extends AbstractMetadataAdaptorPro private static final String VERSION = "2.1-SNAPSHOT"; private static final String HOMEPAGE = "http://www.github.com/comixed/comixed-metadata-comicvine"; + static final String PROPERTY_DELAY = "comic-vine.delay"; /** Creates a default instance. */ public ComicVineMetadataAdaptorProvider() { super(PROVIDER_NAME, VERSION, HOMEPAGE); this.addProperty(PROPERTY_API_KEY); + this.addProperty(PROPERTY_DELAY); } @Override @@ -52,4 +54,9 @@ public MetadataAdaptor create() { log.debug("Creating an instance of the ComicVine metadata adaptor"); return new ComicVineMetadataAdaptor(); } + + @Override + public boolean supportedReference(final String reference) { + return reference.matches(ComicVineMetadataAdaptor.REFERENCE_ID_PATTERN); + } } diff --git a/src/main/java/org/comixedproject/metadata/comicvine/model/ComicVineIssue.java b/src/main/java/org/comixedproject/metadata/comicvine/model/ComicVineIssue.java index 11327c3..4c829ef 100644 --- a/src/main/java/org/comixedproject/metadata/comicvine/model/ComicVineIssue.java +++ b/src/main/java/org/comixedproject/metadata/comicvine/model/ComicVineIssue.java @@ -58,6 +58,10 @@ public class ComicVineIssue { @Getter private String description; + @JsonProperty("site_detail_url") + @Getter + private String siteDetailURL; + @JsonProperty("image") @Getter private ComicVineImage image = new ComicVineImage(); diff --git a/src/test/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesActionTest.java b/src/test/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesActionTest.java index 433bb6d..743de29 100644 --- a/src/test/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesActionTest.java +++ b/src/test/java/org/comixedproject/metadata/comicvine/actions/ComicVineGetVolumesActionTest.java @@ -40,6 +40,7 @@ @RunWith(MockitoJUnitRunner.class) public class ComicVineGetVolumesActionTest { private static final String TEST_API_KEY = "OICU812"; + private static final long TEST_DELAY = 1L; private static final String TEST_VOLUME_NAME = "Action Comics"; private static final String TEST_BAD_RESPONSE_BODY = "This is not JSON"; private static final String TEST_RESPONSE_BODY = @@ -62,6 +63,7 @@ public void setUp() throws IOException, KeyStoreException, NoSuchAlgorithmExcept final String hostname = String.format("http://localhost:%s", this.comicVineServer.getPort()); action.setBaseUrl(hostname); action.setApiKey(TEST_API_KEY); + action.setDelay(TEST_DELAY); action.setSeries(TEST_VOLUME_NAME); } diff --git a/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProviderTest.java b/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProviderTest.java new file mode 100644 index 0000000..48d8125 --- /dev/null +++ b/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorProviderTest.java @@ -0,0 +1,28 @@ +package org.comixedproject.metadata.comicvine.adaptors; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ComicVineMetadataAdaptorProviderTest { + private static final String TEST_GOOD_ADDRESS = + "https://comicvine.gamespot.com/action-comics-futures-end-1-crossroads/4000-463937/"; + private static final String TEST_BAD_ADDRESS = + "https://notcomicvine.gamespot.com/action-comics-futures-end-1-crossroads/4000-463937";; + + @InjectMocks private ComicVineMetadataAdaptorProvider provider; + + @Test + public void testSupportedReferenceWithBadReference() { + assertFalse(provider.supportedReference(TEST_BAD_ADDRESS)); + } + + @Test + public void testSupportedReference() { + assertTrue(provider.supportedReference(TEST_GOOD_ADDRESS)); + } +} diff --git a/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorTest.java b/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorTest.java index 7c2c9a6..156ed02 100644 --- a/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorTest.java +++ b/src/test/java/org/comixedproject/metadata/comicvine/adaptors/ComicVineMetadataAdaptorTest.java @@ -50,10 +50,15 @@ public class ComicVineMetadataAdaptorTest { private static final String TEST_VOLUME_ID = "129"; private static final String TEST_ISSUE_NUMBER = "17"; private static final String TEST_ISSUE_ID = "327"; + private static final String TEST_WEB_ADDRESS = + "https://comicvine.gamespot.com/action-comics-futures-end-1-crossroads/4000-463937"; + private static final String TEST_REFERENCE_ID = "463937"; + private final List volumeMetadataList = new ArrayList<>(); private final List issueMetadataList = new ArrayList<>(); private final List entries = new ArrayList<>(); private final Set metadataSourceProperties = new HashSet<>(); + @InjectMocks private ComicVineMetadataAdaptor adaptor; @Mock private ComicVineGetVolumesAction getVolumesAction; @Mock private ComicVineGetAllIssuesAction getAllIssuesAction; @@ -234,4 +239,13 @@ public void testGetIssueDetails() throws MetadataException { Mockito.verify(getIssueDetailsAction, Mockito.times(1)).setApiKey(TEST_API_KEY); Mockito.verify(getIssueDetailsAction, Mockito.times(1)).setIssueId(TEST_ISSUE_ID); } + + @Test + public void testGetReferenceId() { + final String result = adaptor.getReferenceId(TEST_WEB_ADDRESS); + + assertNotNull(result); + assertFalse(result.isEmpty()); + assertEquals(TEST_REFERENCE_ID, result); + } }