From 0744d823d975d2841a3ac3d27cef1fb68c4f3a7c Mon Sep 17 00:00:00 2001 From: OlegAdmin Date: Thu, 8 Apr 2021 16:08:34 +0300 Subject: [PATCH 1/4] Added city label, fixed parsing start and finish metrics --- .../components/AmServiceMetricsCollector.java | 194 ++++++++++-------- .../service/job/UpdateRepositoryJob.java | 8 +- .../service/service/impl/AmClientImpl.java | 4 +- .../service/service/impl/AmServiceImpl.java | 21 +- .../service/service/impl/RestServiceImpl.java | 4 +- .../service/impl/UpdateServiceImpl.java | 6 +- 6 files changed, 127 insertions(+), 110 deletions(-) diff --git a/src/main/java/com/wine/to/up/am/parser/service/components/AmServiceMetricsCollector.java b/src/main/java/com/wine/to/up/am/parser/service/components/AmServiceMetricsCollector.java index c43e065..5ba5639 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/components/AmServiceMetricsCollector.java +++ b/src/main/java/com/wine/to/up/am/parser/service/components/AmServiceMetricsCollector.java @@ -35,49 +35,59 @@ public class AmServiceMetricsCollector extends CommonMetricsCollector { private static final String WINES_PUBLISHED_TO_KAFKA = "wines_published_to_kafka_count"; private static final String PARSING_COMPLETE_STATUS = "status"; + private static final String REGION_LABEL = "city"; private static final Counter prometheusParsingStartedCounter = Counter.build() .name(PARSING_STARTED) .help("Total number of parsing processes ever started") + .labelNames(REGION_LABEL) .register(); private static final Counter prometheusParsingCompleteCounter = Counter.build() .name(PARSING_COMPLETE) .help("Total number of parsing processes ever completed") - .labelNames(PARSING_COMPLETE_STATUS) + .labelNames(REGION_LABEL, PARSING_COMPLETE_STATUS) .register(); private static final Gauge prometheusParsingInProgressGauge = Gauge.build() .name(PARSING_IN_PROGRESS) .help("Total number of parsing processes currently in progress") + .labelNames(REGION_LABEL) .register(); private static final AtomicInteger micrometerParsingInProgressGauge = Metrics.gauge(PARSING_IN_PROGRESS, new AtomicInteger(0)); private static final Summary prometheusParsingDurationSummary = Summary.build() .name(PARSING_DURATION) .help("The duration of every parsing process completed so far") + .labelNames(REGION_LABEL) .register(); private static final Gauge prometheusTimeSinceLastParsingGauge = Gauge.build() .name(TIME_SINCE_LAST_PARSING) .help("The amount of time since the last successfully completed parsing process") + .labelNames(REGION_LABEL) .register(); private static final AtomicLong micrometerTimeSinceLastParsingGauge = Metrics.gauge(TIME_SINCE_LAST_PARSING, new AtomicLong(0)); private static final Summary prometheusWineDetailsFetchingDurationSummary = Summary.build() .name(WINE_DETAILS_FETCHING_DURATION) .help("The duration of every fetching of a wine details page") + .labelNames(REGION_LABEL) .register(); private static final Summary prometheusWinePageFetchingDurationSummary = Summary.build() .name(WINE_PAGE_FETCHING_DURATION) .help("The duration of every parsing of a wine details page") + .labelNames(REGION_LABEL) .register(); private static final Summary prometheusWineDetailsParsingDurationSummary = Summary.build() .name(WINE_DETAILS_PARSING_DURATION) .help("The duration of every parsing of a wine details page") + .labelNames(REGION_LABEL) .register(); private static final Summary prometheusWinePageParsingDurationSummary = Summary.build() .name(WINE_PAGE_PARSING_DURATION) .help("The duration of every parsing of a wines page") + .labelNames(REGION_LABEL) .register(); private static final Counter prometheusWinesPublishedToKafkaCounter = Counter.build() .name(WINES_PUBLISHED_TO_KAFKA) .help("Number of wines that have been sent to Kafka") + .labelNames(REGION_LABEL) .register(); private static final String PARSED_WINES_SUCCESS = "parsed_wines_success"; @@ -92,203 +102,211 @@ public class AmServiceMetricsCollector extends CommonMetricsCollector { private static final Gauge prometheusParsedWinesSuccessGauge = Gauge.build() .name(PARSED_WINES_SUCCESS) .help("Number of successfully parsed wines") + .labelNames(REGION_LABEL) .register(); private static final AtomicInteger micrometerParsedWinesSuccessGauge = Metrics.gauge(PARSED_WINES_SUCCESS, new AtomicInteger(0)); private static final Gauge prometheusWinesParsedUnsuccessfulGauge = Gauge.build() .name(WINES_PARSED_UNSUCCESSFUL) .help("Number of unsuccessfully parsed wines") + .labelNames(REGION_LABEL) .register(); private static final AtomicInteger micrometerWinesParsedUnsuccessfulGauge = Metrics.gauge(WINES_PARSED_UNSUCCESSFUL, new AtomicInteger(0)); private static final Gauge prometheusIsBannedGauge = Gauge.build() .name(IS_BANNED) .help("Is banned host's IP") + .labelNames(REGION_LABEL) .register(); private static final AtomicInteger micrometerIsBannedGauge = Metrics.gauge(IS_BANNED, new AtomicInteger(0)); private static final Counter prometheusNumberOfWinesCreatedCounter = Counter.build() .name(NUMBER_OF_WINES_CREATED) .help("Number of created wines during 1 parsing") + .labelNames(REGION_LABEL) .register(); private static final Counter prometheusNumberOfWinesUpdatedCounter = Counter.build() .name(NUMBER_OF_WINES_UPDATED) .help("Number of updated wines during 1 parsing") + .labelNames(REGION_LABEL) .register(); private static final Counter prometheusNumberOfWinesDeletedCounter = Counter.build() .name(NUMBER_OF_WINES_DELETED) .help("Number of deleted wines during 1 parsing (due to irrelevance)") + .labelNames(REGION_LABEL) .register(); private static final Gauge prometheusPercentageOfUnsuccessfullyParsedWinesGauge = Gauge.build() .name(PERCENTAGE_OF_UNSUCCESSFULLY_PARSED_WINES) .help("Percent of unsuccessfully parsed wines") + .labelNames(REGION_LABEL) .register(); private static final AtomicInteger micrometerPercentageOfUnsuccessfullyParsedWinesGauge = Metrics.gauge(PERCENTAGE_OF_UNSUCCESSFULLY_PARSED_WINES, new AtomicInteger(0)); private static final Summary prometheusJobExecutionTimeSummary = Summary.build() .name(JOB_EXECUTION_TIME) .help("Job execution time") + .labelNames(REGION_LABEL) .register(); public AmServiceMetricsCollector() { this(SERVICE_NAME); - countParsingStart(0); - incParsingInProgress(0); - decParsingInProgress(0); - countParsingComplete("success", 0); - countParsingComplete("failed", 0); - countNumberOfWinesCreated(0); - countNumberOfWinesUpdated(0); - countNumberOfWinesDeleted(0); - countWinesPublishedToKafka(0); - percentageOfUnsuccessfullyParsedWines(0); - timeParsingDuration(0); - countTimeSinceLastParsing(0); - timeWineDetailsFetchingDuration(0); - timeWineDetailsParsingDuration(0); - timeWinePageParsingDuration(0); - timeWinePageFetchingDuration(0); - parsedWinesSuccess(0); - winesParsedUnsuccessful(0); - isBanned(0); - jobExecutionTime(0); + countParsingStart(0, "Санкт-Петербург"); + incParsingInProgress(0, "Санкт-Петербург"); + decParsingInProgress(0, "Санкт-Петербург"); + countParsingComplete("success", "Санкт-Петербург", 0); + countParsingComplete("failed", "Санкт-Петербург", 0); + countNumberOfWinesCreated(0, "Санкт-Петербург"); + countNumberOfWinesUpdated(0, "Санкт-Петербург"); + countNumberOfWinesDeleted(0, "Санкт-Петербург"); + countWinesPublishedToKafka(0, "Санкт-Петербург"); + percentageOfUnsuccessfullyParsedWines(0, "Санкт-Петербург"); + timeParsingDuration(0, "Санкт-Петербург"); + countTimeSinceLastParsing(0, "Санкт-Петербург"); + timeWineDetailsFetchingDuration(0, "Санкт-Петербург"); + timeWineDetailsParsingDuration(0, "Санкт-Петербург"); + timeWinePageParsingDuration(0, "Санкт-Петербург"); + timeWinePageFetchingDuration(0, "Санкт-Петербург"); + parsedWinesSuccess(0, "Санкт-Петербург"); + winesParsedUnsuccessful(0, "Санкт-Петербург"); + isBanned(0, "Санкт-Петербург"); + jobExecutionTime(0, "Санкт-Петербург"); } private AmServiceMetricsCollector(String serviceName) { super(serviceName); } - public void countParsingStart() { - Metrics.counter(PARSING_STARTED).increment(); - prometheusParsingStartedCounter.inc(); + public void countParsingStart(String region) { + Metrics.counter(PARSING_STARTED, REGION_LABEL, region).increment(); + prometheusParsingStartedCounter.labels(region).inc(); } - public void countParsingStart(double value) { - Metrics.counter(PARSING_STARTED).increment(value); - prometheusParsingStartedCounter.inc(value); + public void countParsingStart(double value, String region) { + Metrics.counter(PARSING_STARTED, REGION_LABEL, region).increment(value); + prometheusParsingStartedCounter.labels(region).inc(value); } - public void countParsingComplete(String status) { + public void countParsingComplete(String status, String region) { Metrics.counter(PARSING_COMPLETE, PARSING_COMPLETE_STATUS, status).increment(); - prometheusParsingCompleteCounter.labels(status).inc(); + prometheusParsingCompleteCounter.labels(region, status).inc(); } - public void countParsingComplete(String status, double value) { + public void countParsingComplete(String status, String region, double value) { Metrics.counter(PARSING_COMPLETE, PARSING_COMPLETE_STATUS, status).increment(value); - prometheusParsingCompleteCounter.labels(status).inc(value); + prometheusParsingCompleteCounter.labels(region, status).inc(value); } - public void incParsingInProgress() { - prometheusParsingInProgressGauge.inc(); + public void incParsingInProgress(String region) { + prometheusParsingInProgressGauge.labels(region).inc(); micrometerParsingInProgressGauge.getAndIncrement(); } - public void incParsingInProgress(double value) { - prometheusParsingInProgressGauge.inc(value); + public void incParsingInProgress(double value, String region) { + prometheusParsingInProgressGauge.labels(region).inc(value); micrometerParsingInProgressGauge.getAndAdd((int)value); } - public void decParsingInProgress() { - prometheusParsingInProgressGauge.dec(); + public void decParsingInProgress(String region) { + prometheusParsingInProgressGauge.labels(region).dec(); micrometerParsingInProgressGauge.getAndDecrement(); } - public void decParsingInProgress(double value) { - prometheusParsingInProgressGauge.dec(value); + public void decParsingInProgress(double value, String region) { + prometheusParsingInProgressGauge.labels(region).dec(value); micrometerParsingInProgressGauge.getAndAdd((int)value); } - public void timeParsingDuration(long nanoTime) { + public void timeParsingDuration(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusParsingDurationSummary.observe(secondsTime); - Metrics.summary(PARSING_DURATION).record(secondsTime); + prometheusParsingDurationSummary.labels(region).observe(secondsTime); + Metrics.summary(PARSING_DURATION, REGION_LABEL, region).record(secondsTime); } - public void countTimeSinceLastParsing(long nanoTime) { + public void countTimeSinceLastParsing(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusTimeSinceLastParsingGauge.set(secondsTime); + prometheusTimeSinceLastParsingGauge.labels(region).set(secondsTime); micrometerTimeSinceLastParsingGauge.set(secondsTime); } - public void timeWineDetailsFetchingDuration(long nanoTime) { + public void timeWineDetailsFetchingDuration(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusWineDetailsFetchingDurationSummary.observe(secondsTime); - Metrics.summary(WINE_DETAILS_FETCHING_DURATION).record(secondsTime); + prometheusWineDetailsFetchingDurationSummary.labels(region).observe(secondsTime); + Metrics.summary(WINE_DETAILS_FETCHING_DURATION, REGION_LABEL, region).record(secondsTime); } - public void timeWinePageFetchingDuration(long nanoTime) { + public void timeWinePageFetchingDuration(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusWinePageFetchingDurationSummary.observe(secondsTime); - Metrics.summary(WINE_PAGE_FETCHING_DURATION).record(secondsTime); + prometheusWinePageFetchingDurationSummary.labels(region).observe(secondsTime); + Metrics.summary(WINE_PAGE_FETCHING_DURATION, REGION_LABEL, region).record(secondsTime); } - public void timeWineDetailsParsingDuration(long nanoTime) { + public void timeWineDetailsParsingDuration(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusWineDetailsParsingDurationSummary.observe(secondsTime); - Metrics.summary(WINE_DETAILS_PARSING_DURATION).record(secondsTime); + prometheusWineDetailsParsingDurationSummary.labels(region).observe(secondsTime); + Metrics.summary(WINE_DETAILS_PARSING_DURATION, REGION_LABEL, region).record(secondsTime); } - public void timeWinePageParsingDuration(long nanoTime) { + public void timeWinePageParsingDuration(long nanoTime, String region) { long secondsTime = TimeUnit.NANOSECONDS.toSeconds(nanoTime); - prometheusWinePageParsingDurationSummary.observe(secondsTime); - Metrics.summary(WINE_PAGE_PARSING_DURATION).record(secondsTime); + prometheusWinePageParsingDurationSummary.labels(region).observe(secondsTime); + Metrics.summary(WINE_PAGE_PARSING_DURATION, REGION_LABEL, region).record(secondsTime); } - public void countWinesPublishedToKafka(double wineNum) { - Metrics.counter(WINES_PUBLISHED_TO_KAFKA).increment(wineNum); - prometheusWinesPublishedToKafkaCounter.inc(wineNum); + public void countWinesPublishedToKafka(double wineNum, String region) { + Metrics.counter(WINES_PUBLISHED_TO_KAFKA, REGION_LABEL, region).increment(wineNum); + prometheusWinesPublishedToKafkaCounter.labels(region).inc(wineNum); } - public void parsedWinesSuccess(int countParsedWinesSuccess) { - prometheusParsedWinesSuccessGauge.set(countParsedWinesSuccess); + public void parsedWinesSuccess(int countParsedWinesSuccess, String region) { + prometheusParsedWinesSuccessGauge.labels(region).set(countParsedWinesSuccess); micrometerParsedWinesSuccessGauge.set(countParsedWinesSuccess); } - public void winesParsedUnsuccessful(int countParsedWinesUnsuccessful) { - prometheusWinesParsedUnsuccessfulGauge.set(countParsedWinesUnsuccessful); + public void winesParsedUnsuccessful(int countParsedWinesUnsuccessful, String region) { + prometheusWinesParsedUnsuccessfulGauge.labels(region).set(countParsedWinesUnsuccessful); micrometerWinesParsedUnsuccessfulGauge.set(countParsedWinesUnsuccessful); } - public void isBanned(int bool) { - prometheusIsBannedGauge.set(bool); + public void isBanned(int bool, String region) { + prometheusIsBannedGauge.labels(region).set(bool); micrometerIsBannedGauge.set(bool); } - public void countNumberOfWinesCreated() { - Metrics.counter(NUMBER_OF_WINES_CREATED).increment(); - prometheusNumberOfWinesCreatedCounter.inc(); + public void countNumberOfWinesCreated(String region) { + Metrics.counter(NUMBER_OF_WINES_CREATED, REGION_LABEL, region).increment(); + prometheusNumberOfWinesCreatedCounter.labels(region).inc(); } - public void countNumberOfWinesCreated(double value) { - Metrics.counter(NUMBER_OF_WINES_CREATED).increment(value); - prometheusNumberOfWinesCreatedCounter.inc(value); + public void countNumberOfWinesCreated(double value, String region) { + Metrics.counter(NUMBER_OF_WINES_CREATED, REGION_LABEL, region).increment(value); + prometheusNumberOfWinesCreatedCounter.labels(region).inc(value); } - public void countNumberOfWinesUpdated() { - Metrics.counter(NUMBER_OF_WINES_UPDATED).increment(); - prometheusNumberOfWinesUpdatedCounter.inc(); + public void countNumberOfWinesUpdated(String region) { + Metrics.counter(NUMBER_OF_WINES_UPDATED, REGION_LABEL, region).increment(); + prometheusNumberOfWinesUpdatedCounter.labels(region).inc(); } - public void countNumberOfWinesUpdated(double value) { - Metrics.counter(NUMBER_OF_WINES_UPDATED).increment(value); - prometheusNumberOfWinesUpdatedCounter.inc(value); + public void countNumberOfWinesUpdated(double value, String region) { + Metrics.counter(NUMBER_OF_WINES_UPDATED, REGION_LABEL, region).increment(value); + prometheusNumberOfWinesUpdatedCounter.labels(region).inc(value); } - public void countNumberOfWinesDeleted() { - Metrics.counter(NUMBER_OF_WINES_DELETED).increment(); - prometheusNumberOfWinesDeletedCounter.inc(); + public void countNumberOfWinesDeleted(String region) { + Metrics.counter(NUMBER_OF_WINES_DELETED, REGION_LABEL, region).increment(); + prometheusNumberOfWinesDeletedCounter.labels(region).inc(); } - public void countNumberOfWinesDeleted(double value) { - Metrics.counter(NUMBER_OF_WINES_DELETED).increment(value); - prometheusNumberOfWinesDeletedCounter.inc(value); + public void countNumberOfWinesDeleted(double value, String region) { + Metrics.counter(NUMBER_OF_WINES_DELETED, REGION_LABEL, region).increment(value); + prometheusNumberOfWinesDeletedCounter.labels(region).inc(value); } - public void percentageOfUnsuccessfullyParsedWines(int percentUnsuccessfullyParsedWines){ - prometheusPercentageOfUnsuccessfullyParsedWinesGauge.set(percentUnsuccessfullyParsedWines); + public void percentageOfUnsuccessfullyParsedWines(int percentUnsuccessfullyParsedWines, String region){ + prometheusPercentageOfUnsuccessfullyParsedWinesGauge.labels(region).set(percentUnsuccessfullyParsedWines); micrometerPercentageOfUnsuccessfullyParsedWinesGauge.set(percentUnsuccessfullyParsedWines); } - public void jobExecutionTime(long nanoTime) { + public void jobExecutionTime(long nanoTime, String region) { long milliTime = TimeUnit.NANOSECONDS.toMillis(nanoTime); - prometheusJobExecutionTimeSummary.observe(milliTime); - Metrics.summary(JOB_EXECUTION_TIME).record(milliTime); + prometheusJobExecutionTimeSummary.labels(region).observe(milliTime); + Metrics.summary(JOB_EXECUTION_TIME, REGION_LABEL, region).record(milliTime); } } diff --git a/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java b/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java index 4f47150..7dd663a 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java +++ b/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java @@ -75,11 +75,15 @@ public UpdateRepositoryJob(AmServiceMetricsCollector metricsCollector, @Scheduled(cron = "${job.cron.update.repository}") @TrackExecutionTime(description = "ActualizeWineJob") public void runJob() { + metricsCollector.countParsingStart("Санкт-Петербург"); + metricsCollector.incParsingInProgress("Санкт-Петербург"); long jobStart = System.nanoTime(); updateService.updateDictionary(); updateService.updateWines(); long jobEnd = System.nanoTime(); - metricsCollector.jobExecutionTime(jobEnd - jobStart); + metricsCollector.jobExecutionTime(jobEnd - jobStart, "Санкт-Петербург"); + metricsCollector.countParsingComplete("SUCCESS", "Санкт-Петербург"); + metricsCollector.decParsingInProgress("Санкт-Петербург"); } @Scheduled(cron = "*/20 * * * * *") @@ -88,7 +92,7 @@ public void parsePages() { long fetchStart = System.nanoTime(); Document document = amClient.getPage(pageCursor.getAndIncrement()); long fetchEnd = System.nanoTime(); - metricsCollector.timeWinePageFetchingDuration(fetchEnd - fetchStart); + metricsCollector.timeWinePageFetchingDuration(fetchEnd - fetchStart, "Санкт-Петербург"); if(document != null) { final Long pageAmount = amService.getCatalogPagesAmount(document); if(pageCursor.longValue() >= pageAmount) { diff --git a/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmClientImpl.java b/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmClientImpl.java index 74c9030..4e39458 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmClientImpl.java +++ b/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmClientImpl.java @@ -59,7 +59,7 @@ public Document getPageByUrl(String url) { while (attempt < maxRetries) { Document document = fetchPage(url); if (document != null) { - metricsCollector.isBanned(0); + metricsCollector.isBanned(0, "Санкт-Петербург"); failedFetches = 0; return document; } @@ -68,7 +68,7 @@ public Document getPageByUrl(String url) { failedFetches++; if(failedFetches >= 5 || url.equals(catalogUrl)) { log.warn("The service appears to have been banned!"); - metricsCollector.isBanned(1); + metricsCollector.isBanned(1, "Санкт-Петербург"); } log.error("Cannot get document by '{}' url in {} attempts, there have been {} failed fetches in a row", url, attempt, failedFetches); return null; diff --git a/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmServiceImpl.java b/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmServiceImpl.java index bc4035e..3a6b376 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmServiceImpl.java +++ b/src/main/java/com/wine/to/up/am/parser/service/service/impl/AmServiceImpl.java @@ -130,32 +130,27 @@ public List getAmWines() { Long lastParse = null; while (page <= pages) { - metricsCollector.countParsingStart(); - metricsCollector.incParsingInProgress(); parseAttemptsCount++; long pageCopy = page; List newWines = getAmWines(page); - metricsCollector.parsedWinesSuccess(newWines.size()); - metricsCollector.winesParsedUnsuccessful(18 - newWines.size()); - metricsCollector.percentageOfUnsuccessfullyParsedWines(newWines.size() / 18); + metricsCollector.parsedWinesSuccess(newWines.size(), "Санкт-Петербург"); + metricsCollector.winesParsedUnsuccessful(18 - newWines.size(), "Санкт-Петербург"); + metricsCollector.percentageOfUnsuccessfullyParsedWines(newWines.size() / 18, "Санкт-Петербург"); page++; amWines.addAll(newWines); pagesProcessed[(int) pageCopy - 1] = true; if (!newWines.isEmpty()) { successfulParseCount++; - metricsCollector.countParsingComplete("SUCCESS"); eventLogger.info(AmServiceNotableEvents.I_WINES_PAGE_PARSED, pageCopy); pagesWithParsedWines[(int) pageCopy - 1] = true; long currentParse = System.nanoTime(); if(lastParse != null) { - metricsCollector.countTimeSinceLastParsing(currentParse - lastParse); + metricsCollector.countTimeSinceLastParsing(currentParse - lastParse, "Санкт-Петербург"); } lastParse = currentParse; } else { - metricsCollector.countParsingComplete("FAILED"); eventLogger.warn(AmServiceNotableEvents.W_WINE_PAGE_PARSING_FAILED, pageCopy, baseUrl + "?page=" + pageCopy); } - metricsCollector.decParsingInProgress(); } StringBuilder lostPagesLog = new StringBuilder("Unprocessed pages: "); @@ -202,7 +197,7 @@ public AdditionalProps getAdditionalProps(String link) { long fetchStart = System.nanoTime(); Document page = client.getPageByUrl(link); long fetchEnd = System.nanoTime(); - metricsCollector.timeWineDetailsFetchingDuration(fetchEnd - fetchStart); + metricsCollector.timeWineDetailsFetchingDuration(fetchEnd - fetchStart, "Санкт-Петербург"); if(page == null) { eventLogger.info(AmServiceNotableEvents.W_WINE_DETAILS_PARSING_FAILED, link); } @@ -226,7 +221,7 @@ private AdditionalProps parseAdditionalProps(Document page) { handleProp(prop, props); } long parseEnd = System.nanoTime(); - metricsCollector.timeWineDetailsParsingDuration(parseEnd - parseStart); + metricsCollector.timeWineDetailsParsingDuration(parseEnd - parseStart, "Санкт-Петербург"); return props; } @@ -264,7 +259,7 @@ private List getAmWines(Long page) { long fetchStart = System.nanoTime(); final Document document = client.getPage(page); long fetchEnd = System.nanoTime(); - metricsCollector.timeWinePageFetchingDuration(fetchEnd - fetchStart); + metricsCollector.timeWinePageFetchingDuration(fetchEnd - fetchStart, "Санкт-Петербург"); return getAmWines(document); } @@ -281,7 +276,7 @@ public List getAmWines(Document document) { }) : Collections.emptyList(); long parseEnd = System.nanoTime(); - metricsCollector.timeWinePageParsingDuration(parseEnd - parseStart); + metricsCollector.timeWinePageParsingDuration(parseEnd - parseStart, "Санкт-Петербург"); return res; } catch (JsonProcessingException e) { log.error("Cannot parse wines with error: {}", e.getMessage()); diff --git a/src/main/java/com/wine/to/up/am/parser/service/service/impl/RestServiceImpl.java b/src/main/java/com/wine/to/up/am/parser/service/service/impl/RestServiceImpl.java index 6c77eec..40bb05e 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/service/impl/RestServiceImpl.java +++ b/src/main/java/com/wine/to/up/am/parser/service/service/impl/RestServiceImpl.java @@ -47,7 +47,7 @@ public void updateWines() { long parseStart = System.nanoTime(); updateService.updateWines(); long parseEnd = System.nanoTime(); - metricsCollector.timeParsingDuration(parseEnd - parseStart); + metricsCollector.timeParsingDuration(parseEnd - parseStart, "Санкт-Петербург"); } @Override @@ -56,7 +56,7 @@ public void updateAll() { long parseStart = System.nanoTime(); updateService.updateWines(); long parseEnd = System.nanoTime(); - metricsCollector.timeParsingDuration(parseEnd - parseStart); + metricsCollector.timeParsingDuration(parseEnd - parseStart, "Санкт-Петербург"); } @Override diff --git a/src/main/java/com/wine/to/up/am/parser/service/service/impl/UpdateServiceImpl.java b/src/main/java/com/wine/to/up/am/parser/service/service/impl/UpdateServiceImpl.java index bd82023..8b2610e 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/service/impl/UpdateServiceImpl.java +++ b/src/main/java/com/wine/to/up/am/parser/service/service/impl/UpdateServiceImpl.java @@ -163,12 +163,12 @@ public void updateWines(List wines) { if (wine != null) { saveWine(wine, amWine, now); updated++; - metricsCollector.countNumberOfWinesUpdated(); + metricsCollector.countNumberOfWinesUpdated("Санкт-Петербург"); } else { wine = new Wine(); saveWine(wine, amWine, now); created++; - metricsCollector.countNumberOfWinesCreated(); + metricsCollector.countNumberOfWinesCreated("Санкт-Петербург"); } } List wineForDeleted = wineRepository.findAllByDateRecIsNot(now); @@ -176,7 +176,7 @@ public void updateWines(List wines) { w.setActual(false); wineRepository.save(w); markForDeleted++; - metricsCollector.countNumberOfWinesDeleted(); + metricsCollector.countNumberOfWinesDeleted("Санкт-Петербург"); } log.info("created {} Wines", created); log.info("updated {} Wines", updated); From bfa048a033aedba77f40ab52099b48551f27ba5f Mon Sep 17 00:00:00 2001 From: OlegAdmin Date: Thu, 8 Apr 2021 16:14:46 +0300 Subject: [PATCH 2/4] Parsing start and finish metrix fix --- .../up/am/parser/service/job/UpdateRepositoryJob.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java b/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java index 7dd663a..99fb2a7 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java +++ b/src/main/java/com/wine/to/up/am/parser/service/job/UpdateRepositoryJob.java @@ -75,20 +75,18 @@ public UpdateRepositoryJob(AmServiceMetricsCollector metricsCollector, @Scheduled(cron = "${job.cron.update.repository}") @TrackExecutionTime(description = "ActualizeWineJob") public void runJob() { - metricsCollector.countParsingStart("Санкт-Петербург"); - metricsCollector.incParsingInProgress("Санкт-Петербург"); long jobStart = System.nanoTime(); updateService.updateDictionary(); updateService.updateWines(); long jobEnd = System.nanoTime(); metricsCollector.jobExecutionTime(jobEnd - jobStart, "Санкт-Петербург"); - metricsCollector.countParsingComplete("SUCCESS", "Санкт-Петербург"); - metricsCollector.decParsingInProgress("Санкт-Петербург"); } @Scheduled(cron = "*/20 * * * * *") @TrackExecutionTime(description = "Actualize wines Job") public void parsePages() { + metricsCollector.countParsingStart("Санкт-Петербург"); + metricsCollector.incParsingInProgress("Санкт-Петербург"); long fetchStart = System.nanoTime(); Document document = amClient.getPage(pageCursor.getAndIncrement()); long fetchEnd = System.nanoTime(); @@ -100,7 +98,11 @@ public void parsePages() { } List wines = amService.getAmWines(document); updateService.updateWines(wines); + metricsCollector.countParsingComplete("SUCCESS", "Санкт-Петербург"); + } else { + metricsCollector.countParsingComplete("FAILURE", "Санкт-Петербург"); } + metricsCollector.decParsingInProgress("Санкт-Петербург"); } @Scheduled(cron = "*/20 * * * * *") From 4945bb155643b00bc0dd512849b6aa0e9794dc26 Mon Sep 17 00:00:00 2001 From: OlegAdmin Date: Thu, 8 Apr 2021 16:16:11 +0300 Subject: [PATCH 3/4] Metrics fix --- .../to/up/am/parser/service/controller/KafkaController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/wine/to/up/am/parser/service/controller/KafkaController.java b/src/main/java/com/wine/to/up/am/parser/service/controller/KafkaController.java index 79fe7cf..fb7e3c5 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/controller/KafkaController.java +++ b/src/main/java/com/wine/to/up/am/parser/service/controller/KafkaController.java @@ -46,7 +46,7 @@ public void sendAllWines() { for (WineDto wineDto : wineDtoList) { wines.add(ProtobufConverter.getProtobufWine(wineDto)); } - amServiceMetricsCollector.countWinesPublishedToKafka(wines.size()); + amServiceMetricsCollector.countWinesPublishedToKafka(wines.size(), "Санкт-Петербург"); List> chunks = chunkify(wines, CHUNK_SIZE); for(List chunk : chunks) { From ab614b6d1acc23d9e68f920ffc68a709e1bfcf7f Mon Sep 17 00:00:00 2001 From: OlegAdmin Date: Thu, 8 Apr 2021 16:21:06 +0300 Subject: [PATCH 4/4] Metrics fix --- .../java/com/wine/to/up/am/parser/service/job/KafkaJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/wine/to/up/am/parser/service/job/KafkaJob.java b/src/main/java/com/wine/to/up/am/parser/service/job/KafkaJob.java index 54f710a..5c6f293 100644 --- a/src/main/java/com/wine/to/up/am/parser/service/job/KafkaJob.java +++ b/src/main/java/com/wine/to/up/am/parser/service/job/KafkaJob.java @@ -74,7 +74,7 @@ public void sendWines() { .addWines(ProtobufConverter.getProtobufWine(wineDto)) .build(); kafkaMessageSender.sendMessage(message); - amServiceMetricsCollector.countWinesPublishedToKafka(1); + amServiceMetricsCollector.countWinesPublishedToKafka(1, "Санкт-Петербург"); } private WineDto wineToWineDto(Wine wine, Double oldPrice, Double price) {