From f0358940748d4f0ccc09e16a0bb7053f1e2b6cda Mon Sep 17 00:00:00 2001 From: gs-gunjan Date: Thu, 10 Aug 2023 12:15:26 +0530 Subject: [PATCH] adding a workflow to maintain information about external schedules --- .../artifacts/ArtifactsRefreshModule.java | 2 +- .../resources/SchedulesResource.java | 6 ++- .../schedules/services/SchedulesFactory.java | 48 +++++++++++++------ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/legend-depot-artifacts-refresh/src/main/java/org/finos/legend/depot/store/artifacts/ArtifactsRefreshModule.java b/legend-depot-artifacts-refresh/src/main/java/org/finos/legend/depot/store/artifacts/ArtifactsRefreshModule.java index 0d21817e5..3f1668a52 100644 --- a/legend-depot-artifacts-refresh/src/main/java/org/finos/legend/depot/store/artifacts/ArtifactsRefreshModule.java +++ b/legend-depot-artifacts-refresh/src/main/java/org/finos/legend/depot/store/artifacts/ArtifactsRefreshModule.java @@ -69,7 +69,7 @@ boolean registerMetrics(PrometheusMetricsHandler metricsHandler) @Named("refresh-all-versions") boolean initVersions(SchedulesFactory schedulesFactory, ArtifactsRefreshService artifactsRefreshService, ArtifactRepositoryProviderConfiguration configuration) { - schedulesFactory.registerSingleInstance(ParentEvent.REFRESH_ALL_VERSION_ARTIFACTS_SCHEDULE.name(), configuration.getVersionsUpdateIntervalInMillis(), configuration.getVersionsUpdateIntervalInMillis(),() -> artifactsRefreshService.refreshAllVersionsForAllProjects(false,false,false, ParentEvent.REFRESH_ALL_VERSION_ARTIFACTS_SCHEDULE.name())); + schedulesFactory.registerExternalTriggerSchedule(ParentEvent.REFRESH_ALL_VERSION_ARTIFACTS_SCHEDULE.name(), configuration.getVersionsUpdateIntervalInMillis(), true, () -> artifactsRefreshService.refreshAllVersionsForAllProjects(false,false,false, ParentEvent.REFRESH_ALL_VERSION_ARTIFACTS_SCHEDULE.name())); return true; } diff --git a/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/resources/SchedulesResource.java b/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/resources/SchedulesResource.java index 257cb74c3..a75c51d1d 100644 --- a/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/resources/SchedulesResource.java +++ b/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/resources/SchedulesResource.java @@ -17,6 +17,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.finos.legend.depot.core.authorisation.api.AuthorisationProvider; import org.finos.legend.depot.core.authorisation.resources.BaseAuthorisedResource; import org.finos.legend.depot.schedules.services.SchedulesFactory; @@ -97,12 +98,13 @@ public List getSchedulerInstances() @Path("/schedules/{scheduleName}") @ApiOperation(ResourceLoggingAndTracing.TRIGGER_SCHEDULE) @Produces(MediaType.APPLICATION_JSON) - public Response forceScheduler(@PathParam("scheduleName") String scheduleName) + public Response forceScheduler(@PathParam("scheduleName") String scheduleName, + @QueryParam("forceRun") @DefaultValue("false") @ApiParam("Whether to run the schedule if disabled") boolean forceRun) { return handle(ResourceLoggingAndTracing.TRIGGER_SCHEDULE, () -> { validateUser(); - schedulesFactory.trigger(scheduleName); + schedulesFactory.trigger(scheduleName, forceRun); return Response.noContent().build(); }); } diff --git a/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/services/SchedulesFactory.java b/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/services/SchedulesFactory.java index ce97acfe6..768802c8c 100644 --- a/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/services/SchedulesFactory.java +++ b/legend-depot-core-schedules/src/main/java/org/finos/legend/depot/schedules/services/SchedulesFactory.java @@ -72,6 +72,11 @@ public void run() } } + public void registerExternalTriggerSchedule(String name, long intervalInMilliseconds, boolean isSingleInstance, Supplier function) + { + createScheduleInfo(name, intervalInMilliseconds, isSingleInstance, function); + } + public void registerSingleInstance(String name, long delayStartInMilliseconds, long intervalInMilliseconds, Supplier function) { register(name, delayStartInMilliseconds, intervalInMilliseconds, true, function); @@ -84,17 +89,22 @@ public void register(String name, long delayStartInMilliseconds, long intervalIn private void register(String name, long delayStartInMilliseconds, long intervalInMilliseconds, boolean singleInstance, Supplier function) { - ScheduleInfo info = schedulesStore.get(name).orElse(new ScheduleInfo(name)); - info.frequency = intervalInMilliseconds; - info.singleInstance = singleInstance; - functions.put(name,function); - schedulesStore.createOrUpdate(info); + createScheduleInfo(name, intervalInMilliseconds, singleInstance, function); TimerTask timerTask = createTimerTask(name); tasksRegistry.put(name,timerTask); timer.scheduleAtFixedRate(timerTask, delayStartInMilliseconds, intervalInMilliseconds); } + private void createScheduleInfo(String name, long intervalInMilliseconds, boolean singleInstance, Supplier function) + { + ScheduleInfo info = schedulesStore.get(name).orElse(new ScheduleInfo(name)); + info.frequency = intervalInMilliseconds; + info.singleInstance = singleInstance; + functions.put(name, function); + schedulesStore.createOrUpdate(info); + } + private TimerTask createTimerTask(String name) { return new TimerTask() @@ -122,7 +132,7 @@ public void run() LOGGER.info("Skipping {} execution", name); return; } - execute(schedule); + execute(schedule, true); } }; } @@ -159,10 +169,10 @@ public void deRegisterAll() }); } - public void trigger(String scheduleName) + public void trigger(String scheduleName, boolean forceRun) { Optional scheduleInfo = schedulesStore.get(scheduleName); - scheduleInfo.ifPresent(schedule -> execute(schedule)); + scheduleInfo.ifPresent(schedule -> execute(schedule, forceRun)); } void run(String scheduleName) @@ -184,21 +194,29 @@ public void toggleDisableAll(boolean toggle) schedulesStore.getAll().forEach(info -> toggleDisable(info.name, toggle)); } - private void execute(ScheduleInfo schedule) + private void execute(ScheduleInfo schedule, boolean forceRun) { try { - if (functions.containsKey(schedule.name)) + if (forceRun || !schedule.isDisabled()) { - this.instancesStore.insert(new ScheduleInstance(schedule.name,toDate(LocalDateTime.now().plusSeconds(schedule.frequency / 1000L)))); - LOGGER.info("Starting schedule {} ", schedule.name); - Object result = functions.get(schedule.name).get(); - LOGGER.info("Schedule {} result {}", schedule.name, result); + if (functions.containsKey(schedule.name)) + { + this.instancesStore.insert(new ScheduleInstance(schedule.name,toDate(LocalDateTime.now().plusSeconds(schedule.frequency / 1000L)))); + LOGGER.info("Starting schedule {} ", schedule.name); + Object result = functions.get(schedule.name).get(); + LOGGER.info("Schedule {} result {}", schedule.name, result); + } + else + { + LOGGER.warn("No function to execute {}", schedule.name); + } } else { - LOGGER.warn("No function to execute {}", schedule.name); + LOGGER.warn("Schedule {} is disabled and force run flag is false", schedule.name); } + } catch (Exception e) {