From 14f1ba49d6ed170c98c4175a683e520ec426780b Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 19 Jul 2021 17:24:50 -0700 Subject: [PATCH 1/6] Allow MAPL to run in reverse with history alarms This commit adds code to allow the CapGridComp to run in reverse while maintaining the ability to trigger history alarms. This may also require a small bugfix in the ESMF 8.0 code. --- generic/MAPL_Generic.F90 | 6 +- gridcomps/Cap/MAPL_CapGridComp.F90 | 338 ++++++++++++++++++++- gridcomps/History/MAPL_HistoryGridComp.F90 | 212 +++++++++++-- 3 files changed, 526 insertions(+), 30 deletions(-) diff --git a/generic/MAPL_Generic.F90 b/generic/MAPL_Generic.F90 index ca613eeaf15b..8e8a70e70e57 100644 --- a/generic/MAPL_Generic.F90 +++ b/generic/MAPL_Generic.F90 @@ -173,9 +173,9 @@ module MAPL_GenericMod public MAPL_RequestService ! MAPL_Util - !public MAPL_GenericStateClockOn - !public MAPL_GenericStateClockOff - !public MAPL_GenericStateClockAdd + public MAPL_GenericStateClockOn + public MAPL_GenericStateClockOff + public MAPL_GenericStateClockAdd public MAPL_TimerOn public MAPL_TimerOff public MAPL_TimerAdd diff --git a/gridcomps/Cap/MAPL_CapGridComp.F90 b/gridcomps/Cap/MAPL_CapGridComp.F90 index 70dd9b1b82a1..80fffe26c6ee 100644 --- a/gridcomps/Cap/MAPL_CapGridComp.F90 +++ b/gridcomps/Cap/MAPL_CapGridComp.F90 @@ -73,6 +73,9 @@ module MAPL_CapGridCompMod procedure :: initialize_history procedure :: run procedure :: step +#ifdef ADJOINT + procedure :: step_reverse +#endif procedure :: finalize procedure :: get_model_duration procedure :: get_am_i_root @@ -203,6 +206,7 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) character(len=ESMF_MAXSTR ) :: DYCORE character(len=ESMF_MAXPATHLEN) :: user_dirpath,tempString logical :: tend,foundPath + integer :: reverseTime logical :: cap_clock_is_present @@ -534,6 +538,16 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) call ESMF_ConfigGetAttribute(cap%cf_root, value=ReplayMode, Label="REPLAY_MODE:", default="NoReplay", rc=status) _VERIFY(STATUS) + ! pass REVERSE_TIME resource to history and root config + call MAPL_GetResource(MAPLOBJ, reverseTime, "REVERSE_TIME:", default = 0, rc = status) + _VERIFY(status) + + call MAPL_ConfigSetAttribute(cap%cf_root, value=reverseTime, Label="REVERSE_TIME:", rc=status) + _VERIFY(STATUS) + + call MAPL_ConfigSetAttribute(cap%cf_hist, value=reverseTime, Label="REVERSE_TIME:", rc=status) + _VERIFY(STATUS) + ! Register the children with MAPL !-------------------------------- @@ -1100,7 +1114,8 @@ subroutine run_MAPL_GridComp(gc, phase, rc) type (ESMF_Gridcomp) :: gc integer, optional, intent(in) :: phase integer, optional, intent(out) :: rc - + + integer :: reverse_time integer :: n, status, phase_ logical :: done @@ -1108,6 +1123,18 @@ subroutine run_MAPL_GridComp(gc, phase, rc) type (MAPL_MetaComp), pointer :: MAPLOBJ procedure(), pointer :: root_set_services + + ! instantiate Alarm lists + ! type(ESMF_Alarm) :: alarm(200), hist_alarm(400) + + ! local variables for Get methods + integer :: ringingAlarmCount ! at any time step (0 to NUMALARMS) + integer :: alarmCount, hist_alarmCount + + ! name, loop counter, result code + character (len=ESMF_MAXSTR) :: name + integer :: i, result + cap => get_CapGridComp_from_gc(gc) call MAPL_GetObjectFromGC(gc, maplobj, rc=status) _VERIFY(status) @@ -1117,6 +1144,73 @@ subroutine run_MAPL_GridComp(gc, phase, rc) if (.not. cap%printspec > 0) then + ! Check if user wants to reverse time + call MAPL_Set(MAPLOBJ, name = cap%name, cf = cap%config, rc = status) + _VERIFY(status) + call MAPL_GetResource(MAPLOBJ, reverse_time, label='REVERSE_TIME:', & + default=0, rc = status) + _VERIFY(STATUS) + + if ( reverse_time == 1 ) then + ! call ESMF_ClockGetAlarmList(cap%clock, ESMF_ALARMLIST_ALL, & + ! alarmList=alarm, alarmCount=alarmCount, rc = status ) + ! _VERIFY(STATUS) + + ! call ESMF_ClockGetAlarmList(cap%clock_hist, ESMF_ALARMLIST_ALL, & + ! alarmList=hist_alarm, alarmCount=hist_alarmCount, rc = status ) + ! _VERIFY(STATUS) + + if (MAPL_Am_I_Root()) THEN + + ! WRITE(*,1003) 'clock', alarmCount + + ! WRITE(*,1003) 'clock_hist', hist_alarmCount + + WRITE(*,1001) cap%nsteps + endif +1001 FORMAT(' MAPL_CapGC running for ', i3, ' timesteps') +1003 FORMAT(3x, a10, ' has ', i3, ' alarms') + + FAKE_TIME_LOOP: do n = 1, cap%nsteps + + if (MAPL_Am_I_Root()) & + WRITE(*,1002) n, cap%nsteps +1002 FORMAT(' MAPL_CapGC running step ', i3, ' of ', i3) + + if (.not.cap%lperp) then + done = ESMF_ClockIsDone(cap%clock_hist, rc = status) + _VERIFY(status) + if (done .and. MAPL_Am_I_Root()) & + WRITE(*,*) ' MAPL_CapGC: No perpetual clock and history is done. Exiting loop' + if (done) exit + endif + ! Advance the Clock before running History and Record + ! --------------------------------------------------- + call ESMF_ClockAdvance(cap%clock, rc = status) + _VERIFY(STATUS) + call ESMF_ClockAdvance(cap%clock_hist, rc = status) + _VERIFY(STATUS) + + ! Update Perpetual Clock + ! ---------------------- + + if (cap%lperp) then + call Perpetual_Clock(cap, status) + _VERIFY(status) + end if + enddo FAKE_TIME_LOOP + + if (MAPL_Am_I_Root()) & + WRITE(*,*) ' MAPL_CapGC finished time loop, reversing clocks' + ! Reverse the direction of the clocks + call ESMF_ClockSet( cap%clock, direction=ESMF_DIRECTION_REVERSE, & + advanceCount=0, rc=status ) + call ESMF_ClockSet( cap%clock_hist, direction=ESMF_DIRECTION_REVERSE, & + advanceCount=0, rc=status ) + cap%nsteps = cap%nsteps + 1 + endif ! reverse_time == 1 + + ! Time Loop starts by checking for Segment Ending Time !----------------------------------------------------- if (cap%compute_throughput) then @@ -1136,12 +1230,29 @@ subroutine run_MAPL_GridComp(gc, phase, rc) _VERIFY(status) if (.not.cap%lperp) then - done = ESMF_ClockIsStopTime(cap%clock_hist, rc = status) + if ( reverse_time == 0 ) then + done = ESMF_ClockIsStopTime(cap%clock_hist, rc = status) + else + done = ESMF_ClockIsDone(cap%clock_hist, rc = status) + endif _VERIFY(status) + if (MAPL_Am_I_Root() .and. done) THEN + call ESMF_ClockPrint(cap%clock_hist, options='currTime string', rc = status) + _VERIFY(status) + call ESMF_ClockPrint(cap%clock_hist, options='stopTime string', rc = status) + _VERIFY(status) + call ESMF_ClockPrint(cap%clock_hist, options='direction', rc = status) + _VERIFY(status) + endif if (done) exit endif - call cap%step(phase=phase_, rc=status) + if ( .not. reverse_time ) then + call cap%step(phase=phase_, rc=status) + else + call cap%step_reverse(phase=phase_, n .eq. 1, rc=status) + endif + _VERIFY(status) ! Reset loop average timer to get a better @@ -1557,7 +1668,224 @@ subroutine rewind_clock(this, time, rc) _RETURN(_SUCCESS) end subroutine rewind_clock +#ifdef ADJOINT + subroutine step_reverse(this, unusable, phase, first, rc) + class(MAPL_CapGridComp), intent(inout) :: this + class(KeywordEnforcer), optional, intent(in ) :: unusable + integer, optional, intent(it) :: phase + logical, intent(in) :: first + integer, intent(out) :: rc + + integer :: status, phase_ + + + type(ESMF_Time) :: currTime + + _UNUSED_DUMMY(unusable) + phase_ = 1 + if (present(phase)) phase_ = phase + + call ESMF_GridCompGet(this%gc, vm = this%vm) + ! Run the ExtData Component + ! -------------------------- + if (phase_ == 1) then + call first_phase(first, rc=status) + _VERIFY(status) + + endif ! phase_ == 1 + + + ! Run the Gridded Component + ! -------------------------- + call ESMF_GridCompRun(this%gcs(this%root_id), importstate = this%child_imports(this%root_id), & + exportstate = this%child_exports(this%root_id), & + clock = this%clock, userrc = status) + _VERIFY(status) + + ! Synchronize for Next TimeStep + ! ----------------------------- + + call ESMF_VMBarrier(this%vm, rc = status) + _VERIFY(STATUS) + + ! Call History Run for Output + ! --------------------------- + + call ESMF_GridCompRun(this%gcs(this%history_id), importstate=this%child_imports(this%history_id), & + exportstate = this%child_exports(this%history_id), & + clock = this%clock_hist, userrc = status) + _VERIFY(status) + + ! Advance the Clock and run History and Record + ! --------------------------------------------------- + if (phase_ == this%n_run_phases) then + + call last_phase(rc=status) + _VERIFY(STATUS) + + endif !phase_ == last + + _RETURN(ESMF_SUCCESS) + + contains + + subroutine first_phase(first, rc) + logical, intent(in) :: first + integer, optional, intent(out) :: rc + + integer :: AGCM_YY, AGCM_MM, AGCM_DD, AGCM_H, AGCM_M, AGCM_S + + if (this%compute_throughput) then + if (.not.this%started_loop_timer) then + this%starts%loop_start_timer = MPI_WTime(status) + this%started_loop_timer=.true. + end if + this%starts%start_timer = MPI_Wtime(status) + end if + + call ESMF_GridCompRun(this%gcs(this%extdata_id), importState = this%child_imports(this%extdata_id), & + exportState = this%child_exports(this%extdata_id), & + clock = this%clock, userrc = status) + _VERIFY(status) + + ! Call Record for intermediate checkpoint (if desired) + ! Note that we are not doing a Record for History. + ! ------------------------------------------------------ + ! don't output reverse checkpoints + ! call ESMF_GridCompWriteRestart(this%gcs(this%root_id), importstate = this%child_imports(this%root_id), & + ! exportstate = this%child_exports(this%root_id), & + ! clock = this%clock_hist, userrc = status) + ! _VERIFY(status) + + ! Advance the Clock before running History and Record + ! --------------------------------------------------- + if (.not. first) THEN + call ESMF_ClockAdvance(this%clock, rc = status) + _VERIFY(STATUS) + call ESMF_ClockAdvance(this%clock_hist, rc = status) + _VERIFY(STATUS) + + ! Update Perpetual Clock + ! ---------------------- + + if (this%lperp) then + call Perpetual_Clock(this, status) + _VERIFY(status) + end if + end if + + if (this%compute_throughput) then + call ESMF_VMBarrier(this%vm,rc=status) + _VERIFY(status) + this%starts%start_run_timer = MPI_WTime(status) + end if + + + call ESMF_ClockGet(this%clock, CurrTime = currTime, rc = status) + _VERIFY(status) + call ESMF_TimeGet(CurrTime, YY = AGCM_YY, & + MM = AGCM_MM, & + DD = AGCM_DD, & + H = AGCM_H , & + M = AGCM_M , & + S = AGCM_S, rc=status) + _VERIFY(status) + if (this%AmIRoot) write(6,1000) AGCM_YY,AGCM_MM,AGCM_DD,AGCM_H,AGCM_M,AGCM_S +1000 format(1x,'AGCM Date: ',i4.4,'/',i2.2,'/',i2.2,2x,'Time: ',i2.2,':',i2.2,':',i2.2) + + + _RETURN(_SUCCESS) + + end subroutine + + subroutine last_phase(rc) + integer, optional, intent(out) :: rc + integer :: status + + if (this%compute_throughput) then + call ESMF_VMBarrier(this%vm,rc=status) + _VERIFY(status) + end_run_timer = MPI_WTime(status) + end if + + call ESMF_GridCompRun(this%gcs(this%history_id), importstate=this%child_imports(this%history_id), & + exportstate = this%child_exports(this%history_id), & + clock = this%clock_hist, userrc = status) + _VERIFY(status) + ! Estimate throughput times + ! --------------------------- + if (this%compute_throughput) then + call print_throughput(rc=status) + _VERIFY(STATUS) + end if + + _RETURN(_SUCCESS) + + end subroutine + + subroutine print_throughput(rc) + integer, optional, intent(out) :: rc + integer :: status, n + + real(kind=REAL64) :: TIME_REMAINING + real(kind=REAL64) :: LOOP_THROUGHPUT + real(kind=REAL64) :: INST_THROUGHPUT + real(kind=REAL64) :: RUN_THROUGHPUT + real :: mem_total, mem_commit, mem_committed_percent + real :: mem_used, mem_used_percent + type(ESMF_Time) :: currTime + type(ESMF_TimeInterval) :: delt + integer :: AGCM_YY, AGCM_MM, AGCM_DD, AGCM_H, AGCM_M, AGCM_S + integer :: HRS_R, MIN_R, SEC_R + + + call ESMF_ClockGet(this%clock, CurrTime = currTime, rc = status) + _VERIFY(status) + call ESMF_TimeGet(CurrTime, YY = AGCM_YY, & + MM = AGCM_MM, & + DD = AGCM_DD, & + H = AGCM_H , & + M = AGCM_M , & + S = AGCM_S, rc=status) + _VERIFY(status) + delt=currTime-this%cap_restart_time + ! Call system clock to estimate throughput simulated Days/Day + call ESMF_VMBarrier( this%vm, RC=STATUS ) + _VERIFY(STATUS) + END_TIMER = MPI_Wtime(status) + n=this%get_step_counter() + !GridCompRun Timer [Inst] + RUN_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_RUN_TIMER-this%starts%start_run_timer) + ! Time loop throughput [Inst] + INST_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-this%starts%start_timer) + ! Time loop throughput [Avg] + LOOP_THROUGHPUT = REAL(n*this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-this%starts%loop_start_timer) + ! Estimate time remaining (seconds) + TIME_REMAINING = REAL((this%nsteps-n)*this%HEARTBEAT_DT,kind=REAL64)/LOOP_THROUGHPUT + HRS_R = FLOOR(TIME_REMAINING/3600.0) + MIN_R = FLOOR(TIME_REMAINING/60.0 - 60.0*HRS_R) + SEC_R = FLOOR(TIME_REMAINING - 3600.0*HRS_R - 60.0*MIN_R) + ! Reset Inst timer + this%starts%start_timer = END_TIMER + ! Get percent of used memory + call MAPL_MemUsed ( mem_total, mem_used, mem_used_percent, RC=STATUS ) + _VERIFY(STATUS) + ! Get percent of committed memory + call MAPL_MemCommited ( mem_total, mem_commit, mem_committed_percent, RC=STATUS ) + _VERIFY(STATUS) + + if( mapl_am_I_Root(this%vm) ) write(6,1000) + LOOP_THROUGHPUT,INST_THROUGHPUT,RUN_THROUGHPUT,HRS_R,MIN_R,SEC_R,& + mem_committed_percent,mem_used_percent + 1000 format(2x,'Throughput(days/day)[Avg Tot Run]: ',f8.1,1x,f8.1,1x,f8.1,2x,'TimeRemaining(Est) ',i3.3,':'i2.2,':',i2.2,2x, & + f5.1,'% : ',f5.1,'% Mem Comm:Used') + + _RETURN(_SUCCESS) + + end subroutine + end subroutine step_reverse +#endif ! !IROUTINE: MAPL_ClockInit -- Sets the clock ! !INTERFACE: @@ -1741,7 +2069,7 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) _ASSERT(NUM_DT>=0, 'NUM_DT should be >= 0.') _ASSERT(DEN_DT> 0, 'DEN_DT should be > 0.') _ASSERT(NUM_DT=0, 'HEARTBEAT_DT should be >= 0.') + !_ASSERT(HEARTBEAT_DT>=0, 'HEARTBEAT_DT should be >= 0.') ! initialize calendar to be Gregorian type ! ---------------------------------------- @@ -1841,6 +2169,8 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) rc = STATUS ) _VERIFY(STATUS) + if (endTime < startTime) duration = duration * -1 + stopTime = currTime + duration ! initialize model time step diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 867204b157ae..b9187f5227f9 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -276,6 +276,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_State), pointer :: export (:) => null() type(ESMF_State), pointer :: exptmp (:) type(ESMF_Time) :: StartTime + type(ESMF_Time) :: EndTime type(ESMF_Time) :: CurrTime type(ESMF_Time) :: RingTime type(ESMF_Time) :: RefTime @@ -283,6 +284,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_Time) :: nextMonth type(ESMF_TimeInterval) :: oneMonth, dur type(ESMF_TimeInterval) :: Frequency + type(ESMF_TimeInterval) :: OneSecond type(ESMF_Array) :: array type(ESMF_Field) :: field type(ESMF_Field) :: f @@ -424,6 +426,14 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_Field), allocatable :: fldList(:) character(len=ESMF_MAXSTR), allocatable :: regexList(:) +! debug variables + type(ESMF_Time) :: debugTime + type(ESMF_TimeInterval) :: timeStep + character(len=ESMF_MAXSTR) :: TimeString + logical :: ringing + integer :: alarmCount +! adjoint variable + integer :: reverseTime ! Begin !------ @@ -463,6 +473,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGet ( clock, calendar=cal, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_ClockGet ( clock, currTime=CurrTime, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_ClockGet ( clock, StartTime=StartTime,rc=STATUS ) ; _VERIFY(STATUS) + call ESMF_ClockGet ( clock, stopTime=EndTime, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_TimeGet ( StartTime, TimeString=string ,rc=STATUS ) ; _VERIFY(STATUS) read(string( 1: 4),'(i4.4)') year @@ -555,6 +566,13 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute(config, value=intstate%version, & label='VERSION:', default=0, rc=status) _VERIFY(STATUS) + + ! Are we running the adjoint? + call ESMF_ConfigGetAttribute(config, reverseTime, & + Label="REVERSE_TIME:" , & + Default=0, RC=STATUS) + _VERIFY(STATUS) + if( MAPL_AM_I_ROOT() ) then print * print *, 'EXPSRC:',trim(INTSTATE%expsrc) @@ -565,6 +583,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'MarkDone: ' , INTSTATE%MarkDone print *, 'PrePost: ' , INTSTATE%PrePost print * + if (reverseTime .eq. 1) THEN + print *, 'REVERSE_TIME = "', reverseTime, '"' + endif endif ! Determine Number of Output Streams @@ -798,9 +819,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_date, default=nymdc, & label=trim(string) // 'ref_date:',rc=status ) _VERIFY(STATUS) - _ASSERT(is_valid_date(list(n)%ref_date),'Invalid ref_date') call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_time, default=000000, & label=trim(string) // 'ref_time:',rc=status ) + _VERIFY(STATUS) _ASSERT(is_valid_time(list(n)%ref_time),'Invalid ref_time') @@ -871,7 +892,6 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%ref_time < 0 .OR. & list(n)%duration < 0 ) list(n)%disabled = .true. - old_fields_style = .true. ! unless if (intstate%version >= 2) then call ESMF_ConfigGetAttribute ( cfg, value=field_set_name, label=trim(string)//'field_set:', & @@ -1184,6 +1204,15 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) if (list(n)%disabled) cycle + ! if(list(n)%mode == "instantaneous" .or. list(n)%ForceOffsetZero) then + ! sec = 0 + ! else + ! IntState%average(n) = .true. + ! sec = MAPL_nsecf(list(n)%acc_interval) / 2 + ! endif + ! call ESMF_TimeIntervalSet( INTSTATE%STAMPOFFSET(n), S=sec, rc=status ) + ! _VERIFY(STATUS) + ! His and Seg Alarms based on Reference Date and Time ! --------------------------------------------------- REF_TIME(1) = list(n)%ref_date/10000 @@ -1225,19 +1254,86 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! Added Logic to eliminate BEG_DATE = cap_restart date problem ! ------------------------------------------------------------ if (RefTime == startTime) then - RingTime = RefTime + Frequency + if (list(n)%backwards) then + RingTime = RefTime + else + RingTime = RefTime + Frequency + endif end if ! if (RingTime < currTime .and. sec /= 0 ) then - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + if (list(n)%backwards) then + RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency + else + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + endif + endif + if(.true. .and. MAPL_AM_I_ROOT() ) then + write(6,*) "Setting history alarm for species ", n + + call ESMF_TimeGet ( RingTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute + write(6,'(1X,"RingTime: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " backwards:",L1)') & + year, month, day, hour, minute, list(n)%backwards + + call ESMF_ClockGet(clock, currtime=debugtime, timeStep=timestep,rc=status) ; _VERIFY(STATUS) + call ESMF_TimeIntervalGet ( timestep, h=hour,m=minute, rc=status ) ; _VERIFY(STATUS) + + if (hour < 0 .or. minute < 0) then + write(6,'("Clock Timestep = -", i2.2, ":", i2.2)') hour, abs(minute) + else + write(6,'("Clock Timestep = ", i2.2, ":", i2.2)') hour, abs(minute) + end if + call ESMF_TimeGet ( DebugTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute + write(6,'(1X,"Current Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " backwards:",L1)') & + year, month, day, hour, minute, list(n)%backwards endif if ( list(n)%backwards ) then - list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) + list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) else list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) endif _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + + ! I don't understand what's going on here. For some reason when I create the alarm when the clock is running + ! with a negative timestep, it immediately rings, even though it's an hour in the past... + ! if (ESMF_AlarmIsRinging(list(n)%his_alarm)) THEN + ! call ESMF_AlarmSet(list(n)%his_alarm, RingTime=RingTime, ringing=.false., rc=status); _VERIFY(STATUS) + ! endif + if(.false. .and. MAPL_AM_I_ROOT() ) then + call ESMF_AlarmGet(list(n)%his_alarm, RingTime=DebugTime, ringing=ringing, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( DebugTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " ringing: ", L1)') & + n, year, month, day, hour, minute, ringing + endif + + + !ALT if monthly overwrite duration and frequency + if (list(n)%monthly) then + list(n)%duration = 1 !ALT simply non-zero + end if if( list(n)%duration.ne.0 ) then if (.not.list(n)%monthly) then sec = MAPL_nsecf( list(n)%duration ) @@ -1248,16 +1344,17 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! and for debugging print call WRITE_PARALLEL("DEBUG: monthly averaging is active for collection "//trim(list(n)%collection)) end if + ! RingTime = RefTime + IntState%StampOffset(n) RingTime = RefTime if (RingTime < currTime) then RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency endif - if ( list(n)%backwards ) then - list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) - else - list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) - endif + list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) + _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount if (list(n)%monthly .and. (currTime == RingTime)) then call ESMF_AlarmRingerOn( list(n)%his_alarm,rc=status ) _VERIFY(STATUS) @@ -1268,6 +1365,10 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, enabled=.false., & ringTime=currTime, name='historyNewSegment', rc=status ) _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount endif ! Mon Alarm based on 1st of Month 00Z @@ -1291,11 +1392,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) do while ( RingTime < currTime ) RingTime = RingTime + Frequency enddo - if ( list(n)%backwards ) then - list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) - else - list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) - endif + list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) if(list(n)%monthly) then !ALT this is temporary workaround. It has a memory leak @@ -1305,6 +1402,10 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%his_alarm = list(n)%mon_alarm intState%stampOffset(n) = Frequency ! we go to the beginning of the month end if + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount ! End Alarm based on end_date and end_time ! ---------------------------------------- @@ -1323,19 +1424,27 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) M = REF_TIME(5), & S = REF_TIME(6), calendar=cal, rc=rc ) - if ( list(n)%backwards ) then - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, rc=status ) - else - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, sticky=.false., rc=status ) - endif + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) else - if ( list(n)%backwards ) then - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=CurrTime, rc=status ) + if (reverseTime .eq. 1) then + if (MAPL_Am_I_Root()) & + WRITE(*,*) ' Setting end_alarm to disabled!' + call ESMF_TimeIntervalSet( OneSecond, S = 1, rc=rc ) + ringTime = startTime - oneSecond + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., enabled=.false., rc=status ) else - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=CurrTime, sticky=.false., rc=status ) + call ESMF_TimeIntervalSet( OneSecond, S = 1, rc=rc ) + ringTime = endTime + oneSecond + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., rc=status ) endif + + _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + call ESMF_AlarmRingerOff(list(n)%end_alarm, rc=status ) _VERIFY(STATUS) endif @@ -2199,6 +2308,12 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) else + if (.false. .and. MAPL_AM_I_ROOT()) THEN + WRITE(*,*) 'REFRESH = ', REFRESH, ' AVGINT = ', AVGINT + WRITE(*,*) 'acc_int = ', MAPL_nsecf(list(n)%acc_interval), & + ' freq = ', MAPL_nsecf(list(n)%frequency) + endif + call MAPL_VarSpecCreateInList(INTSTATE%SRCS(n)%SPEC, & SHORT_NAME = SHORT_NAME, & LONG_NAME = LONG_NAME, & @@ -3300,6 +3415,15 @@ subroutine Run ( gc, import, export, clock, rc ) ! ErrLog vars integer :: status +! Debug variables + type(ESMF_Time) :: CurrTime + character(len=ESMF_MAXSTR) :: TimeString + integer :: year,month,day,hour,minute + logical :: alarmEnabled + type(ESMF_Time) :: RingTime + character(len=ESMF_MAXSTR) :: tmpstring + + !============================================================================= ! Begin... @@ -3336,7 +3460,45 @@ subroutine Run ( gc, import, export, clock, rc ) allocate(Ignore (nlist), stat=status) _VERIFY(STATUS) Ignore = .false. + + if(.false. .and. MAPL_AM_I_ROOT() ) then + write(6,*) "Checking history time" + call ESMF_ClockGet ( clock, currTime=CurrTime ,rc=STATUS ) ; _VERIFY(STATUS) + + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"CurrTime: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " FWD:",L1)') & + year, month, day, hour, minute, FWD + do n=1,nlist + call ESMF_AlarmGet(list(n)%his_alarm, ringTime=CurrTime, ringing=alarmEnabled, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " ringing: ", L1)') & + n, year, month, day, hour, minute, alarmEnabled + + call ESMF_AlarmGet(list(n)%his_alarm, prevRingTime=CurrTime, enabled=alarmEnabled, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Prev Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " enabled: ", L1)') & + n, year, month, day, hour, minute, alarmEnabled + end do + endif ! decide if clock direction and collections' backwards mode agree do n=1,nlist @@ -3417,6 +3579,10 @@ subroutine Run ( gc, import, export, clock, rc ) if (list(n)%disabled .or. ESMF_AlarmIsRinging(list(n)%end_alarm) ) then list(n)%disabled = .true. Writing(n) = .false. + if (MAPL_am_I_Root()) THEN + WRITE(*, 1999) n + ENDIF +1999 FORMAT('Not Writing alarm ', i3, ' because end_alarm is ringing') else if (list(n)%timeseries_output) then Writing(n) = .true. else From ced717e1342bc72a2fe5e0b8311e648a2690ccd2 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 27 Jul 2021 06:29:29 -0700 Subject: [PATCH 2/6] Removed ifdefs for reverse_step subroutine GMAO doesn't like preprocessor defines so I removed most of them but had forgotten to do the step_reverse function. This change fixes that. --- gridcomps/Cap/MAPL_CapGridComp.F90 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gridcomps/Cap/MAPL_CapGridComp.F90 b/gridcomps/Cap/MAPL_CapGridComp.F90 index 80fffe26c6ee..a7e3721903d9 100644 --- a/gridcomps/Cap/MAPL_CapGridComp.F90 +++ b/gridcomps/Cap/MAPL_CapGridComp.F90 @@ -73,9 +73,7 @@ module MAPL_CapGridCompMod procedure :: initialize_history procedure :: run procedure :: step -#ifdef ADJOINT procedure :: step_reverse -#endif procedure :: finalize procedure :: get_model_duration procedure :: get_am_i_root @@ -1668,7 +1666,6 @@ subroutine rewind_clock(this, time, rc) _RETURN(_SUCCESS) end subroutine rewind_clock -#ifdef ADJOINT subroutine step_reverse(this, unusable, phase, first, rc) class(MAPL_CapGridComp), intent(inout) :: this class(KeywordEnforcer), optional, intent(in ) :: unusable @@ -1885,7 +1882,7 @@ subroutine print_throughput(rc) end subroutine end subroutine step_reverse -#endif + ! !IROUTINE: MAPL_ClockInit -- Sets the clock ! !INTERFACE: From f5f351d5f2e61ec222969d1b00524e78b38bef2e Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 26 Oct 2021 17:43:24 -0700 Subject: [PATCH 3/6] Updated changelog to reflect reverse run changes --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 891bd2a27ff4..34414f83be7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- Fixed issues with alarms when clocks are run in reverse. - Fixed bug with MAPL_FindChild gfortran debug compilation ### Added - +- Added option to run in reverse + ### Changed - Removed last `NETCDF_LIBRARIES` reference from CMake From 689b6868cc4b7d770ebd3d503ea0161219a2e790 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 26 Oct 2021 17:50:07 -0700 Subject: [PATCH 4/6] Removed dead code I had made some changes in previous tests to get the reverse model running but forgot to remove these two. --- gridcomps/Cap/MAPL_CapGridComp.F90 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gridcomps/Cap/MAPL_CapGridComp.F90 b/gridcomps/Cap/MAPL_CapGridComp.F90 index a7e3721903d9..fd04c66da24e 100644 --- a/gridcomps/Cap/MAPL_CapGridComp.F90 +++ b/gridcomps/Cap/MAPL_CapGridComp.F90 @@ -2066,7 +2066,7 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) _ASSERT(NUM_DT>=0, 'NUM_DT should be >= 0.') _ASSERT(DEN_DT> 0, 'DEN_DT should be > 0.') _ASSERT(NUM_DT=0, 'HEARTBEAT_DT should be >= 0.') + _ASSERT(HEARTBEAT_DT>=0, 'HEARTBEAT_DT should be >= 0.') ! initialize calendar to be Gregorian type ! ---------------------------------------- @@ -2166,8 +2166,6 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) rc = STATUS ) _VERIFY(STATUS) - if (endTime < startTime) duration = duration * -1 - stopTime = currTime + duration ! initialize model time step From c0133fe9a7ddbaa1ada1f1a918d2bef8f142ee44 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 8 Nov 2022 13:54:19 -0500 Subject: [PATCH 5/6] Fix changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90ceb2520566..5422be7b67d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed + - Fixed issues with alarms when clocks are run in reverse. ### Added + - Added option to run in reverse - + ### Changed ### Removed From 08acef4e39d0e8a73d1921e7fffee018202a6650 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 8 Nov 2022 14:03:44 -0500 Subject: [PATCH 6/6] Add missing code from merge --- gridcomps/History/MAPL_HistoryGridComp.F90 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 8a5430d88479..0f9d40481848 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -2362,6 +2362,13 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) if (allocated(ungridded_coord)) deallocate(ungridded_coord) else + + if (.false. .and. MAPL_AM_I_ROOT()) THEN + WRITE(*,*) 'REFRESH = ', REFRESH, ' AVGINT = ', AVGINT + WRITE(*,*) 'acc_int = ', MAPL_nsecf(list(n)%acc_interval), & + ' freq = ', MAPL_nsecf(list(n)%frequency) + + endif call MAPL_VarSpecCreateInList(INTSTATE%SRCS(n)%SPEC, & SHORT_NAME = SHORT_NAME, &