Skip to content

Commit

Permalink
Make big-bang stepper tests more robust
Browse files Browse the repository at this point in the history 
In big-bang tests, if the tick frequency is high, some to-draw steps
(e.g. the one immediately following the launch of big-bang) can be
absent in stepper because no such call are made.

This commit adds the `(repetition ,lo ,hi ,steps) form to stepper tests to
account for skippable steps.
  • Loading branch information
@shhyou
shhyou committed Jan 1, 2025
1 parent d54d3c7 commit d2412ac
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 44 deletions.
1 change: 0 additions & 1 deletion htdp-test/info.rkt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
"redex-lib"
"racket-index"
"scheme-lib"
"srfi-lite-lib"
"compatibility-lib"
"gui-lib"
"racket-test"
Expand Down
48 changes: 33 additions & 15 deletions htdp-test/tests/stepper/test-abbrev.rkt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#lang racket

(require (for-syntax scheme/mpair))
(require (for-syntax scheme/mpair racket/list))

(provide t)

Expand Down Expand Up @@ -49,25 +49,43 @@
(define (process stx)
(split (map (lambda (s)
(if (and (identifier? s)
(memq (syntax-e s) '(:: -> error:)))
(memq (syntax-e s) '(:: ::* -> error:)))
(syntax-e s)
(process-hilites s)))
(syntax->list stx))))
(define (parse-:: rest)
(syntax-case rest (:: ::* -> error:)
[(error: (err)) (list #'(error err))]
[() (list #'(finished-stepping))]
[(x -> y) (list #'(before-after x y) #'(finished-stepping))]
[(x -> error: (err)) (list #'(before-error x err))]
[(x -> y :: . rest)
(cons #'(before-after x y) (parse-:: #'rest))]
[(x -> y ::*([lo hi] . snd) . rest)
(cons #'(before-after x y) (parse-::* #'lo #'hi #'(snd . rest)))]
[(x -> y -> . rest)
(cons #'(before-after x y) (parse-:: #'(y -> . rest)))]))
(define (parse-::* lo hi curr)
(define-values (prefix suffix)
(let rep-scan-loop ([prevs '()] [curr curr])
(syntax-case curr (:: ::*)
[() (values (reverse prevs) curr)]
[(:: . rest) (values (reverse prevs) curr)]
[(::* . rest) (values (reverse prevs) curr)]
[(fst . rest) (rep-scan-loop (cons #'fst prevs) #'rest)])))
(cons #`(repetition #,lo #,hi #,(drop-right (parse-:: prefix) 1))
;; ^ drop-right to remove the trailing (finished-stepping) in repetition steps
(syntax-case suffix (:: ::*)
[() (parse-:: suffix)]
[(:: rest ...) (parse-:: #'(rest ...))]
[(::* ([lo hi] snd ...) rest ...) (parse-::* #'lo #'hi #'((snd ...) rest ...))])))
(define (parse l)
(syntax-case l (::)
(syntax-case l (:: ::*)
[(fst :: rest ...)
(cons #'fst
(let loop ([rest #'(rest ...)])
(syntax-case rest (:: -> error:)
[(error: (err)) (list #'(error err))]
[() (list #'(finished-stepping))]
[(x -> y) (list #'(before-after x y) #'(finished-stepping))]
[(x -> error: (err)) (list #'(before-error x err))]
[(x -> y :: . rest)
(cons #'(before-after x y) (loop #'rest))]
[(x -> y -> . rest)
(cons #'(before-after x y) (loop #'(y -> . rest)))])))]))
(syntax-case stx (::)
(cons #'fst (parse-:: #'(rest ...)))]
[(fst ::* ([lo hi] snd ...) rest ...)
(cons #'fst (parse-::* #'lo #'hi #'((snd ...) rest ...)))]))
(syntax-case stx (:: ::*)
[(_ name ll-models . rest)
(with-syntax ([(exprs arg ...) (parse (process #'rest))])
(quasisyntax/loc stx
Expand Down
25 changes: 18 additions & 7 deletions htdp-test/tests/stepper/test-cases.rkt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,12 @@
;; (before-after exps exps)
;; (before-error exps str)
;; (error str)
;; (finished)
;; (repetition lo hi (non-empty-listof step)) steps are repeated for [lo,hi] times (inclusive)
;; (finished-stepping)
;;
;; The repetition form matches the steps greedily---it is not a regular expression form.
;; It will try to match as many steps as possible.
;;
;; an exps is a list of s-expressions with certain non-hygienic extensions:
;; - (hilite X) denotes the s-expression X, only highlighted
;; - any denotes any s-expression (matches everything)
Expand All @@ -50,6 +55,10 @@
;; expr1 ... :: expr2 ... -> expr3 ...)
;; means that `expr1 ...' is the original, the first step is
;; (before-after (expr2 ...) (expr3 ...))
;; to express repetitions, write
;; ::*[lo hi] expr2 ... -> expr3 ... -> expr4 ...
;; where the range of ::* extends until the next :: or the next ::*
;;
;; Cute stuff:
;; * use `::' to mark a new step that doesn't continue the previous one
;; e1 :: e2 -> e3 -> e4
Expand Down Expand Up @@ -1230,9 +1239,10 @@
[on-tick sub1 1/8])
;; The initial `draw` before the clock starts ticking
:: ,@defs {(empty-scene 50 50)} -> ,@defs {,img}
:: ... -> ,@defs {(empty-scene 50 50)} -> ,@defs {,img} ;; `draw` for w = 3
:: ... -> ,@defs {(empty-scene 50 50)} -> ,@defs {,img} ;; `draw` for w = 2
:: ... -> ,@defs {(empty-scene 50 50)} -> ,@defs {,img})) ;; `draw` for w = 1
::*[3 99] ... -> ,@defs {(empty-scene 50 50)} -> ,@defs {,img})) ;; `draw` for w=3..1
;; ^ ^~~ the additional repetition count is for potentially extra `draw` steps
;; | provided `draw` is called more than once in one tick
;; +~~ if the tick frequency is too high, reduce the lower bound.

;; Testing the current big-bang stepping behavior
;; functions defined in user code are stepped, but lambdas in big-bang are not
Expand All @@ -1248,12 +1258,13 @@
[to-draw (lambda (w)
(overlay (drawobj (first w))
(empty-scene 60 60)))]
[on-tick next 1/5])
[on-tick next 1/8])
:: ,@defs {(circle 25 "solid" "red")} -> ,@defs {,img}
::*[0 99] ... -> ,@defs {(circle 25 "solid" "red")} -> ,@defs {,img} ;; drawobj
:: ... -> ,@defs {(rest (list 25 18 11))} -> ,@defs {(list 18 11)} ;; next
:: ... -> ,@defs {(circle 18 "solid" "red")} -> ,@defs {,img} ;; drawobj
::*[1 99] ... -> ,@defs {(circle 18 "solid" "red")} -> ,@defs {,img} ;; drawobj
:: ... -> ,@defs {(rest (list 18 11))} -> ,@defs {(list 11)} ;; next
:: ... -> ,@defs {(circle 11 "solid" "red")} -> ,@defs {,img} ;; drawobj
::*[1 99] ... -> ,@defs {(circle 11 "solid" "red")} -> ,@defs {,img} ;; drawobj
:: ... -> ,@defs {(rest (list 11))} -> ,@defs {empty})) ;; next

;;;;;;;;;;;;
Expand Down
152 changes: 131 additions & 21 deletions htdp-test/tests/stepper/test-engine.rkt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
stepper/private/model
tests/utils/sexp-diff
lang/run-teaching-program
(only-in srfi/13 string-contains)
racket/string
racket/match
racket/contract
racket/file
Expand Down Expand Up @@ -110,9 +110,16 @@
;; - `(before-error ,before ,err-msg) where before is an sexp-with-hilite and err-msg is a string
;; - `(finished-stepping)
;; or
;; - `(repetition ,(>=/c 0) ,(>/c 0) ,(non-empty-listof step)) (possible skippable) repetitions
;; or
;; - `(ignore)
(define (step? sexp)
(match sexp
[`(repetition ,lo ,hi ,rep-expected-steps)
(and (exact-integer? lo) (>= lo 0)
(exact-integer? hi) (> hi 0)
(not (null? rep-expected-steps))
(andmap step? rep-expected-steps))]
[(list 'before-after before after) #t]
[(list 'error (? string? msg)) #t]
[(list 'before-error before (? string? msg)) #t]
Expand Down Expand Up @@ -336,39 +343,140 @@
(set-box! remaining (cdr (unbox remaining)))
next]))))

(struct sstk-data ([warns #:mutable] [hd #:mutable])
#:transparent)
(struct sstk-last sstk-data ()
#:transparent)
(struct sstk-cons sstk-data ([received-results #:mutable] rep-hi rep-steps tl)
#:transparent)

(define (steps-stack-last? stk)
(sstk-last? stk))

(define (make-steps-stack all-steps)
(sstk-last '() all-steps))

(define (steps-stack-push stk repetition)
(match-define `(repetition 0 ,hi ,rep-expected-steps)
repetition)
(sstk-cons '() rep-expected-steps '() hi rep-expected-steps stk))

(define (steps-stack-pop stk) (sstk-cons-tl stk))
(define (steps-stack-top-repetition-hi stk) (sstk-cons-rep-hi stk))
(define (steps-stack-top-repetition-steps stk) (sstk-cons-rep-steps stk))
(define (steps-stack-top stk) (sstk-data-hd stk))

(define (steps-stack-top-next! stk)
(set-sstk-data-hd! stk (cdr (sstk-data-hd stk))))
(define (steps-stack-top-prepend! stk new-steps)
(set-sstk-data-hd! stk (append new-steps (sstk-data-hd stk))))

(define (steps-stack-top-received-results stk)
(reverse (sstk-cons-received-results stk)))

(define (steps-stack-top-receive! stk result)
(when (sstk-cons? stk)
(set-sstk-cons-received-results! stk
(cons result (sstk-cons-received-results stk)))))

(define (steps-stack-top-warn stk error-box name fmt . args)
(set-sstk-data-warns! stk
(cons (list error-box name fmt args)
(sstk-data-warns stk))))

(define (steps-stack-top-commit-warns! stk)
(for ([warn-data (in-list (sstk-data-warns stk))])
(match-define (list error-box name fmt args)
warn-data)
(apply warn error-box name fmt args))
(set-sstk-data-warns! stk '()))

;; this is a front end for calling the stepper's "go"; the main
;; responsibility here is to fake the behavior of DrRacket and collect the
;; resulting steps.
(define (test-sequence/core render-settings expanded-thunk dynamic-requirer expected-steps error-box)
(define current-error-display-handler (error-display-handler))
(define all-steps expected-steps)
;; stacks of all-steps for backtracking
(define all-steps-stk (make-steps-stack expected-steps))
;; the values of certain parameters aren't surviving; create
;; lexical bindings for them:
(define current-show-all-steps (show-all-steps))
(define current-display-only-errors (display-only-errors))
(define test-finished-semaphore (make-semaphore))
(define receive-result
(λ (result)
(if (null? all-steps)
(if (null? (steps-stack-top all-steps-stk))
(begin (warn error-box
'test-sequence
"ran out of expected steps. Given result: ~v" result)
;; test is finished, release the semaphore:
(semaphore-post test-finished-semaphore))
(begin
(if (compare-steps result (car all-steps) error-box)
(match (car (steps-stack-top all-steps-stk))
;; non-skippable repetition => unroll it
[`(repetition ,lo ,hi ,rep-expected-steps)
#:when (> lo 0)
(steps-stack-top-next! all-steps-stk)
(define unrolled-expected-steps
(append rep-expected-steps
(if (> hi 0)
`((repetition ,(sub1 lo) ,(sub1 hi) ,rep-expected-steps))
'())))
(steps-stack-top-prepend! all-steps-stk unrolled-expected-steps)
(receive-result result)]
[`(repetition 0 ,hi ,rep-expected-steps)
;; lo = 0, the repetition form can match ε
(steps-stack-top-next! all-steps-stk)
(set! all-steps-stk
(steps-stack-push all-steps-stk `(repetition 0 ,hi ,rep-expected-steps)))
(receive-result result)]
[expected
(steps-stack-top-receive! all-steps-stk result)
(cond
[(compare-steps result expected error-box all-steps-stk)
(when (and current-show-all-steps (not current-display-only-errors))
(printf "test-sequence: steps match for expected result: ~v\n"
(car all-steps)))
(warn error-box
'test-sequence
"steps do not match\n given: ~v\nexpected: ~v"
(show-result result error-box)
(car all-steps)))
(set! all-steps (cdr all-steps))
(when (null? all-steps)
;; test is happy, release the semaphore:
(semaphore-post test-finished-semaphore))))))
expected))
(steps-stack-top-next! all-steps-stk)
(when (null? (steps-stack-top all-steps-stk))
(cond [(steps-stack-last? all-steps-stk)
(steps-stack-top-commit-warns! all-steps-stk)
;; test is happy, release the semaphore:
(semaphore-post test-finished-semaphore)]
[(> (steps-stack-top-repetition-hi all-steps-stk) 1)
(define rep-step
`(repetition
0
,(- (steps-stack-top-repetition-hi all-steps-stk) 1)
,(steps-stack-top-repetition-steps all-steps-stk)))
(steps-stack-top-commit-warns! all-steps-stk)
(set! all-steps-stk (steps-stack-pop all-steps-stk))
(steps-stack-top-prepend! all-steps-stk (list rep-step))]
[else
(steps-stack-top-commit-warns! all-steps-stk)
(set! all-steps-stk (steps-stack-pop all-steps-stk))]))]
[else
(cond
;; no more backtracking point, i.e. the failure is not caused by
;; a skippable (0-repeated-times) repetition form
[(steps-stack-last? all-steps-stk)
(steps-stack-top-commit-warns! all-steps-stk)
;; record the error but continue to compare new results
(warn error-box
'test-sequence
"steps do not match\n given: ~v\nexpected: ~v"
(show-result result error-box)
expected)
(steps-stack-top-next! all-steps-stk)
(when (null? (steps-stack-top all-steps-stk))
;; test is happy, release the semaphore:
(semaphore-post test-finished-semaphore))]
;; the current skippable repetition form failed.
;; start from the backtracking point with all received forms
[else
(define accumulated-results
(steps-stack-top-received-results all-steps-stk))
(set! all-steps-stk (steps-stack-pop all-steps-stk))
(for-each receive-result accumulated-results)])])]))))
(define iter-caller
(λ (init iter)
(init)
Expand Down Expand Up @@ -431,7 +539,7 @@


;; (-> step-result? sexp? boolean?)
(define (compare-steps actual expected error-box)
(define (compare-steps actual expected error-box all-steps-stk)
(match expected
[`(before-after ,before ,after)
(and (Before-After-Result? actual)
Expand All @@ -444,21 +552,23 @@
exps)
expected
name
error-box))
error-box
all-steps-stk))
(list (map sstx-s (Before-After-Result-pre-exps actual))
(map sstx-s (Before-After-Result-post-exps actual)))
(list before after)
(list 'before 'after)))]
[`(error ,err-msg)
(and (Error-Result? actual)
(string-contains (Error-Result-err-msg actual) err-msg))]
(string-contains? (Error-Result-err-msg actual) err-msg))]
[`(before-error ,before ,err-msg)
(and (Before-Error-Result? actual)
(and (noisy-equal? (map syntax->hilite-datum
(map sstx-s (Before-Error-Result-pre-exps actual)))
before
'before
error-box)
error-box
all-steps-stk)
(equal? err-msg (Before-Error-Result-err-msg actual))))]
[`(finished-stepping) (eq? 'finished-stepping actual)]
[`(ignore) (warn error-box
Expand Down Expand Up @@ -487,10 +597,10 @@

;; noisy-equal? : (any any . -> . boolean)
;; like equal?, but prints a noisy error message
(define (noisy-equal? actual expected name error-box)
(define (noisy-equal? actual expected name error-box all-steps-stk)
(if (equal? actual expected)
#t
(begin (warn error-box 'not-equal?
(begin (steps-stack-top-warn all-steps-stk error-box 'not-equal?
"~.s:\nactual: ~e =/= \nexpected: ~e\n here's the diff: ~e" name actual expected (sexp-diff actual expected))
#f)))

0 comments on commit d2412ac

Please sign in to comment.