Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Alphametics tests to handle timeouts for longer tests #1029

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Alphametics tests to handle timeouts for longer tests #1029

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 34 additions & 23 deletions exercises/practice/alphametics/AlphameticsTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,41 +6,45 @@ open Xunit
open Alphametics

[<Fact>]
let ``Puzzle with three letters`` () =
let ``Puzzle with three letters`` () = async {
let puzzle = "I + BB == ILL"
let expected =
[ ('I', 1);
('B', 9);
('L', 0) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Solution must have unique value for each letter`` () =
let ``Solution must have unique value for each letter`` () = async {
let puzzle = "A == B"
let expected = None
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Leading zero solution is invalid`` () =
let ``Leading zero solution is invalid`` () = async {
let puzzle = "ACA + DD == BD"
let expected = None
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with two digits final carry`` () =
let ``Puzzle with two digits final carry`` () = async {
let puzzle = "A + A + A + A + A + A + A + A + A + A + A + B == BCC"
let expected =
[ ('A', 9);
('B', 1);
('C', 0) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with four letters`` () =
let ``Puzzle with four letters`` () = async {
let puzzle = "AS + A == MOM"
let expected =
[ ('A', 9);
Expand All @@ -49,10 +53,11 @@ let ``Puzzle with four letters`` () =
('O', 0) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with six letters`` () =
let ``Puzzle with six letters`` () = async {
let puzzle = "NO + NO + TOO == LATE"
let expected =
[ ('N', 7);
Expand All @@ -63,10 +68,11 @@ let ``Puzzle with six letters`` () =
('E', 2) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with seven letters`` () =
let ``Puzzle with seven letters`` () = async {
let puzzle = "HE + SEES + THE == LIGHT"
let expected =
[ ('E', 4);
Expand All @@ -78,10 +84,11 @@ let ``Puzzle with seven letters`` () =
('T', 7) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with eight letters`` () =
[<Fact(Timeout=2000,Skip = "Remove this Skip property to run this test")>]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice idea. The actual timeouts are slightly tricky to determine, but I'm fine with having some value here. That said, we currently can't merge this as the test runner will strip the attribute before it runs the tests:

type EnableAllTests() =
    inherit SyntaxVisitor()

    override _.VisitSynAttribute(attr: SynAttribute) : SynAttribute =
        match attr.TypeName with
        | LongIdentWithDots ([ ident ], _) when ident.idText = "Fact" ->
            base.VisitSynAttribute
                { attr with
                      ArgExpr = SynExpr.Const(SynConst.Unit, attr.ArgExpr.Range) }
        | _ -> base.VisitSynAttribute(attr)

Would you be interested in fixing that too?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup will have a look at that shortly.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will look at this. Which repo and file is it?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think timeouts should be discussed at the practise exercise specification, hence my other PR to update canonical json with a timeout property. This might include relative timings between tests as implied here between 8 versus 10 letter tests. The actual implementation would depend on the track maintainers factoring in the built in online test runner timeouts and language performance etc. I would probably default to 10 seconds per test, a propos of anything more specific.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I already wrote a reply but it has inexplicably disappeared. What files do I need to look at?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See #1029 (comment). I should have posted that comment here.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@martinfreedman Let me know if you need any help/pointers. The F# syntax tree can be tricky to work with.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will do. And yes the Diamond exercise does not use [<Fact>]! I will add this into my solution when I have a chance (I have Covid right now). Any other exercises tests with non-standard Attributes?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(I have Covid right now)

Oh no! All the best.

Any other exercises tests with non-standard Attributes?

The simple-linked-list exercise uses a [<Theory>] attribute (which should actually be converted to multiple [<Fact>] attributes.

let ``Puzzle with eight letters`` () = async {
let puzzle = "SEND + MORE == MONEY"
let expected =
[ ('S', 9);
Expand All @@ -94,10 +101,11 @@ let ``Puzzle with eight letters`` () =
('Y', 2) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with ten letters`` () =
[<Fact(Timeout=5000,Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with ten letters`` () = async {
let puzzle = "AND + A + STRONG + OFFENSE + AS + A + GOOD == DEFENSE"
let expected =
[ ('A', 5);
Expand All @@ -112,10 +120,11 @@ let ``Puzzle with ten letters`` () =
('T', 9) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}

[<Fact(Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with ten letters and 199 addends`` () =
[<Fact(Timeout=5000,Skip = "Remove this Skip property to run this test")>]
let ``Puzzle with ten letters and 199 addends`` () = async {
let puzzle = "THIS + A + FIRE + THEREFORE + FOR + ALL + HISTORIES + I + TELL + A + TALE + THAT + FALSIFIES + ITS + TITLE + TIS + A + LIE + THE + TALE + OF + THE + LAST + FIRE + HORSES + LATE + AFTER + THE + FIRST + FATHERS + FORESEE + THE + HORRORS + THE + LAST + FREE + TROLL + TERRIFIES + THE + HORSES + OF + FIRE + THE + TROLL + RESTS + AT + THE + HOLE + OF + LOSSES + IT + IS + THERE + THAT + SHE + STORES + ROLES + OF + LEATHERS + AFTER + SHE + SATISFIES + HER + HATE + OFF + THOSE + FEARS + A + TASTE + RISES + AS + SHE + HEARS + THE + LEAST + FAR + HORSE + THOSE + FAST + HORSES + THAT + FIRST + HEAR + THE + TROLL + FLEE + OFF + TO + THE + FOREST + THE + HORSES + THAT + ALERTS + RAISE + THE + STARES + OF + THE + OTHERS + AS + THE + TROLL + ASSAILS + AT + THE + TOTAL + SHIFT + HER + TEETH + TEAR + HOOF + OFF + TORSO + AS + THE + LAST + HORSE + FORFEITS + ITS + LIFE + THE + FIRST + FATHERS + HEAR + OF + THE + HORRORS + THEIR + FEARS + THAT + THE + FIRES + FOR + THEIR + FEASTS + ARREST + AS + THE + FIRST + FATHERS + RESETTLE + THE + LAST + OF + THE + FIRE + HORSES + THE + LAST + TROLL + HARASSES + THE + FOREST + HEART + FREE + AT + LAST + OF + THE + LAST + TROLL + ALL + OFFER + THEIR + FIRE + HEAT + TO + THE + ASSISTERS + FAR + OFF + THE + TROLL + FASTS + ITS + LIFE + SHORTER + AS + STARS + RISE + THE + HORSES + REST + SAFE + AFTER + ALL + SHARE + HOT + FISH + AS + THEIR + AFFILIATES + TAILOR + A + ROOFS + FOR + THEIR + SAFE == FORTRESSES"
let expected =
[ ('A', 1);
Expand All @@ -130,5 +139,7 @@ let ``Puzzle with ten letters and 199 addends`` () =
('T', 9) ]
|> Map.ofList
|> Some
solve puzzle |> should equal expected
return solve puzzle |> should equal expected
}