From f21ca22982aa7e1233caa0368e7ea65eff5a8e36 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 16 Dec 2024 12:55:41 -0700 Subject: [PATCH 01/14] Allow quietOnSuccess and maxRejected fscheck config --- Expecto.FsCheck/FsCheck.fs | 4 ++-- Expecto.FsCheck3/FsCheck3.fs | 3 ++- Expecto/Model.fs | 10 ++++++++-- README.md | 10 +++++++--- RELEASE_NOTES.md | 3 +++ 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 3758cd20..823babdb 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -89,14 +89,14 @@ module ExpectoFsCheck = let test (config: FsCheckConfig) = let config = { MaxTest = config.maxTest - MaxFail = 1000 + MaxFail = config.maxRejected // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the // values given in the test output, which are only ints when running FsCheck 2 Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma)) Name = name StartSize = config.startSize EndSize = config.endSize - QuietOnSuccess = true + QuietOnSuccess = config.quietOnSuccess Every = fun _ _ -> String.Empty EveryShrink = fun _ -> String.Empty Arbitrary = config.arbitrary diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 55c8e4c4..1cf844b0 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -95,9 +95,10 @@ module ExpectoFsCheck = .WithName(name) .WithStartSize(config.startSize) .WithEndSize(config.endSize) - .WithQuietOnSuccess(true) + .WithQuietOnSuccess(config.quietOnSuccess) .WithArbitrary(config.arbitrary) .WithRunner(runner config) + .WithMaxRejected(config.maxRejected) Check.One(config, property) diff --git a/Expecto/Model.fs b/Expecto/Model.fs index aeb3c7dd..8f1fc809 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -9,8 +9,8 @@ type SourceLocation = static member empty = { sourcePath = ""; lineNumber = 0 } type FsCheckConfig = - /// The maximum number of tests that are run. - { maxTest: int + { /// The maximum number of tests that are run. + maxTest: int /// The size to use for the first test. startSize: int /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. @@ -36,6 +36,10 @@ type FsCheckConfig = finishedTest: FsCheckConfig -> (* test name *) string -> Async + /// If set, suppresses the output from the test if the test is successful. + quietOnSuccess: bool + /// The maximum number of tests where values are rejected, e.g. as the result of ==> + maxRejected: int } static member defaultConfig = @@ -47,6 +51,8 @@ type FsCheckConfig = receivedArgs = fun _ _ _ _ -> async.Return () successfulShrink = fun _ _ _ -> async.Return () finishedTest = fun _ _ -> async.Return () + quietOnSuccess = true + maxRejected = 1000 } /// Actual test function; either an async one, or a synchronous one. diff --git a/README.md b/README.md index f1b33efd..b8ac8053 100644 --- a/README.md +++ b/README.md @@ -718,14 +718,14 @@ like the following. ```fsharp type FsCheckConfig = - /// The maximum number of tests that are run. - { maxTest: int + { /// The maximum number of tests that are run. + maxTest: int /// The size to use for the first test. startSize: int /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. endSize: int /// If set, the seed to use to start testing. Allows reproduction of previous runs. - replay: (uint64 * uint64) option + replay: (uint64 * uint64 * int) option /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be /// overridden) @@ -745,6 +745,10 @@ type FsCheckConfig = finishedTest: FsCheckConfig -> (* test name *) string -> Async + /// If set, suppresses the output from the test if the test is successful. + quietOnSuccess: bool + /// The maximum number of tests where values are rejected, e.g. as the result of ==> + maxRejected: int } ``` diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 61192c29..60156d41 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +### 11.0.0-alpha4 - 2024-12-16 +* Allow quietOnSuccess and maxRejected fscheck config + ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder From 8d765df6e1f264376eea4a430bcbed255e835623 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 16 Dec 2024 15:14:17 -0700 Subject: [PATCH 02/14] Allow tests to pass with a message --- Expecto.Tests.CSharp/Tests.cs | 5 +++++ Expecto/CSharp.fs | 2 ++ Expecto/Expecto.Impl.fs | 27 +++++++++++++++++++++++++++ Expecto/Model.fs | 1 + Expecto/TestResults.fs | 7 +++++++ 5 files changed, 42 insertions(+) diff --git a/Expecto.Tests.CSharp/Tests.cs b/Expecto.Tests.CSharp/Tests.cs index a34f1150..3dff8386 100644 --- a/Expecto.Tests.CSharp/Tests.cs +++ b/Expecto.Tests.CSharp/Tests.cs @@ -30,6 +30,11 @@ Task ITestPrinter.Failed(string value1, string value2, TimeSpan value3) return Task.CompletedTask; } + public Task PassedWithMessage(string value1, string value2, TimeSpan value3) + { + return Task.CompletedTask; + } + Task ITestPrinter.Ignored(string value1, string value2) { return Task.CompletedTask; diff --git a/Expecto/CSharp.fs b/Expecto/CSharp.fs index b238f627..2c7dd7f8 100644 --- a/Expecto/CSharp.fs +++ b/Expecto/CSharp.fs @@ -16,6 +16,7 @@ type ITestPrinter = abstract member BeforeEach : string -> Task abstract member Info : string -> Task abstract member Passed : string * TimeSpan -> Task + abstract member PassedWithMessage : string * string * TimeSpan -> Task abstract member Ignored : string * string -> Task abstract member Failed : string * string * TimeSpan -> Task abstract member Exn : string * exn * TimeSpan -> Task @@ -30,6 +31,7 @@ module Runner = { beforeRun = fun t -> async { return! i.BeforeRun(t) |> Async.AwaitTask } beforeEach = fun s -> async { return! i.BeforeEach(s) |> Async.AwaitTask } passed = fun n d -> async { return! i.Passed(n, d) |> Async.AwaitTask } + passedWithMessage = fun n m d -> async { return! i.PassedWithMessage(n, m, d) |> Async.AwaitTask } info = fun s -> async { return! i.Info(s) |> Async.AwaitTask } ignored = fun n m -> async { return! i.Ignored(n, m) |> Async.AwaitTask } failed = fun n m d -> async { return! i.Failed(n, m, d) |> Async.AwaitTask } diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 7987a07d..54a5d4e8 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -37,18 +37,21 @@ module Impl = type TestResult = | Passed + | PassedWithMessage of string | Ignored of string | Failed of string | Error of exn override x.ToString() = match x with | Passed -> "Passed" + | PassedWithMessage m -> "Passed: " + m | Ignored reason -> "Ignored: " + reason | Failed error -> "Failed: " + error | Error e -> "Exception: " + exnWithInnerMsg e "" member x.tag = match x with | Passed -> 0 + | PassedWithMessage _ -> 0 | Ignored _ -> 1 | Failed _ -> 2 | Error _ -> 3 @@ -57,11 +60,13 @@ module Impl = match x with | Ignored _ -> 0 | Passed -> 1 + | PassedWithMessage _ -> 1 | Failed _ -> 2 | Error _ -> 3 member x.isPassed = match x with | Passed -> true + | PassedWithMessage _ -> true | _ -> false member x.isIgnored = match x with @@ -213,6 +218,8 @@ module Impl = info: string -> Async /// test name -> time taken -> unit passed: string -> TimeSpan -> Async + /// test name -> message -> time taken -> unit + passedWithMessage: string -> string -> TimeSpan -> Async /// test name -> ignore message -> unit ignored: string -> string -> Async /// test name -> other message -> time taken -> unit @@ -226,6 +233,7 @@ module Impl = let name = config.joinWith.format test.name match result.result with | Passed -> config.printer.passed name result.duration + | PassedWithMessage message -> config.printer.passedWithMessage name message result.duration | Failed message -> config.printer.failed name message result.duration | Ignored message -> config.printer.ignored name message | Error e -> config.printer.exn name e result.duration @@ -235,6 +243,7 @@ module Impl = beforeEach = fun _ -> async.Zero() info = fun _ -> async.Zero() passed = fun _ _ -> async.Zero() + passedWithMessage = fun _ _ _ -> async.Zero() ignored = fun _ _ -> async.Zero() failed = fun _ _ _ -> async.Zero() exn = fun _ _ _ -> async.Zero() @@ -258,6 +267,13 @@ module Impl = >> setField "testName" n >> setField "duration" d) + passedWithMessage = fun n m d -> + logger.logWithAck Debug ( + eventX "{testName} passed in {duration}. {message}" + >> setField "testName" n + >> setField "message" m + >> setField "duration" d) + ignored = fun n m -> logger.logWithAck Debug ( eventX "{testName} was ignored. {reason}" @@ -417,6 +433,13 @@ module Impl = "name", formatName n "duration", d.TotalMilliseconds |> int |> string ] } + passedWithMessage = fun n m d -> async { + do! innerPrinter.passedWithMessage n m d + tcLog "testFinished" [ + "flowId", formatName n + "name", formatName n + "duration", d.TotalMilliseconds |> int |> string ] } + info = fun s -> innerPrinter.info s @@ -463,6 +486,7 @@ module Impl = beforeEach = fun n -> runTwoAsyncs (first.beforeEach n) (second.beforeEach n) info = fun s -> runTwoAsyncs (first.info s) (second.info s) passed = fun n d -> runTwoAsyncs (first.passed n d) (second.passed n d) + passedWithMessage = fun n m d -> runTwoAsyncs (first.passedWithMessage n m d) (second.passedWithMessage n m d) ignored = fun n m -> runTwoAsyncs (first.ignored n m) (second.ignored n m) failed = fun n m d -> runTwoAsyncs (first.failed n m d) (second.failed n m d) exn = fun n e d -> runTwoAsyncs (first.exn n e d) (second.exn n e d) @@ -595,6 +619,9 @@ module Impl = w.Stop() return TestSummary.single Passed (float w.ElapsedMilliseconds) with + | :? PassWithMessage as e -> + w.Stop() + return TestSummary.single (PassedWithMessage ("\n"+e.Message)) (float w.ElapsedMilliseconds) | :? AssertException as e -> w.Stop() let msg = diff --git a/Expecto/Model.fs b/Expecto/Model.fs index 8f1fc809..dadf9244 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -100,6 +100,7 @@ type Test = | Sequenced of SequenceMethod * Test type ExpectoException(msg) = inherit Exception(msg) +type PassWithMessage(msg) = inherit ExpectoException(msg) type AssertException(msg) = inherit ExpectoException(msg) type FailedException(msg) = inherit ExpectoException(msg) type IgnoreException(msg) = inherit ExpectoException(msg) diff --git a/Expecto/TestResults.fs b/Expecto/TestResults.fs index 3b435d3c..a92e816f 100644 --- a/Expecto/TestResults.fs +++ b/Expecto/TestResults.fs @@ -40,6 +40,7 @@ let writeNUnitSummary file (summary: TestRunSummary) = |> addAttribute "executed" match test.result with + | PassedWithMessage _ | Passed -> "Success" | Error _ | Failed _ -> "Failure" @@ -47,6 +48,7 @@ let writeNUnitSummary file (summary: TestRunSummary) = |> addAttribute "result" match test.result with + | PassedWithMessage _ | Passed -> addAttribute "success" "True" | Error _ | Failed _ -> addAttribute "success" "False" @@ -59,11 +61,15 @@ let writeNUnitSummary file (summary: TestRunSummary) = // TODO: implement it. addAttribute "asserts" "0" + let passNode = XElement(XName.Get "pass") let failureNode = XElement(XName.Get "failure") // Some more details that explain why a test was not executed. match test.result with | Passed -> () + | PassedWithMessage m -> + passNode.Add(XName.Get "message", XCData m) + element.Add passNode | Error e -> failureNode.Add(XName.Get "message", XCData e.Message) failureNode.Add(XName.Get "stack-trace", XCData e.StackTrace) @@ -136,6 +142,7 @@ let writeJUnitSummary file (summary: Impl.TestRunSummary) = XAttribute(XName.Get "message", message)) match test.result with | Passed -> [||] + | PassedWithMessage msg -> [|makeMessageNode "pass" msg|] | Error e -> let message = makeMessageNode "error" e.Message message.Add(XCData(e.ToString())) From ae8a68e0902ac8667006fbb1dfe790472b291dcb Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 16 Dec 2024 15:24:24 -0700 Subject: [PATCH 03/14] Docs for `passedWithMessage` --- Expecto/Expecto.Impl.fs | 2 ++ README.md | 1 + RELEASE_NOTES.md | 2 ++ 3 files changed, 5 insertions(+) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 54a5d4e8..12858a32 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -219,6 +219,8 @@ module Impl = /// test name -> time taken -> unit passed: string -> TimeSpan -> Async /// test name -> message -> time taken -> unit + ///
+ /// Same as `passed`, but the test provided a message to be displayed. passedWithMessage: string -> string -> TimeSpan -> Async /// test name -> ignore message -> unit ignored: string -> string -> Async diff --git a/README.md b/README.md index b8ac8053..0e759aca 100644 --- a/README.md +++ b/README.md @@ -1378,3 +1378,4 @@ This might be due to how terminals/the locking thereof work: try running your te ### 11.0.0 - Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`. - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2) +- `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers. It should be mostly the same as `passed`, but also print the given message. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 60156d41..8bf3a3f8 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,7 @@ ### 11.0.0-alpha4 - 2024-12-16 * Allow quietOnSuccess and maxRejected fscheck config +* Tests can now pass with a message + * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder From 40cbf989cde80da86ad3db0cdd65bd4cbbd96458 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 16 Dec 2024 15:47:03 -0700 Subject: [PATCH 04/14] Show number of tests and stamps on fscheck test pass --- Expecto.FsCheck/FsCheck.fs | 6 +++++- RELEASE_NOTES.md | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 823babdb..5a7d02c7 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -42,7 +42,11 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.True (_testData,_b) -> () + | TestResult.True (data,suppressOutput) -> + if not suppressOutput then + $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}" + |> PassWithMessage + |> raise | TestResult.False (_,_,_, Outcome.Exception (:? IgnoreException as e),_) -> raise e diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8bf3a3f8..5361f807 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,8 @@ ### 11.0.0-alpha4 - 2024-12-16 -* Allow quietOnSuccess and maxRejected fscheck config * Tests can now pass with a message - * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers + * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers +* Allow quietOnSuccess and maxRejected fscheck config + * When `quietOnSuccess` is false, the number of tests and stamps (`Prop.collect/classify/trivial` etc) will be shown when the test passes. ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder From 0f22d2db6996af6c26da53b2e40f36298520beb2 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 6 Jan 2025 12:27:23 -0700 Subject: [PATCH 05/14] Revert "Show number of tests and stamps on fscheck test pass" This reverts commit 40cbf989cde80da86ad3db0cdd65bd4cbbd96458. --- Expecto.FsCheck/FsCheck.fs | 6 +----- RELEASE_NOTES.md | 5 ++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 5a7d02c7..823babdb 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -42,11 +42,7 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.True (data,suppressOutput) -> - if not suppressOutput then - $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}" - |> PassWithMessage - |> raise + | TestResult.True (_testData,_b) -> () | TestResult.False (_,_,_, Outcome.Exception (:? IgnoreException as e),_) -> raise e diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5361f807..8bf3a3f8 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,7 @@ ### 11.0.0-alpha4 - 2024-12-16 -* Tests can now pass with a message - * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers * Allow quietOnSuccess and maxRejected fscheck config - * When `quietOnSuccess` is false, the number of tests and stamps (`Prop.collect/classify/trivial` etc) will be shown when the test passes. +* Tests can now pass with a message + * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder From 54b9b9e970f9d9684fd096a7afb55b4be588e401 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 6 Jan 2025 12:27:24 -0700 Subject: [PATCH 06/14] Revert "Docs for `passedWithMessage`" This reverts commit ae8a68e0902ac8667006fbb1dfe790472b291dcb. --- Expecto/Expecto.Impl.fs | 2 -- README.md | 1 - RELEASE_NOTES.md | 2 -- 3 files changed, 5 deletions(-) diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 12858a32..54a5d4e8 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -219,8 +219,6 @@ module Impl = /// test name -> time taken -> unit passed: string -> TimeSpan -> Async /// test name -> message -> time taken -> unit - ///
- /// Same as `passed`, but the test provided a message to be displayed. passedWithMessage: string -> string -> TimeSpan -> Async /// test name -> ignore message -> unit ignored: string -> string -> Async diff --git a/README.md b/README.md index 0e759aca..b8ac8053 100644 --- a/README.md +++ b/README.md @@ -1378,4 +1378,3 @@ This might be due to how terminals/the locking thereof work: try running your te ### 11.0.0 - Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`. - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2) -- `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers. It should be mostly the same as `passed`, but also print the given message. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8bf3a3f8..60156d41 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,5 @@ ### 11.0.0-alpha4 - 2024-12-16 * Allow quietOnSuccess and maxRejected fscheck config -* Tests can now pass with a message - * `TestPrinters` now has a new field `passedWithMessage` which may need to be implemented for custom printers ### 11.0.0-alpha3 - 2024-10-13 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder From ea95b62b70eb2acdfd3af4c127e4d59d47b410ec Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 6 Jan 2025 12:27:24 -0700 Subject: [PATCH 07/14] Revert "Allow tests to pass with a message" This reverts commit 8d765df6e1f264376eea4a430bcbed255e835623. --- Expecto.Tests.CSharp/Tests.cs | 5 ----- Expecto/CSharp.fs | 2 -- Expecto/Expecto.Impl.fs | 27 --------------------------- Expecto/Model.fs | 1 - Expecto/TestResults.fs | 7 ------- 5 files changed, 42 deletions(-) diff --git a/Expecto.Tests.CSharp/Tests.cs b/Expecto.Tests.CSharp/Tests.cs index 3dff8386..a34f1150 100644 --- a/Expecto.Tests.CSharp/Tests.cs +++ b/Expecto.Tests.CSharp/Tests.cs @@ -30,11 +30,6 @@ Task ITestPrinter.Failed(string value1, string value2, TimeSpan value3) return Task.CompletedTask; } - public Task PassedWithMessage(string value1, string value2, TimeSpan value3) - { - return Task.CompletedTask; - } - Task ITestPrinter.Ignored(string value1, string value2) { return Task.CompletedTask; diff --git a/Expecto/CSharp.fs b/Expecto/CSharp.fs index 2c7dd7f8..b238f627 100644 --- a/Expecto/CSharp.fs +++ b/Expecto/CSharp.fs @@ -16,7 +16,6 @@ type ITestPrinter = abstract member BeforeEach : string -> Task abstract member Info : string -> Task abstract member Passed : string * TimeSpan -> Task - abstract member PassedWithMessage : string * string * TimeSpan -> Task abstract member Ignored : string * string -> Task abstract member Failed : string * string * TimeSpan -> Task abstract member Exn : string * exn * TimeSpan -> Task @@ -31,7 +30,6 @@ module Runner = { beforeRun = fun t -> async { return! i.BeforeRun(t) |> Async.AwaitTask } beforeEach = fun s -> async { return! i.BeforeEach(s) |> Async.AwaitTask } passed = fun n d -> async { return! i.Passed(n, d) |> Async.AwaitTask } - passedWithMessage = fun n m d -> async { return! i.PassedWithMessage(n, m, d) |> Async.AwaitTask } info = fun s -> async { return! i.Info(s) |> Async.AwaitTask } ignored = fun n m -> async { return! i.Ignored(n, m) |> Async.AwaitTask } failed = fun n m d -> async { return! i.Failed(n, m, d) |> Async.AwaitTask } diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 54a5d4e8..7987a07d 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -37,21 +37,18 @@ module Impl = type TestResult = | Passed - | PassedWithMessage of string | Ignored of string | Failed of string | Error of exn override x.ToString() = match x with | Passed -> "Passed" - | PassedWithMessage m -> "Passed: " + m | Ignored reason -> "Ignored: " + reason | Failed error -> "Failed: " + error | Error e -> "Exception: " + exnWithInnerMsg e "" member x.tag = match x with | Passed -> 0 - | PassedWithMessage _ -> 0 | Ignored _ -> 1 | Failed _ -> 2 | Error _ -> 3 @@ -60,13 +57,11 @@ module Impl = match x with | Ignored _ -> 0 | Passed -> 1 - | PassedWithMessage _ -> 1 | Failed _ -> 2 | Error _ -> 3 member x.isPassed = match x with | Passed -> true - | PassedWithMessage _ -> true | _ -> false member x.isIgnored = match x with @@ -218,8 +213,6 @@ module Impl = info: string -> Async /// test name -> time taken -> unit passed: string -> TimeSpan -> Async - /// test name -> message -> time taken -> unit - passedWithMessage: string -> string -> TimeSpan -> Async /// test name -> ignore message -> unit ignored: string -> string -> Async /// test name -> other message -> time taken -> unit @@ -233,7 +226,6 @@ module Impl = let name = config.joinWith.format test.name match result.result with | Passed -> config.printer.passed name result.duration - | PassedWithMessage message -> config.printer.passedWithMessage name message result.duration | Failed message -> config.printer.failed name message result.duration | Ignored message -> config.printer.ignored name message | Error e -> config.printer.exn name e result.duration @@ -243,7 +235,6 @@ module Impl = beforeEach = fun _ -> async.Zero() info = fun _ -> async.Zero() passed = fun _ _ -> async.Zero() - passedWithMessage = fun _ _ _ -> async.Zero() ignored = fun _ _ -> async.Zero() failed = fun _ _ _ -> async.Zero() exn = fun _ _ _ -> async.Zero() @@ -267,13 +258,6 @@ module Impl = >> setField "testName" n >> setField "duration" d) - passedWithMessage = fun n m d -> - logger.logWithAck Debug ( - eventX "{testName} passed in {duration}. {message}" - >> setField "testName" n - >> setField "message" m - >> setField "duration" d) - ignored = fun n m -> logger.logWithAck Debug ( eventX "{testName} was ignored. {reason}" @@ -433,13 +417,6 @@ module Impl = "name", formatName n "duration", d.TotalMilliseconds |> int |> string ] } - passedWithMessage = fun n m d -> async { - do! innerPrinter.passedWithMessage n m d - tcLog "testFinished" [ - "flowId", formatName n - "name", formatName n - "duration", d.TotalMilliseconds |> int |> string ] } - info = fun s -> innerPrinter.info s @@ -486,7 +463,6 @@ module Impl = beforeEach = fun n -> runTwoAsyncs (first.beforeEach n) (second.beforeEach n) info = fun s -> runTwoAsyncs (first.info s) (second.info s) passed = fun n d -> runTwoAsyncs (first.passed n d) (second.passed n d) - passedWithMessage = fun n m d -> runTwoAsyncs (first.passedWithMessage n m d) (second.passedWithMessage n m d) ignored = fun n m -> runTwoAsyncs (first.ignored n m) (second.ignored n m) failed = fun n m d -> runTwoAsyncs (first.failed n m d) (second.failed n m d) exn = fun n e d -> runTwoAsyncs (first.exn n e d) (second.exn n e d) @@ -619,9 +595,6 @@ module Impl = w.Stop() return TestSummary.single Passed (float w.ElapsedMilliseconds) with - | :? PassWithMessage as e -> - w.Stop() - return TestSummary.single (PassedWithMessage ("\n"+e.Message)) (float w.ElapsedMilliseconds) | :? AssertException as e -> w.Stop() let msg = diff --git a/Expecto/Model.fs b/Expecto/Model.fs index dadf9244..8f1fc809 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -100,7 +100,6 @@ type Test = | Sequenced of SequenceMethod * Test type ExpectoException(msg) = inherit Exception(msg) -type PassWithMessage(msg) = inherit ExpectoException(msg) type AssertException(msg) = inherit ExpectoException(msg) type FailedException(msg) = inherit ExpectoException(msg) type IgnoreException(msg) = inherit ExpectoException(msg) diff --git a/Expecto/TestResults.fs b/Expecto/TestResults.fs index a92e816f..3b435d3c 100644 --- a/Expecto/TestResults.fs +++ b/Expecto/TestResults.fs @@ -40,7 +40,6 @@ let writeNUnitSummary file (summary: TestRunSummary) = |> addAttribute "executed" match test.result with - | PassedWithMessage _ | Passed -> "Success" | Error _ | Failed _ -> "Failure" @@ -48,7 +47,6 @@ let writeNUnitSummary file (summary: TestRunSummary) = |> addAttribute "result" match test.result with - | PassedWithMessage _ | Passed -> addAttribute "success" "True" | Error _ | Failed _ -> addAttribute "success" "False" @@ -61,15 +59,11 @@ let writeNUnitSummary file (summary: TestRunSummary) = // TODO: implement it. addAttribute "asserts" "0" - let passNode = XElement(XName.Get "pass") let failureNode = XElement(XName.Get "failure") // Some more details that explain why a test was not executed. match test.result with | Passed -> () - | PassedWithMessage m -> - passNode.Add(XName.Get "message", XCData m) - element.Add passNode | Error e -> failureNode.Add(XName.Get "message", XCData e.Message) failureNode.Add(XName.Get "stack-trace", XCData e.StackTrace) @@ -142,7 +136,6 @@ let writeJUnitSummary file (summary: Impl.TestRunSummary) = XAttribute(XName.Get "message", message)) match test.result with | Passed -> [||] - | PassedWithMessage msg -> [|makeMessageNode "pass" msg|] | Error e -> let message = makeMessageNode "error" e.Message message.Add(XCData(e.ToString())) From 37098ebd9c69848b92c3446bf033572724a38d2d Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 6 Jan 2025 12:43:02 -0700 Subject: [PATCH 08/14] Add logger field to fscheck config --- Expecto/Model.fs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Expecto/Model.fs b/Expecto/Model.fs index 8f1fc809..6e866ea7 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -40,6 +40,8 @@ type FsCheckConfig = quietOnSuccess: bool /// The maximum number of tests where values are rejected, e.g. as the result of ==> maxRejected: int + /// If set, this logger is used to show test information for passing tests. + logger: Logging.Logger option } static member defaultConfig = @@ -53,6 +55,7 @@ type FsCheckConfig = finishedTest = fun _ _ -> async.Return () quietOnSuccess = true maxRejected = 1000 + logger = None } /// Actual test function; either an async one, or a synchronous one. From 47d15bb5ae92fcb406f022b9c012917eda36da8a Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 6 Jan 2025 12:45:44 -0700 Subject: [PATCH 09/14] Log number of tests and test distribution on property test pass --- Expecto.FsCheck/FsCheck.fs | 4 +++- Expecto.FsCheck3/FsCheck3.fs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 823babdb..f0f29823 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -42,7 +42,9 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.True (_testData,_b) -> () + | TestResult.True (data, suppressOutput) -> + if not suppressOutput && config.logger.IsSome then + config.logger.Value.log Logging.LogLevel.Info (Logging.Message.eventX $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}") |> Async.RunSynchronously | TestResult.False (_,_,_, Outcome.Exception (:? IgnoreException as e),_) -> raise e diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 1cf844b0..2a8969d0 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -42,7 +42,9 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.Passed (_testData,_b) -> () + | TestResult.Passed (data, suppressOutput) -> + if not suppressOutput && config.logger.IsSome then + config.logger.Value.log Logging.LogLevel.Info (Logging.Message.eventX $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}") |> Async.RunSynchronously | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e From 4240e6d9aa7e819e4af42c92771bb54ecf1f43ff Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 13 Jan 2025 13:20:25 -0700 Subject: [PATCH 10/14] Add fscheck test data to `FsCheckConfig.finishedTest` --- Expecto.FsCheck/FsCheck.fs | 15 ++++++++++++++- Expecto.FsCheck3/FsCheck3.fs | 18 +++++++++++++++--- Expecto/Model.fs | 9 ++++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index f0f29823..7ef86099 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -29,7 +29,20 @@ module ExpectoFsCheck = /// Called whenever all tests are done, either True, False or Exhausted. member __.OnFinished (fsCheckTestName, testResult) = - config.finishedTest config fsCheckTestName + let testData = + match testResult with + | TestResult.True(testData, _) -> testData + | TestResult.False(testData, _, _, _, _) -> testData + | TestResult.Exhausted testData -> testData + + let testData = { + FsCheckTestData.Stamps = testData.Stamps + NumberOfTests = testData.NumberOfTests + NumberOfShrinks = testData.NumberOfShrinks + Labels = testData.Labels + } + + config.finishedTest config fsCheckTestName testData |> Async.RunSynchronously let numTests i = if i = 1 then "1 test" else sprintf "%i tests" i diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 2a8969d0..42ecd288 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -28,9 +28,21 @@ module ExpectoFsCheck = |> Async.RunSynchronously /// Called whenever all tests are done, either True, False or Exhausted. - member __.OnFinished (fsCheckTestName, testResult) = - config.finishedTest config fsCheckTestName - |> Async.RunSynchronously + member __.OnFinished(fsCheckTestName, testResult) = + let testData = + match testResult with + | TestResult.Passed(testData, _) -> testData + | TestResult.Failed(testData, _, _, _, _, _, _) -> testData + | TestResult.Exhausted testData -> testData + + let testData = { + FsCheckTestData.Stamps = testData.Stamps + NumberOfTests = testData.NumberOfTests + NumberOfShrinks = testData.NumberOfShrinks + Labels = testData.Labels + } + + config.finishedTest config fsCheckTestName testData |> Async.RunSynchronously let numTests i = if i = 1 then "1 test" else sprintf "%i tests" i diff --git a/Expecto/Model.fs b/Expecto/Model.fs index 6e866ea7..e8f4ab1b 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -8,6 +8,12 @@ type SourceLocation = { sourcePath: string; lineNumber: int } static member empty = { sourcePath = ""; lineNumber = 0 } +type FsCheckTestData = + { Labels: Set + NumberOfShrinks: int + NumberOfTests: int + Stamps: seq> } + type FsCheckConfig = { /// The maximum number of tests that are run. maxTest: int @@ -35,6 +41,7 @@ type FsCheckConfig = /// Callback when the test case has finished finishedTest: FsCheckConfig -> (* test name *) string + -> FsCheckTestData -> Async /// If set, suppresses the output from the test if the test is successful. quietOnSuccess: bool @@ -52,7 +59,7 @@ type FsCheckConfig = arbitrary = [] receivedArgs = fun _ _ _ _ -> async.Return () successfulShrink = fun _ _ _ -> async.Return () - finishedTest = fun _ _ -> async.Return () + finishedTest = fun _ _ _ -> async.Return () quietOnSuccess = true maxRejected = 1000 logger = None From 78d43dac9b43c1924b74f736c06d246500c56657 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 13 Jan 2025 15:01:04 -0700 Subject: [PATCH 11/14] Revert "Log number of tests and test distribution on property test pass" This reverts commit 47d15bb5ae92fcb406f022b9c012917eda36da8a. --- Expecto.FsCheck/FsCheck.fs | 4 +--- Expecto.FsCheck3/FsCheck3.fs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index 7ef86099..2a115dd5 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -55,9 +55,7 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.True (data, suppressOutput) -> - if not suppressOutput && config.logger.IsSome then - config.logger.Value.log Logging.LogLevel.Info (Logging.Message.eventX $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}") |> Async.RunSynchronously + | TestResult.True (_testData,_b) -> () | TestResult.False (_,_,_, Outcome.Exception (:? IgnoreException as e),_) -> raise e diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 42ecd288..fe36822c 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -54,9 +54,7 @@ module ExpectoFsCheck = | xs -> sprintf "%s\n" (String.concat "\n" xs) match testResult with - | TestResult.Passed (data, suppressOutput) -> - if not suppressOutput && config.logger.IsSome then - config.logger.Value.log Logging.LogLevel.Info (Logging.Message.eventX $"Passed {numTests data.NumberOfTests}.\n{stampsToString data.Stamps}") |> Async.RunSynchronously + | TestResult.Passed (_testData,_b) -> () | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) -> raise e From a3ae6fddbde909c7a5d7b15e07391ef5982cc130 Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 13 Jan 2025 15:01:05 -0700 Subject: [PATCH 12/14] Revert "Add logger field to fscheck config" This reverts commit 37098ebd9c69848b92c3446bf033572724a38d2d. --- Expecto/Model.fs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Expecto/Model.fs b/Expecto/Model.fs index e8f4ab1b..406a8fc1 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -47,8 +47,6 @@ type FsCheckConfig = quietOnSuccess: bool /// The maximum number of tests where values are rejected, e.g. as the result of ==> maxRejected: int - /// If set, this logger is used to show test information for passing tests. - logger: Logging.Logger option } static member defaultConfig = @@ -62,7 +60,6 @@ type FsCheckConfig = finishedTest = fun _ _ _ -> async.Return () quietOnSuccess = true maxRejected = 1000 - logger = None } /// Actual test function; either an async one, or a synchronous one. From 1afcd6da7b95f4f9603adef44ce1f4a523b6d28e Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Mon, 13 Jan 2025 15:07:36 -0700 Subject: [PATCH 13/14] Update RELEASE_NOTES.md and README.md --- README.md | 1 + RELEASE_NOTES.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 951199fc..cf2822d8 100644 --- a/README.md +++ b/README.md @@ -744,6 +744,7 @@ type FsCheckConfig = /// Callback when the test case has finished finishedTest: FsCheckConfig -> (* test name *) string + -> FsCheckTestData -> Async /// If set, suppresses the output from the test if the test is successful. quietOnSuccess: bool diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f6e0a8d5..076be682 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,5 @@ ### 11.0.0-alpha5 - 2025-01-13 -* Allow quietOnSuccess and maxRejected fscheck config +* Add third argument to `FsCheckConfig.testFinished` containing information about the finished test. ### 11.0.0-alpha4 - 2025-01-06 * Breaking change: Add third item to `FsCheckConfig.replay` indicating the size From f6fbdd9651c7094ae29f0da0e5e94551eb09316b Mon Sep 17 00:00:00 2001 From: Calum Sieppert Date: Wed, 12 Feb 2025 11:54:33 -0700 Subject: [PATCH 14/14] Revert old changes: maxRejected, quietOnSuccess, replay --- Expecto.FsCheck/FsCheck.fs | 4 ++-- Expecto.FsCheck3/FsCheck3.fs | 3 +-- Expecto/Model.fs | 11 ++--------- README.md | 10 +++------- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Expecto.FsCheck/FsCheck.fs b/Expecto.FsCheck/FsCheck.fs index c490ae79..13514d41 100644 --- a/Expecto.FsCheck/FsCheck.fs +++ b/Expecto.FsCheck/FsCheck.fs @@ -102,14 +102,14 @@ module ExpectoFsCheck = let test (config: FsCheckConfig) = let config = { MaxTest = config.maxTest - MaxFail = config.maxRejected + MaxFail = 1000 // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the // values given in the test output, which are only ints when running FsCheck 2 Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma)) Name = name StartSize = config.startSize EndSize = config.endSize - QuietOnSuccess = config.quietOnSuccess + QuietOnSuccess = true Every = fun _ _ -> String.Empty EveryShrink = fun _ -> String.Empty Arbitrary = config.arbitrary diff --git a/Expecto.FsCheck3/FsCheck3.fs b/Expecto.FsCheck3/FsCheck3.fs index 737ef873..1bcc3304 100644 --- a/Expecto.FsCheck3/FsCheck3.fs +++ b/Expecto.FsCheck3/FsCheck3.fs @@ -109,10 +109,9 @@ module ExpectoFsCheck = .WithName(name) .WithStartSize(config.startSize) .WithEndSize(config.endSize) - .WithQuietOnSuccess(config.quietOnSuccess) + .WithQuietOnSuccess(true) .WithArbitrary(config.arbitrary) .WithRunner(runner config) - .WithMaxRejected(config.maxRejected) Check.One(config, property) diff --git a/Expecto/Model.fs b/Expecto/Model.fs index f9be3dc0..c5f910e6 100644 --- a/Expecto/Model.fs +++ b/Expecto/Model.fs @@ -15,8 +15,8 @@ type FsCheckTestData = Stamps: seq> } type FsCheckConfig = - { /// The maximum number of tests that are run. - maxTest: int + /// The maximum number of tests that are run. + { maxTest: int /// The size to use for the first test. startSize: int /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. @@ -44,10 +44,6 @@ type FsCheckConfig = -> (* test name *) string -> FsCheckTestData -> Async - /// If set, suppresses the output from the test if the test is successful. - quietOnSuccess: bool - /// The maximum number of tests where values are rejected, e.g. as the result of ==> - maxRejected: int } static member defaultConfig = @@ -59,8 +55,6 @@ type FsCheckConfig = receivedArgs = fun _ _ _ _ -> async.Return () successfulShrink = fun _ _ _ -> async.Return () finishedTest = fun _ _ _ -> async.Return () - quietOnSuccess = true - maxRejected = 1000 } /// Actual test function; either an async one, or a synchronous one. @@ -140,4 +134,3 @@ type private TestNameHolder() = static member Name with get () = TestNameHolder.name and set name = TestNameHolder.name <- name - diff --git a/README.md b/README.md index 5ccbec4a..cf0b6d09 100644 --- a/README.md +++ b/README.md @@ -718,14 +718,14 @@ like the following. ```fsharp type FsCheckConfig = - { /// The maximum number of tests that are run. - maxTest: int + /// The maximum number of tests that are run. + { maxTest: int /// The size to use for the first test. startSize: int /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize. endSize: int /// If set, the seed to use to start testing. Allows reproduction of previous runs. - replay: (uint64 * uint64 * int) option + replay: (uint64 * uint64) option /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be /// overridden) @@ -746,10 +746,6 @@ type FsCheckConfig = -> (* test name *) string -> FsCheckTestData -> Async - /// If set, suppresses the output from the test if the test is successful. - quietOnSuccess: bool - /// The maximum number of tests where values are rejected, e.g. as the result of ==> - maxRejected: int } ```