tasty-expected-failure-0.12.3/0000755000000000000000000000000007346545000014356 5ustar0000000000000000tasty-expected-failure-0.12.3/LICENSE0000644000000000000000000000204407346545000015363 0ustar0000000000000000Copyright (c) 2015 Joachim Breitner Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. tasty-expected-failure-0.12.3/README.md0000755000000000000000000000352507346545000015645 0ustar0000000000000000tasty-expected-failure ====================== What is this? ------------- With the function `expectFail` in the provided module `ExpectedFailure`, you can mark that you expect test cases to fail, and not to pass. This can for example be used for test-driven development: Create the tests, mark them with `expectFail`, and you can still push to the main branch, without your continuous integration branch failing. Once someone implements the feature or fixes the bug (maybe unknowingly), the test suite will tell him so, due to the now unexpectedly passing test, and he can remove the `expectFail` marker. The module also provides `ignoreTest` to avoid running a test. Both funtions are implemented via the more general `wrapTest`, which is also provided. Why is this not provided by tasty? ---------------------------------- `` The author of the tasty library prefers to provide a minimal experience in the tasty library, instead of a batteries-included approach, and chose not to include these 39 lines of code in tasty. See the [issue](https://github.com/feuerbach/tasty/issues/114) for the discussion. Instead I wrote 37 lines of cabal file, a similar number of lines of README, created a git repository, created a travis file, run travis to figure out on what versions it builds (something that would have happened automatically with a pull request for tasty), upload to hackage, add to stackage. Furthermore, there is little discoverability: If it were part of the tasty API, users would stumble over it. Now they likely won’t. And if they do, they have to worry about whether it is still in sync with `tasty`, they have to add it to their build-depends, they have to import yet another module. Distribution packagers will have yet another package where they have to create the packaging, check the copyright, and run autobuilders for. Sigh. `` tasty-expected-failure-0.12.3/Setup.hs0000644000000000000000000000005607346545000016013 0ustar0000000000000000import Distribution.Simple main = defaultMain tasty-expected-failure-0.12.3/Test/Tasty/0000755000000000000000000000000007346545000016401 5ustar0000000000000000tasty-expected-failure-0.12.3/Test/Tasty/ExpectedFailure.hs0000644000000000000000000001745707346545000022024 0ustar0000000000000000{-# LANGUAGE CPP #-} {-# LANGUAGE DeriveDataTypeable, ScopedTypeVariables, LambdaCase #-} module Test.Tasty.ExpectedFailure (expectFail, expectFailBecause, ignoreTest, ignoreTestBecause, wrapTest) where import Test.Tasty.Options import Test.Tasty.Runners import Test.Tasty.Providers #if MIN_VERSION_tasty(1,3,1) import Test.Tasty.Providers.ConsoleFormat ( ResultDetailsPrinter(..) ) #endif import Test.Tasty ( Timeout(..), askOption, localOption ) import Data.Typeable import Data.Tagged import Data.Maybe import Data.Monoid import Control.Exception ( displayException, evaluate, try, SomeException ) import Control.Concurrent.Timeout ( timeout ) data WrappedTest t = WrappedTest Timeout (IO Result -> IO Result) t deriving Typeable instance forall t. IsTest t => IsTest (WrappedTest t) where run opts (WrappedTest tmout wrap t) prog = -- Re-implement timeouts and exception handling *inside* the -- wrapper. The primary location for timeout and exception -- handling is in `executeTest` in the Tasty module's -- Test.Tasty.Run implementation, but that handling is above the -- level of this wrapper which therefore cannot absorb timeouts -- and exceptions as *expected* failures. let (pre,post) = case tmout of NoTimeout -> (fmap Just, fromJust) Timeout t s -> (timeout t, fromMaybe (timeoutResult t s)) timeoutResult t s = Result { resultOutcome = Failure $ TestTimedOut t , resultDescription = "Timed out after " <> s , resultShortDescription = "TIMEOUT" , resultTime = fromIntegral t #if MIN_VERSION_tasty(1,3,1) , resultDetailsPrinter = ResultDetailsPrinter . const . const $ return () #endif } exceptionResult e = Result { resultOutcome = Failure $ TestThrewException e , resultDescription = "Exception: " ++ displayException e , resultShortDescription = "FAIL" , resultTime = 0 #if MIN_VERSION_tasty(1,3,1) , resultDetailsPrinter = ResultDetailsPrinter . const . const $ return () #endif } forceList = foldr seq () in wrap $ try (pre (run opts t prog -- Ensure exceptions trying to show the -- failure result are caught as "expected" -- (see Issue #24 and note below) >>= \r -> evaluate (forceList (resultDescription r) `seq` forceList (resultShortDescription r) `seq` resultOutcome r `seq` r))) >>= \case Right r -> return (post r) Left (e :: SomeException) -> return $ exceptionResult e testOptions = retag (testOptions :: Tagged t [OptionDescription]) -- Note regarding post-run evaluate above: -- -- The normal behavior of tasty-expected-failure is to run the -- test, show the failure result, but then note that the failure -- is expected and not count that against a test failure. If the -- test unexpectedly succeeds, a message to that effect is -- printed, but there is no resultDescription display of the test -- inputs. -- -- As of Tasty 1.4, the core tasty code was enhanced to fix issue -- #280 in tasty: essentially the test result report is forced. -- However, when used with tests expected to fail that also throw -- exceptions when attempting to show the result, the forcing in -- Tasty 1.4 causes an exception to be thrown after the -- tasty-expected-failure protections but still within the realm -- where tasty would count it as a failure. The fix here attempts -- to `show` the failing value here in tasty-expected-failure; if -- an exception occurs during that `show` then code here will -- report it (somewhat incorrectly) via the exceptionResult above, -- where tasty's subsequent forcing of the text of that -- exceptionResult no longer causes an exception *there*. Since -- the value is only shown if there was already a failure, the -- reason is misleading but the outcome is consistent with the -- intent of tasty-expected-failure handling. -- | 'wrapTest' allows you to modify the behaviour of the tests, e.g. by -- modifying the result or not running the test at all. It is used to implement -- 'expectFail' and 'ignoreTest'. wrapTest :: (IO Result -> IO Result) -> TestTree -> TestTree wrapTest wrap = go where go (SingleTest n t) = askOption $ \(old_timeout :: Timeout) -> localOption NoTimeout $ -- disable Tasty's timeout; handled here instead SingleTest n (WrappedTest old_timeout wrap t) go (TestGroup name tests) = TestGroup name (map go tests) go (PlusTestOptions plus tree) = PlusTestOptions plus (go tree) go (WithResource spec gentree) = WithResource spec (go . gentree) go (AskOptions f) = AskOptions (go . f) -- | Marks all tests in the given test suite as expected failures: The tests will -- still be run, but if they succeed, it is reported as a test suite failure, -- and conversely a the failure of the test is ignored. -- -- Any output of a failing test is still printed. -- -- This is useful if, in a test driven development, tests are written and -- commited to the master branch before their implementation: It allows the -- tests to fail (as expected) without making the whole test suite fail. -- -- Similarly, regressions and bugs can be documented in the test suite this -- way, until a fix is commited, and if a fix is applied (intentionally or -- accidentially), the test suite will remind you to remove the 'expectFail' -- marker. expectFail :: TestTree -> TestTree expectFail = expectFail' Nothing -- | Like 'expectFail' but with additional comment expectFailBecause :: String -> TestTree -> TestTree expectFailBecause reason = expectFail' (Just reason) expectFail' :: Maybe String -> TestTree -> TestTree expectFail' reason = wrapTest (fmap change) where change r | resultSuccessful r = r { resultOutcome = Failure TestFailed , resultDescription = resultDescription r <> " (unexpected success" <> comment <> ")" , resultShortDescription = resultShortDescription r <> " (unexpected" <> comment <> ")" } | otherwise = r { resultOutcome = Success , resultDescription = resultDescription r <> " (expected failure)" , resultShortDescription = resultShortDescription r <> " (expected" <> comment <> ")" } "" `append` s = s t `append` s | last t == '\n' = t ++ s ++ "\n" | otherwise = t ++ "\n" ++ s comment = maybe "" (mappend ": ") reason -- | Prevents the tests from running and reports them as succeeding. -- -- This may be be desireable as an alternative to commenting out the tests. This -- way, they are still typechecked (preventing bitrot), and the test report -- lists them, which serves as a reminder that there are ignored tests. -- -- Note that any setup/teardown actions executed by 'Test.Tasty.withResource' -- are still executed. You can bypass this manually as in the following example: -- -- @ -- askOption $ \\(MyFlag b) -> if b -- then withResource mytest -- else ignoreTest . mytest $ return junkvalue -- @ ignoreTest :: TestTree -> TestTree ignoreTest = ignoreTest' Nothing -- | Like 'ignoreTest' but with additional comment ignoreTestBecause :: String -> TestTree -> TestTree ignoreTestBecause reason = ignoreTest' (Just reason) ignoreTest' :: Maybe String -> TestTree -> TestTree ignoreTest' reason = wrapTest $ const $ return $ (testPassed $ fromMaybe "" reason) { resultShortDescription = "IGNORED" } tasty-expected-failure-0.12.3/tasty-expected-failure.cabal0000644000000000000000000000462007346545000021734 0ustar0000000000000000name: tasty-expected-failure version: 0.12.3 synopsis: Mark tasty tests as failure expected description: With the function 'Test.Tasty.ExpectedFailure.expectFail' in the provided module "Test.Tasty.ExpectedFailure", you can mark that you expect test cases to fail, and not to pass. . This can for example be used for test-driven development: Create the tests, mark them with 'Test.Tasty.ExpectedFailure.expectFail', and you can still push to the main branch, without your continuous integration branch failing. . Once someone implements the feature or fixes the bug (maybe unknowingly), the test suite will tell him so, due to the now unexpectedly passing test, and he can remove the 'Test.Tasty.ExpectedFailure.expectFail' marker. . The module also provides 'Test.Tasty.ExpectedFailure.ignoreTest' to avoid running a test. Both funtions are implemented via the more general 'Test.Tasty.ExpectedFailure.wrapTest', which is also provided. homepage: http://github.com/nomeata/tasty-expected-failure license: MIT license-file: LICENSE author: Joachim Breitner maintainer: mail@joachim-breitner.de copyright: 2015 Joachim Breitner category: Testing build-type: Simple extra-source-files: README.md cabal-version: >=1.10 tested-with: GHC == 8.0.2, GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.5, GHC == 8.8.3, GHC == 8.10.1 library exposed-modules: Test.Tasty.ExpectedFailure build-depends: base >= 4.9 && <5, tagged >= 0.7 && < 0.9, tasty >= 0.11, unbounded-delays < 0.2 default-language: Haskell2010 source-repository head type: git location: git://github.com/nomeata/tasty-expected-failure test-suite expected-fail-tests type: exitcode-stdio-1.0 default-language: Haskell2010 hs-source-dirs: tests main-is: TestExpectedFails.hs build-depends: base, tasty, tasty-hunit, tasty-golden, tasty-expected-failure test-suite expected-fail-hh-tests type: exitcode-stdio-1.0 default-language: Haskell2010 hs-source-dirs: tests main-is: TestExpectedFailsHH.hs build-depends: base, hedgehog, tasty, tasty-hedgehog, tasty-expected-failure tasty-expected-failure-0.12.3/tests/0000755000000000000000000000000007346545000015520 5ustar0000000000000000tasty-expected-failure-0.12.3/tests/TestExpectedFails.hs0000644000000000000000000000325007346545000021434 0ustar0000000000000000import Control.Concurrent (threadDelay) import Test.Tasty import Test.Tasty.ExpectedFailure import Test.Tasty.Golden import Test.Tasty.HUnit -- n.b. running via `cabal v2-test` outputs plaintext, but running via -- `cabal v2-run test:expected-fail-tests` will generate colorized -- output. It's adviseable to visually inspect this output to ensure -- that "PASS (unexpected)" is rendered in Red and "FAIL (expected)" -- is rendered in Green. main = defaultMain $ localOption (mkTimeout 1000000) $ -- 1s testGroup "Expected Failures" $ [ testCase "clearly good" $ 1 + 1 @=? 2 , expectFail $ testCase "clearly bad" $ 1 + 1 @=? 3 -- n.b. uncomment this to observe the results of a test that was -- , expectFail $ testCase "also good" $ 1 + 2 @=? 3 , expectFail $ expectFail $ testCase "two wrongs make a right" $ 1 + 1 @=? 2 , expectFail $ testCase "throws failure" $ fail "bad" , expectFail $ testCase "throws error" $ error "also bad" , expectFail $ testCase "takes too long" $ threadDelay 2000000 , expectFail $ goldenVsString "hello" "hello.out" $ return $ error "not golden" -- Issue 24 , expectFail $ testCase "this is expected to fail" (mrtOne (someFunc "Hello, world!") @?= "") , expectFail $ testCase "this is also expected to fail" (someFunc "Hello, world!" @?= MyResultType { mrtOne = "", mrtTwo = "" }) ] data MyResultType = MyResultType { mrtOne :: String , mrtTwo :: String } deriving (Eq, Show) someFunc :: String -> MyResultType someFunc fp = MyResultType { mrtOne = fp , mrtTwo = undefined } tasty-expected-failure-0.12.3/tests/TestExpectedFailsHH.hs0000644000000000000000000000266707346545000021667 0ustar0000000000000000import Control.Concurrent ( threadDelay ) import Control.Monad.IO.Class ( liftIO ) import Hedgehog import qualified Hedgehog.Gen as Gen import qualified Hedgehog.Range as Range import Test.Tasty import Test.Tasty.ExpectedFailure import Test.Tasty.Hedgehog -- n.b. running via `cabal v2-test` outputs plaintext, but running via -- `cabal v2-run test:expected-fail-tests` will generate colorized -- output. It's adviseable to visually inspect this output to ensure -- that "PASS (unexpected)" is rendered in Red and "FAIL (expected)" -- is rendered in Green. main = defaultMain $ localOption (mkTimeout 1000000) $ -- 1s testGroup "Expected Hedgehog Failures" $ [ testProperty "good" $ property $ success , expectFail $ testProperty "rarely good" $ property $ do xs <- forAll $ Gen.list (Range.linear 0 10) Gen.alpha reverse xs === xs -- n.b. uncomment this to observe the results of a test that was -- expected to fail but actually passes. -- , expectFail $ testProperty "surprisingly good" $ property $ success , expectFail $ testProperty "giving up" $ property $ discard , expectFail $ expectFail $ testProperty "the failure of a failure is my good" $ property $ success , expectFail $ testProperty "throws failure" $ property $ fail "bad" , expectFail $ testProperty "too slow" $ property $ do liftIO $ threadDelay 2000000 success ]