Implement labelledExamples

core/shared/src/main/scala/hedgehog/core/Coverage.scala

@@ -89,6 +89,17 @@ object Coverage {
   def empty[A]: Coverage[A] =
     Coverage(Map.empty[LabelName, Label[A]])
 
+  def labels(cv: Coverage[_]): List[LabelName] =
+    cv.labels.keysIterator.toList
+
+  def covers(cv: Coverage[Cover], name: LabelName): Boolean =
+    cv.labels.get(name).exists(_.annotation match {
+      case Cover.Cover =>
+        true
+      case Cover.NoCover =>
+        false
+    })
+
   def count(cv: Coverage[Cover]): Coverage[CoverCount] =
     cv.copy(labels = cv.labels.map { case (k, l) =>
       k -> l.copy(annotation = CoverCount.fromCover(l.annotation))
@@ -105,3 +116,16 @@ object Coverage {
   }
 }
 
+case class Examples(examples: Map[LabelName, List[Log]])
+
+object Examples {
+
+  def empty[A]: Examples =
+    Examples(Map.empty[LabelName, List[Log]])
+
+  def addTo(examples: Examples, labels: List[LabelName])(seek: LabelName => List[Log]): Examples = {
+    Examples(labels.foldLeft(examples.examples) { (m, name) =>
+      m.updated(name, m.get(name).filter(_.nonEmpty).getOrElse(seek(name)))
+    })
+  }
+}

core/shared/src/main/scala/hedgehog/core/PropertyT.scala

@@ -46,12 +46,13 @@ case class PropertyConfig(
     testLimit: SuccessCount
   , discardLimit: DiscardCount
   , shrinkLimit: ShrinkLimit
+  , withExamples: WithExamples
   )
 
 object PropertyConfig {
 
   def default: PropertyConfig =
-    PropertyConfig(SuccessCount(100), DiscardCount(100), ShrinkLimit(1000))
+    PropertyConfig(SuccessCount(100), DiscardCount(100), ShrinkLimit(1000), WithExamples.NoExamples)
 }
 
 case class PropertyT[A](
@@ -132,38 +133,51 @@ object PropertyT {
 trait PropertyTReporting {
 
   @annotation.tailrec
-  final def takeSmallest(n: ShrinkCount, slimit: ShrinkLimit, t: Tree[Option[(List[Log], Option[Result])]]): Status =
-    t.value match {
+  final def takeSmallestG[A, B](n: ShrinkCount, slimit: ShrinkLimit, t: Tree[A])(p: A => Boolean)(e: (ShrinkCount, A) => B): B = {
+    if (n.value < slimit.value && p(t.value)) {
+      findMap(t.children.value)(m => if (p(m.value)) some(m) else Option.empty) match {
+        case None =>
+          e(n, t.value)
+
+        case Some(m) =>
+          takeSmallestG(n.inc, slimit, m)(p)(e)
+      }
+    } else {
+      e(n, t.value)
+    }
+  }
+
+  def takeSmallest(n: ShrinkCount, slimit: ShrinkLimit, t: Tree[Option[(Journal, Option[Result])]]): Status =
+    takeSmallestG(n, slimit, t) {
       case None =>
+        false
+
+      case Some((_, r)) =>
+        r.forall(!_.success)
+    } {
+      case (_, None) =>
         Status.gaveUp
 
-      case Some((w, r)) =>
-        if (r.forall(!_.success)) {
-          if (n.value >= slimit.value) {
-            Status.failed(n, w ++ r.map(_.logs).getOrElse(Nil))
-          } else {
-            findMap(t.children.value)(m =>
-              m.value match {
-                case Some((_, None)) =>
-                  some(m)
-                case Some((_, Some(r2))) =>
-                  if (!r2.success)
-                    some(m)
-                  else
-                    Option.empty
-                case None =>
-                  Option.empty
-              }
-            ) match {
-              case Some(m) =>
-                takeSmallest(n.inc, slimit, m)
-              case None =>
-                Status.failed(n, w ++ r.map(_.logs).getOrElse(Nil))
-            }
-          }
-        } else {
+      case (n, Some((j, r))) =>
+        if (r.forall(!_.success))
+          Status.failed(n, j.logs ++ r.map(_.logs).getOrElse(Nil))
+        else
           Status.ok
-        }
+    }
+
+  def takeSmallestExample(n: ShrinkCount, slimit: ShrinkLimit, name: LabelName, t: Tree[Option[(Journal, Option[Result])]]): List[Log] =
+    takeSmallestG(n, slimit, t) {
+      case None =>
+        false
+
+      case Some((j, r)) =>
+        r.exists(_.success) && Coverage.covers(j.coverage, name)
+    } {
+      case (_, None) =>
+        Nil
+
+      case (_, Some((j, _))) =>
+        j.logs
     }
 
   def report(config: PropertyConfig, size0: Option[Size], seed0: Seed, p: PropertyT[Result]): Report = {
@@ -172,17 +186,20 @@ trait PropertyTReporting {
     // Start the size at whatever remainder we have to ensure we run with "max" at least once
     val sizeInit = Size((Size.max % Math.min(config.testLimit.value, Size.max)) + sizeInc.value)
     @annotation.tailrec
-    def loop(successes: SuccessCount, discards: DiscardCount, size: Size, seed: Seed, coverage: Coverage[CoverCount]): Report =
+    def loop(successes: SuccessCount, discards: DiscardCount, size: Size, seed: Seed, coverage: Coverage[CoverCount], examples: Examples): Report =
       if (successes.value >= config.testLimit.value)
         // we've hit the test limit
         Coverage.split(coverage, successes) match {
           case (_, Nil) =>
-            Report(successes, discards, coverage, OK)
+            if (examples.examples.exists(_._2.isEmpty))
+              Report(successes, discards, coverage, examples, Status.failed(ShrinkCount(0), List("Insufficient examples.")))
+            else
+              Report(successes, discards, coverage, examples, OK)
           case _ =>
-            Report(successes, discards, coverage, Status.failed(ShrinkCount(0), List("Insufficient coverage.")))
+            Report(successes, discards, coverage, examples, Status.failed(ShrinkCount(0), List("Insufficient coverage.")))
         }
       else if (discards.value >= config.discardLimit.value)
-        Report(successes, discards, coverage, GaveUp)
+        Report(successes, discards, coverage, examples, GaveUp)
       else {
         val x =
           try {
@@ -191,23 +208,28 @@ trait PropertyTReporting {
             case e: Exception =>
               Property.error(e).run.run(size, seed)
           }
-        val t = x.map(_._2.map { case (l, r) => (l.logs, r) })
+        val t = x.map(_._2)
         x.value._2 match {
           case None =>
-            loop(successes, discards.inc, size.incBy(sizeInc), x.value._1, coverage)
-
-          case Some((_, None)) =>
-            Report(successes, discards, coverage, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
-
-          case Some((j, Some(r))) =>
-            if (!r.success){
-              Report(successes, discards, coverage, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
-            } else
-              loop(successes.inc, discards, size.incBy(sizeInc), x.value._1,
-                Coverage.union(Coverage.count(j.coverage), coverage)(_ + _))
+            loop(successes, discards.inc, size.incBy(sizeInc), x.value._1, coverage, examples)
+
+          case Some((j, r)) =>
+            if (r.forall(!_.success)) {
+              Report(successes, discards, coverage, examples, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
+            } else {
+              val coverage2 = Coverage.union(Coverage.count(j.coverage), coverage)(_ + _)
+              val examples2 =
+                Examples.addTo(examples, Coverage.labels(j.coverage)) { name =>
+                  if (Coverage.covers(j.coverage, name))
+                    takeSmallestExample(ShrinkCount(0), config.shrinkLimit, name, t)
+                  else
+                    Nil
+                }
+              loop(successes.inc, discards, size.incBy(sizeInc), x.value._1, coverage2, examples2)
+            }
         }
       }
-    loop(SuccessCount(0), DiscardCount(0), size0.getOrElse(sizeInit), seed0, Coverage.empty)
+    loop(SuccessCount(0), DiscardCount(0), size0.getOrElse(sizeInit), seed0, Coverage.empty, Examples.empty)
   }
 
   def recheck(config: PropertyConfig, size: Size, seed: Seed)(p: PropertyT[Result]): Report =
@@ -247,6 +269,15 @@ case class DiscardCount(value: Int) {
     DiscardCount(value + 1)
 }
 
+/** Whether the report should include an example for each label. */
+sealed trait WithExamples
+
+object WithExamples {
+
+  case object WithExamples extends WithExamples
+  case object NoExamples extends WithExamples
+}
+
 /**
  * The status of a property test run.
  *
@@ -270,4 +301,4 @@ object Status {
     OK
 }
 
-case class Report(tests: SuccessCount, discards: DiscardCount, coverage: Coverage[CoverCount], status: Status)
+case class Report(tests: SuccessCount, discards: DiscardCount, coverage: Coverage[CoverCount], examples: Examples, status: Status)

doc/tutorial.md

@@ -16,6 +16,7 @@ Tutorial
     - [Filtering](#filtering)
     - [Sized](#sized)
   - [Shrinking](#shrinking)
+  - [Classifications](#classifications)
 - [State](#state)
 
 ## Thanks
@@ -669,6 +670,54 @@ Using seed from environment variable HEDGEHOG_SEED: 58973622580784
 + hedgehog.PropertyTest$.applicative shrink: OK, passed 100 tests
 ```
 
+
+### Classifications
+
+Using `classify` you can add classifications to your generator's data, for example:
+
+```scala
+  def testReverse: Property =
+    for {
+      xs <- Gen.alpha.list(Range.linear(0, 10)).forAll
+         .classify("empty", _.isEmpty)
+         .classify("nonempty", _.nonEmpty)
+    } yield xs.reverse.reverse ==== xs
+```
+
+Running that property will produce a result like:
+
+```
+[info] + hedgehog.examples.ReverseTest.reverse: OK, passed 100 tests
+[info] > 69% nonempty List(a)
+[info] > 31% empty List()
+```
+
+Notice how, in addition to the percentage, it also presents a shrunk example for that classifier.
+
+Using `cover` you may also specify a minimum coverage percentage for the given classification:
+
+```scala
+  def testReverse: Property =
+    for {
+      xs <- Gen.alpha.list(Range.linear(0, 10)).forAll
+         .cover(50, "empty", _.isEmpty)
+         .cover(50, "nonempty", _.nonEmpty)
+    } yield xs.reverse.reverse ==== xs
+```
+
+```
+[info] - hedgehog.examples.ReverseTest.reverse: Falsified after 100 passed tests
+[info] > Insufficient coverage.
+[info] > 93% nonempty 50% ✓ List(a)
+[info] > 7% empty 50% ✗ List()
+```
+
+Finally:
+
+* `label(name)` is an alias for `classify(name, _ => true)`, and
+* `collect` is an alias for `labal` using the value's `toString` as the classification (label name)
+
+
 ## State
 
 For a separate tutorial on state-based property testing please continue

example/shared/src/main/scala/hedgehog/examples/ReverseTest.scala

@@ -13,5 +13,7 @@ object ReverseTest extends Properties {
   def testReverse: Property =
     for {
       xs <- Gen.alpha.list(Range.linear(0, 100)).forAll
+         .cover(50, "empty", _.isEmpty)
+         .cover(50, "nonempty", _.nonEmpty)
     } yield xs.reverse.reverse ==== xs
 }

runner/shared/src/main/scala/hedgehog/runner/Properties.scala

@@ -36,6 +36,9 @@ class Test(
 
   def noShrinking: Test =
     config(_.copy(shrinkLimit = ShrinkLimit(0)))
+
+  def withExamples: Test =
+    config(_.copy(withExamples = WithExamples.WithExamples))
 }
 
 object Test {
@@ -62,7 +65,7 @@ object Test {
       }
     }
 
-    val coverage = renderCoverage(report.coverage, report.tests)
+    val coverage = renderCoverage(report.coverage, report.tests, report.examples)
     report.status match {
       case Failed(shrinks, log) =>
         render(false, s"Falsified after ${report.tests.value} passed tests", log.map(renderLog) ++ coverage)
@@ -86,7 +89,7 @@ object Test {
         sw.toString
     }
 
-  def renderCoverage(coverage: Coverage[CoverCount], tests: SuccessCount): List[String] =
+  def renderCoverage(coverage: Coverage[CoverCount], tests: SuccessCount, examples: Examples): List[String] =
     coverage.labels.values.toList
       .sortBy(_.annotation.percentage(tests).toDouble.toInt * -1)
       .map(l => {
@@ -97,6 +100,20 @@ object Test {
             l.minimum.toDouble.toInt.toString + "%"
           , if (Label.covered(l, tests)) "✓" else "✗"
           ) else Nil
+        , renderExample(examples, l.name)
         ).flatten.mkString(" ")
       })
+
+  def renderExample(examples: Examples, name: LabelName): List[String] =
+    examples.examples.getOrElse(name, Nil).map(renderLog) match {
+      case Nil =>
+        Nil
+      case x :: Nil =>
+        if (x == name.render) // i.e. `.collect`
+          Nil
+        else
+          List(x)
+      case xs =>
+        List(xs.mkString)
+    }
 }

test/shared/src/test/scala/hedgehog/GenTest.scala

@@ -70,12 +70,12 @@ object GenTest extends Properties {
 
   def testFromSomeSome: Result = {
     val r = Property.checkRandom(PropertyConfig.default, Gen.fromSome(Gen.constant(Result.success).option).forAll)
-    r ==== Report(SuccessCount(100), DiscardCount(0), Coverage.empty, OK)
+    r ==== Report(SuccessCount(100), DiscardCount(0), Coverage.empty, Examples.empty, OK)
   }
 
   def testFromSomeNone: Result = {
     val r = Property.checkRandom(PropertyConfig.default, Gen.fromSome(Gen.constant(Option.empty[Result])).forAll)
-    r ==== Report(SuccessCount(0), DiscardCount(100), Coverage.empty, GaveUp)
+    r ==== Report(SuccessCount(0), DiscardCount(100), Coverage.empty, Examples.empty, GaveUp)
   }
 
   def testApplicative: Result = {

test/shared/src/test/scala/hedgehog/LabelledExamplesTest.scala

@@ -0,0 +1,34 @@
+package hedgehog
+
+import hedgehog.core._
+import hedgehog.runner._
+
+object LabelledExamplesTest extends Properties {
+
+  def tests: List[Test] =
+    List(
+      property("testLabelledExamples", testLabelledExamples)
+    , property("testProperty", prop)
+    )
+
+  def prop: Property =
+    for {
+      _ <- Gen.int(Range.linear(0, 10)).list(Range.linear(0, 10)).forAll
+        .classify("empty", _.isEmpty)
+        .classify("nonempty", _.nonEmpty)
+    } yield Result.success
+
+  def testLabelledExamples: Property = {
+    for {
+      examples <- Gen.generate { (size, seed) =>
+        val config = PropertyConfig.default
+        val labelledExamples = Property.report(config, Some(size), seed, prop)
+        Seed(seed.seed.next) -> labelledExamples.examples
+      }.forAll
+    } yield
+      examples ==== Examples(Map(
+        LabelName("empty") -> List(Info("List()"))
+      , LabelName("nonempty") -> List(Info("List(0)"))
+      ))
+  }
+}