From c085e81d02b8a747fdea8f3bff4075e072f89242 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 20 Jul 2012 12:01:15 -0600 Subject: [PATCH 01/53] In case of failure, print the whole object which caused it --- qc/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 3713be2..f9d611f 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) 2009-2011, Dan Bravender import random -import os +import os, sys import functools def integers(low=0, high=100): @@ -90,7 +90,13 @@ def wrapped(*inargs, **inkwargs): from pprint import pprint pprint(random_kwargs) random_kwargs.update(**inkwargs) - f(*inargs, **random_kwargs) + try: + f(*inargs, **random_kwargs) + except Exception, e: + if sys.version_info[0] < 3: + raise e.__class__("%s, caused a failure\n %s" % (random_kwargs, e)), None, sys.exc_traceback + else: + raise e.__class__("{} caused a failure\n".format(random_kwargs)).with_traceback(e.__traceback__) return wrapped return wrap forall.verbose = False # if enabled will print out the random test cases From b4af8326233791ddd341bd85aca9a790235d2bfa Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 20 Jul 2012 13:38:25 -0600 Subject: [PATCH 02/53] Initialized the random seed generator, either at random, or with user supplied seed. Printing the user supplied seed in case of failures --- qc/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index f9d611f..a0b2358 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -79,7 +79,13 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): setattr(obj, k, v.next()) yield obj -def forall(tries=100, **kwargs): +def forall(tries=100, seed=None, **kwargs): + if seed is None: + try: + seed = hash(os.urandom(16)) + except NotImplementedError, e: + seed = random.random() + random.seed(seed) def wrap(f): @functools.wraps(f) def wrapped(*inargs, **inkwargs): @@ -94,9 +100,11 @@ def wrapped(*inargs, **inkwargs): f(*inargs, **random_kwargs) except Exception, e: if sys.version_info[0] < 3: - raise e.__class__("%s, caused a failure\n %s" % (random_kwargs, e)), None, sys.exc_traceback + raise e.__class__("%s, generated with seed %s, caused a FAIL\n%s" % + (random_kwargs, seed, e)), None, sys.exc_traceback else: - raise e.__class__("{} caused a failure\n".format(random_kwargs)).with_traceback(e.__traceback__) + raise e.__class__("{0}, generated with seed {1}, caused a FAIL\n".format( + random_kwargs, seed)).with_traceback(e.__traceback__) return wrapped return wrap forall.verbose = False # if enabled will print out the random test cases From 2fd67568c583ffdd6cf9e3cafb6eebcbc4d2c21e Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 20 Jul 2012 14:33:23 -0600 Subject: [PATCH 03/53] first, ugly and rudimentary, shrinking attempt --- qc/__init__.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index a0b2358..bb3c573 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -79,7 +79,32 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): setattr(obj, k, v.next()) yield obj -def forall(tries=100, seed=None, **kwargs): +def shrink(something): + if isinstance(something, list): + l = len(something)/2 + yield something[:l] + yield something[l:] + else: + yield [] + +def call_and_shrink(f, tryshrink, seed, *inargs, **random_kwargs): + try: + f(*inargs, **random_kwargs) + except Exception, e: + if tryshrink: + for k in random_kwargs: + for s in shrink(random_kwargs[k]): + shrinked_kwargs = random_kwargs.copy() + shrinked_kwargs[k] = s + call_and_shrink(f, tryshrink, seed, *inargs, **shrinked_kwargs) + if sys.version_info[0] < 3: + raise e.__class__("%s, generated with seed %s, caused a FAIL\n%s" % + (random_kwargs, seed, e)), None, sys.exc_traceback + else: + raise e.__class__("{0}, generated with seed {1}, caused a FAIL\n".format( + random_kwargs, seed)).with_traceback(e.__traceback__) + +def forall(tries=100, shrink=True, seed=None, **kwargs): if seed is None: try: seed = hash(os.urandom(16)) @@ -96,15 +121,7 @@ def wrapped(*inargs, **inkwargs): from pprint import pprint pprint(random_kwargs) random_kwargs.update(**inkwargs) - try: - f(*inargs, **random_kwargs) - except Exception, e: - if sys.version_info[0] < 3: - raise e.__class__("%s, generated with seed %s, caused a FAIL\n%s" % - (random_kwargs, seed, e)), None, sys.exc_traceback - else: - raise e.__class__("{0}, generated with seed {1}, caused a FAIL\n".format( - random_kwargs, seed)).with_traceback(e.__traceback__) + call_and_shrink(f, shrink, seed, *inargs, **random_kwargs) return wrapped return wrap forall.verbose = False # if enabled will print out the random test cases From 4f32ee8624eeabf7d172ae14fb0bf6fa177300cb Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 26 Jul 2012 12:00:29 -0600 Subject: [PATCH 04/53] Shrinking for a generic Exception would case ad infinitum recursion, unless the exception is AssertionError. --- qc/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qc/__init__.py b/qc/__init__.py index bb3c573..31529e1 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -90,7 +90,7 @@ def shrink(something): def call_and_shrink(f, tryshrink, seed, *inargs, **random_kwargs): try: f(*inargs, **random_kwargs) - except Exception, e: + except AssertionError, e: # shrink only when there is AssertionErrors, in other cases is ad infinitum recursion if tryshrink: for k in random_kwargs: for s in shrink(random_kwargs[k]): From 88769ff676d3a77601afb885ee57553af44b4bbb Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 26 Jul 2012 12:47:36 -0600 Subject: [PATCH 05/53] Report of what happened. --- qc/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qc/__init__.py b/qc/__init__.py index 31529e1..d6bc084 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -122,8 +122,12 @@ def wrapped(*inargs, **inkwargs): pprint(random_kwargs) random_kwargs.update(**inkwargs) call_and_shrink(f, shrink, seed, *inargs, **random_kwargs) + if forall.printsummary: + from pprint import pprint + pprint(f.__name__+": passed "+str(tries)+" tests [OK]") return wrapped return wrap +forall.printsummary = False forall.verbose = False # if enabled will print out the random test cases __all__ = ['integers', 'floats', 'lists', 'tuples', 'unicodes', 'characters', 'objects', 'forall'] From 8147d4089ef24281a7bbbadcde2b4c784d6a1846 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Wed, 1 Aug 2012 16:14:18 -0600 Subject: [PATCH 06/53] Added tests for shrink function, and implemented what was needed to make them pass - making it more pythonic (permission vs forgiveness) --- qc/__init__.py | 11 ++++++++--- tests/test_qc.py | 24 +++++++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index d6bc084..9ee44dc 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -80,12 +80,17 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): yield obj def shrink(something): - if isinstance(something, list): + try: l = len(something)/2 yield something[:l] yield something[l:] - else: - yield [] + except TypeError: + pass + try: + if abs(something) >= 2: + yield something/2 + except TypeError: + pass def call_and_shrink(f, tryshrink, seed, *inargs, **random_kwargs): try: diff --git a/tests/test_qc.py b/tests/test_qc.py index c3f23e6..a99f651 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,4 +1,4 @@ -from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall +from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, shrink @forall(tries=10, i=integers()) def test_integers(i): @@ -93,6 +93,28 @@ def test_dicts_size(d): assert type(x) == unicode assert type(y) == list +@forall(full_l=lists()) +def test_shrink_lists(full_l): + for sub_l in shrink(full_l): + assert len(sub_l) <= len(full_l)/2 + 1 + +@forall(full_i=integers(low=-100)) +def test_shrink_integers(full_i): + for i in shrink(full_i): + assert abs(i) <= abs(full_i)/2 + 1 + assert cmp(i,0) == cmp(full_i, 0) # shrink shall not change sign + assert isinstance(i, int) + +@forall(full_f=floats(low=-5.0, high=5.0)) +def test_shrink_floats(full_f): + for f in shrink(full_f): + if abs(full_f) > 1: + assert abs(f) <= abs(full_f)/2 + 1 + else: + assert abs(f) >= abs(full_f)/2 + 1 + assert cmp(f,0) == cmp(full_f, 0) # shrink shall not change sign + assert isinstance(f, float) + @forall(tries=10, i=integers(low=0, high=10)) def each_integer_from_0_to_10(i, target_low, target_high): assert i >= target_low and i<= target_high From 0a3243b4f331b7f5ca7584e56dd3d10fd870b049 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 10:03:41 -0600 Subject: [PATCH 07/53] A decent README --- README | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/README b/README index 32a2077..cc42d6a 100644 --- a/README +++ b/README @@ -1,6 +1,77 @@ +============ +Introduction +============ +This framework does Random and Combinatorial Testing. Before you leave +horrified, please have a look at the following videos from professor +John Regehr, University of Utah (less than 15min total): +http://www.youtube.com/watch?v=cwhC19Fa_84 - introduction to random testing +http://www.youtube.com/watch?v=PrJZ6144eeM - why random testing is good (1) +http://www.youtube.com/watch?v=btlfWwyzSXQ - why random testing is good (2) +http://www.youtube.com/watch?v=iw6BtJxPT8A - why random testing is good (3) +http://www.youtube.com/watch?v=QrLtkSdMDgw - why random testing is good (4) + +Random testing, is not just randomly feeding your software a random +stream of bytes. It requires more thoughts. The Udacity course on +testing (http://www.udacity.com/overview/Course/cs258/CourseRev/1) has +units 3 and 4 entirely dedicated to Random Testing, describing many +things you need to know about Random Testing: how to create valid, +good, random test cases (3.25-26 and 4.5-15), mutators (3.29), oracles +(3.30-34), test case reduction (4.5-6), tradeoffs (4.18-20), and more. +Random Testing is also mentioned elsewhere in the class (and expected +to be know in the final exam). Very worth watching (the introductory +videos linked above are from this class) + +Ok, so you're convinced that Random Testing may be worth exploring. +Why this framework? It is a python framework inspired by Haskell's +http://en.wikipedia.org/wiki/QuickCheck + +It does many things for you, including automatic test case reduction +(currently only with bisection techniques, and thus in logarithmic time). + ============ Installation ============ +The easy, system-wide way (requires administrative privileges): + sudo easy_install pip +sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc +(I hope to get it merged upstream, so you will install with sudo pip install -e git://github.com/dbravender/qc.git#egg=qc +instead) + +If you don't want to pollute the whole system with this library, +or would like to test it before committing, or simply don't have +root access on your machine, just copy the qc directory and its +content (a mere __init__.py file) into the location of your choice. +To have qc available to your programs you will have to set the +PYTHONPATH environmental variable or have the qc directory as +a subdirectory of the tree where you are running. + +============ + Examples +============ + +TBD + +============ + TODO +============ + +* provide the option to not stop in case of failures, and instead +logging and continue (or better handled by the underlying test +framework?? see http://stackoverflow.com/questions/4732827/ for a +discussion) + +* improve the current test case reduction from bisection only to +delta-debugging (see http://www.st.cs.uni-saarland.de/dd/ +http://delta.tigris.org/ +http://classes.soe.ucsc.edu/cmps290g/Winter04/lectures/flanagan-290g-8.pdf +for details) + +* integration with git-bisect (maybe) + +* better edge cases coverage: e.g. for integers use a range of +[-sys.maxint-1, sys.maxint] instead of [0, 100] or at the very +least some negative cases... Note that the range needs to be different +for long integers. From 8fd011b3dba67684130265fe6d12e9e513aa8e6b Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 10:46:13 -0600 Subject: [PATCH 08/53] If a length of subscriptable object is zero the previous shrink implementation was going to recurse forever. --- qc/__init__.py | 2 ++ tests/test_qc.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/qc/__init__.py b/qc/__init__.py index 9ee44dc..aa1f84e 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -81,6 +81,8 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): def shrink(something): try: + if len(something) == 0: # never shrink a zero-len object, since it + return # will lead to infinite recursion l = len(something)/2 yield something[:l] yield something[l:] diff --git a/tests/test_qc.py b/tests/test_qc.py index a99f651..01c0a54 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -93,6 +93,12 @@ def test_dicts_size(d): assert type(x) == unicode assert type(y) == list +def test_shrink_empty_list(): + empty_list_has_been_shrunk = False + for x in shrink([]): + empty_list_has_been_shrunk = True + assert empty_list_has_been_shrunk == False, "Empty lists must not be shrunk" + @forall(full_l=lists()) def test_shrink_lists(full_l): for sub_l in shrink(full_l): From a0926d400f361dd23e36fa11db9317947c18d73e Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 11:23:44 -0600 Subject: [PATCH 09/53] first unittest example --- README | 8 ++++++-- example1_unittest.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 example1_unittest.py diff --git a/README b/README index cc42d6a..6437e52 100644 --- a/README +++ b/README @@ -23,7 +23,8 @@ videos linked above are from this class) Ok, so you're convinced that Random Testing may be worth exploring. Why this framework? It is a python framework inspired by Haskell's -http://en.wikipedia.org/wiki/QuickCheck +http://hackage.haskell.org/package/QuickCheck and Scala's +https://github.com/rickynils/scalacheck It does many things for you, including automatic test case reduction (currently only with bisection techniques, and thus in logarithmic time). @@ -52,7 +53,10 @@ a subdirectory of the tree where you are running. Examples ============ -TBD +example1_unittest.py provides a simple example (borrowed from scalacheck) + on how to use this framework with python native + PyUnit framework (aka unittest module). + Just run it with "python example1_unittest.py" ============ TODO diff --git a/example1_unittest.py b/example1_unittest.py new file mode 100644 index 0000000..1fafda5 --- /dev/null +++ b/example1_unittest.py @@ -0,0 +1,30 @@ +from unittest import TestCase, main +from qc import forall, unicodes + +# This example is adapted from Scala's +# https://github.com/rickynils/scalacheck +# and we are pretending to test the string +# concatenation, slicing and the len builtin + +class TestString(TestCase): + @forall(tries=2000, a=unicodes(), b=unicodes()) + def testStartswith(self, a, b): + concat = a + b + self.assertTrue(concat.startswith(a)) + + @forall(tries=10, a=unicodes(), b=unicodes()) + def testConcatenation(self, a, b): + concat = a + b + # the following is meant to fail as an example of a failure + self.assertTrue(len(concat) > len(a)) + self.assertTrue(len(concat) > len(b)) + + @forall(a=unicodes(), b=unicodes(), c=unicodes()) + def testSubstring(self, a, b, c): + concat = a + b + c + start = len(a) + stop = len(a) + len(b) + self.assertEqual(concat[start: stop], b) + +main() + From fc3901a53b4f3e40397fc53b237faafe8e48480b Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 11:25:07 -0600 Subject: [PATCH 10/53] Since there will be at least 4 examples, a directory has been created to hold them --- example1_unittest.py => examples/example1_unittest.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename example1_unittest.py => examples/example1_unittest.py (100%) diff --git a/example1_unittest.py b/examples/example1_unittest.py similarity index 100% rename from example1_unittest.py rename to examples/example1_unittest.py From a5323fc33d6c5850cc3ac0f59e901d8f7390c14a Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 11:35:58 -0600 Subject: [PATCH 11/53] Ported the first example to nose --- README | 11 +++++++---- examples/example1_nose.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 examples/example1_nose.py diff --git a/README b/README index 6437e52..dc41c3c 100644 --- a/README +++ b/README @@ -53,10 +53,13 @@ a subdirectory of the tree where you are running. Examples ============ -example1_unittest.py provides a simple example (borrowed from scalacheck) - on how to use this framework with python native - PyUnit framework (aka unittest module). - Just run it with "python example1_unittest.py" +examples/example1_unittest.py and examples/example1_nose.py + + These examples provide a simple example (borrowed from scalacheck) + on how to use this framework with python native PyUnit framework + (aka unittest module) and with the popular nose framework. + Just run "python examples/example1_unittest.py" + or "nosetests examples/example1_nose.py' ============ TODO diff --git a/examples/example1_nose.py b/examples/example1_nose.py new file mode 100644 index 0000000..7bdf314 --- /dev/null +++ b/examples/example1_nose.py @@ -0,0 +1,27 @@ +from qc import forall, unicodes + +# This example is adapted from Scala's +# https://github.com/rickynils/scalacheck +# and we are pretending to test the string +# concatenation, slicing and the len builtin + +@forall(tries=2000, a=unicodes(), b=unicodes()) +def testStartswith(a, b): + concat = a + b + assert concat.startswith(a) + +@forall(tries=10, a=unicodes(), b=unicodes()) +def testConcatenation(a, b): + concat = a + b + # the following is meant to fail as an example of a failure + assert len(concat) > len(a) + assert len(concat) > len(b) + +@forall(a=unicodes(), b=unicodes(), c=unicodes()) +def testSubstring(a, b, c): + concat = a + b + c + start = len(a) + stop = len(a) + len(b) + assert concat[start: stop] == b + + From 4bd2bd09ad4f1123cf8b74c70b456a123d15aba6 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Aug 2012 12:00:44 -0600 Subject: [PATCH 12/53] Clarification on test case reduction in the README --- README | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index dc41c3c..6fe9b55 100644 --- a/README +++ b/README @@ -27,7 +27,9 @@ http://hackage.haskell.org/package/QuickCheck and Scala's https://github.com/rickynils/scalacheck It does many things for you, including automatic test case reduction -(currently only with bisection techniques, and thus in logarithmic time). +(currently only with bisection techniques, and thus it reduce the test case +in logarithmic time, which means it is very fast althugh it may reduce to +a case potentially larger than the absolute smallest one) ============ Installation From dc41701dd6cf6e216923a0d9f7d503b805f45e71 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 10:18:54 -0600 Subject: [PATCH 13/53] Example 3: simple one on how to have qc generate your own custom random objects --- README | 6 +++++- examples/ex3_airplane.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 examples/ex3_airplane.py diff --git a/README b/README index 6fe9b55..b42a1c8 100644 --- a/README +++ b/README @@ -28,7 +28,7 @@ https://github.com/rickynils/scalacheck It does many things for you, including automatic test case reduction (currently only with bisection techniques, and thus it reduce the test case -in logarithmic time, which means it is very fast althugh it may reduce to +in logarithmic time, which means it is very fast although it may reduce to a case potentially larger than the absolute smallest one) ============ @@ -63,6 +63,10 @@ examples/example1_unittest.py and examples/example1_nose.py Just run "python examples/example1_unittest.py" or "nosetests examples/example1_nose.py' +examples/ex3_airplane.py + Simple example on how to have qc generate your own custom (random) + objects and how to use them in practice + ============ TODO ============ diff --git a/examples/ex3_airplane.py b/examples/ex3_airplane.py new file mode 100644 index 0000000..bc44fa8 --- /dev/null +++ b/examples/ex3_airplane.py @@ -0,0 +1,41 @@ +class Location(object): + def __init__(self, lat, lon, height): + self.lat=lat + self.lon=lon + self.h=height + def pr(self): + return "LAT: " + str(self.lat) + " LON: " + str(self.lon) + " H: " + str(self.h) + +class Plane(object): # class under test + def __init__(self): + self.location = Location(46.22, -112.1, 4968) + def fly_to(self, loc): + if (-75 < loc.lon < -70 and + 20 < loc.lat < 30 and + loc.h < 6000): + raise Exception("Plane has disappeared in the Bermuda triangle") + else: + self.location = loc + def has_gas(self): + return True + +from unittest import TestCase, main +from qc import forall, floats + +def locations(lat = floats(-90,+90), + lon = floats(-180, +180), + height = floats(0, 30000)): + while True: + yield Location(lat.next(), lon.next(), height.next()) + +class TestPlane(TestCase): + def setUp(self): + self.cessna172 = Plane() + + @forall(tries=50000, l=locations()) + def testFly(self, l): + self.cessna172.fly_to(l) + self.assertTrue(self.cessna172.has_gas()) + +main() + From 826c307314b28315ddfe1543f7510be74b4ca0db Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 10:28:42 -0600 Subject: [PATCH 14/53] A minimal code clean-up and a bunch of comments --- examples/ex3_airplane.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/examples/ex3_airplane.py b/examples/ex3_airplane.py index bc44fa8..b35c7c3 100644 --- a/examples/ex3_airplane.py +++ b/examples/ex3_airplane.py @@ -1,12 +1,14 @@ +# Location is a (lame) class needed by the +# class under test. In real life it will be +# a non-lame object you really need class Location(object): def __init__(self, lat, lon, height): self.lat=lat self.lon=lon self.h=height - def pr(self): - return "LAT: " + str(self.lat) + " LON: " + str(self.lon) + " H: " + str(self.h) -class Plane(object): # class under test +# This is the class under test +class Plane(object): def __init__(self): self.location = Location(46.22, -112.1, 4968) def fly_to(self, loc): @@ -22,12 +24,29 @@ def has_gas(self): from unittest import TestCase, main from qc import forall, floats +# This is the qc-related auxiliary method +# used to create random objects. In this +# simple example it is overkill, but the +# purpose here is just to show how it is +# done: in real life you may have several +# testing methods requiring random locations +# and defining the method this way will +# allow @forall to be able to inject +# Location's everywhere you need them def locations(lat = floats(-90,+90), lon = floats(-180, +180), height = floats(0, 30000)): while True: yield Location(lat.next(), lon.next(), height.next()) + +# This is the PyUnit test case +# In this example we are just +# flying the plane to random locations +# and assert that it does not run out +# of gas. In real life you would assert +# that it *does* run out of gas, but it +# does *not* fall apart in other ways. class TestPlane(TestCase): def setUp(self): self.cessna172 = Plane() From 950249d1e1cb6c6497f86d9288f23fad882376f6 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 11:46:22 -0600 Subject: [PATCH 15/53] In case of single-element collections (e.g. lists), there was a bug that caused repeated (infinite) recursion in some circumstances. This patch avoid it. --- qc/__init__.py | 4 ++++ tests/test_qc.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/qc/__init__.py b/qc/__init__.py index aa1f84e..7865f91 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -83,6 +83,10 @@ def shrink(something): try: if len(something) == 0: # never shrink a zero-len object, since it return # will lead to infinite recursion + if len(something) == 1: # if single-object collection + yield something[:0] # try the empty collection + return # if it works, no need to try the single-object + # collection again l = len(something)/2 yield something[:l] yield something[l:] diff --git a/tests/test_qc.py b/tests/test_qc.py index 01c0a54..20b40d8 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -99,6 +99,14 @@ def test_shrink_empty_list(): empty_list_has_been_shrunk = True assert empty_list_has_been_shrunk == False, "Empty lists must not be shrunk" +def test_shrink_single_element_list(): + repeated = False + l = [0] + for x in shrink(l): + if x == l: + repeated = True + assert repeated == False, "Shrink must not repeat things already seen" + @forall(full_l=lists()) def test_shrink_lists(full_l): for sub_l in shrink(full_l): From 1aaa73c713fb5489fd45dbe0837a73b916fd2de0 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 14:08:58 -0600 Subject: [PATCH 16/53] The bookcase example draft --- README | 3 +++ examples/ex4_bookcase.py | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 examples/ex4_bookcase.py diff --git a/README b/README index b42a1c8..489c2ac 100644 --- a/README +++ b/README @@ -67,6 +67,9 @@ examples/ex3_airplane.py Simple example on how to have qc generate your own custom (random) objects and how to use them in practice +examples/ex4_bookcase.py + A more elaborate example, for now only drafted + ============ TODO ============ diff --git a/examples/ex4_bookcase.py b/examples/ex4_bookcase.py new file mode 100644 index 0000000..15544ff --- /dev/null +++ b/examples/ex4_bookcase.py @@ -0,0 +1,42 @@ +# This is at the same time the class under test, +# and the class that you need to randomly generate +# For simplicity a book is uniquely identified +# by a just a positive integer +class Bookcase(object): + def __init__(self, num_shelves, books): + self.num_shelves = num_shelves + self.books = books + def put(self, book): + if not self.full(): + self.books.append(book) + def take(self, book): + if book in self.books: + self.books.remove(book) + if 13 in self.books: # artificially introduced bug + return 13 + else: + return book + def full(self): + return False # infinite bookcase + def __repr__(self): + return "Bookcase with " + str(self.num_shelves) + " shelves and containing books: " + self.books.__repr__() + +from unittest import TestCase, main +from qc import forall, integers, lists + +# This is the qc-related auxiliary method +# used to create random objects (bookcases) +def bookcases(nshelves = integers(1,10), book_set=lists()): + while True: + yield Bookcase(nshelves.next(), book_set.next()) + +# This is the PyUnit test case +class TestBookcase(TestCase): + @forall(bc=bookcases(), book=integers()) + def testPutAndTake_noshrink(self, bc, book): + bc.put(book) + self.assertEqual(book, bc.take(book)) + +main() + + From c1f24010dc7039e4d64445895f7c07f90a2da969 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 15:47:34 -0600 Subject: [PATCH 17/53] Now it is possible to specify a custom shrink function --- qc/__init__.py | 12 ++++++------ tests/test_qc.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 7865f91..35f903b 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -79,7 +79,7 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): setattr(obj, k, v.next()) yield obj -def shrink(something): +def qc_shrink(something): try: if len(something) == 0: # never shrink a zero-len object, since it return # will lead to infinite recursion @@ -98,16 +98,16 @@ def shrink(something): except TypeError: pass -def call_and_shrink(f, tryshrink, seed, *inargs, **random_kwargs): +def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs): try: f(*inargs, **random_kwargs) except AssertionError, e: # shrink only when there is AssertionErrors, in other cases is ad infinitum recursion if tryshrink: for k in random_kwargs: - for s in shrink(random_kwargs[k]): + for s in custom_shrink(random_kwargs[k]): shrinked_kwargs = random_kwargs.copy() shrinked_kwargs[k] = s - call_and_shrink(f, tryshrink, seed, *inargs, **shrinked_kwargs) + call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **shrinked_kwargs) if sys.version_info[0] < 3: raise e.__class__("%s, generated with seed %s, caused a FAIL\n%s" % (random_kwargs, seed, e)), None, sys.exc_traceback @@ -115,7 +115,7 @@ def call_and_shrink(f, tryshrink, seed, *inargs, **random_kwargs): raise e.__class__("{0}, generated with seed {1}, caused a FAIL\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) -def forall(tries=100, shrink=True, seed=None, **kwargs): +def forall(tries=100, shrink=True, seed=None, custom_shrink=qc_shrink, **kwargs): if seed is None: try: seed = hash(os.urandom(16)) @@ -132,7 +132,7 @@ def wrapped(*inargs, **inkwargs): from pprint import pprint pprint(random_kwargs) random_kwargs.update(**inkwargs) - call_and_shrink(f, shrink, seed, *inargs, **random_kwargs) + call_and_shrink(f, shrink, seed, custom_shrink, *inargs, **random_kwargs) if forall.printsummary: from pprint import pprint pprint(f.__name__+": passed "+str(tries)+" tests [OK]") diff --git a/tests/test_qc.py b/tests/test_qc.py index 20b40d8..02cecb3 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,4 +1,4 @@ -from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, shrink +from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink @forall(tries=10, i=integers()) def test_integers(i): @@ -95,33 +95,33 @@ def test_dicts_size(d): def test_shrink_empty_list(): empty_list_has_been_shrunk = False - for x in shrink([]): + for x in qc_shrink([]): empty_list_has_been_shrunk = True assert empty_list_has_been_shrunk == False, "Empty lists must not be shrunk" def test_shrink_single_element_list(): repeated = False l = [0] - for x in shrink(l): + for x in qc_shrink(l): if x == l: repeated = True assert repeated == False, "Shrink must not repeat things already seen" @forall(full_l=lists()) def test_shrink_lists(full_l): - for sub_l in shrink(full_l): + for sub_l in qc_shrink(full_l): assert len(sub_l) <= len(full_l)/2 + 1 @forall(full_i=integers(low=-100)) def test_shrink_integers(full_i): - for i in shrink(full_i): + for i in qc_shrink(full_i): assert abs(i) <= abs(full_i)/2 + 1 assert cmp(i,0) == cmp(full_i, 0) # shrink shall not change sign assert isinstance(i, int) @forall(full_f=floats(low=-5.0, high=5.0)) def test_shrink_floats(full_f): - for f in shrink(full_f): + for f in qc_shrink(full_f): if abs(full_f) > 1: assert abs(f) <= abs(full_f)/2 + 1 else: From f594e5c3fed0b0ad01b6565a23ad30316d4f9dbf Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 24 Aug 2012 16:08:23 -0600 Subject: [PATCH 18/53] Conclusions on the bookcase example --- README | 5 ++++- examples/ex4_bookcase.py | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/README b/README index 489c2ac..7253382 100644 --- a/README +++ b/README @@ -68,7 +68,10 @@ examples/ex3_airplane.py objects and how to use them in practice examples/ex4_bookcase.py - A more elaborate example, for now only drafted + A more elaborate example, showing the power of automatic shrinking + and showing how to write your own shrinker. In this example you can + see how qc automagically finds the root cause of the bug, compare + the output with or without shrinking! ============ TODO diff --git a/examples/ex4_bookcase.py b/examples/ex4_bookcase.py index 15544ff..bf58e4a 100644 --- a/examples/ex4_bookcase.py +++ b/examples/ex4_bookcase.py @@ -17,12 +17,12 @@ def take(self, book): else: return book def full(self): - return False # infinite bookcase + return False # This bookcase has infinite capacity def __repr__(self): return "Bookcase with " + str(self.num_shelves) + " shelves and containing books: " + self.books.__repr__() from unittest import TestCase, main -from qc import forall, integers, lists +from qc import forall, integers, lists, qc_shrink # This is the qc-related auxiliary method # used to create random objects (bookcases) @@ -30,6 +30,21 @@ def bookcases(nshelves = integers(1,10), book_set=lists()): while True: yield Bookcase(nshelves.next(), book_set.next()) +# This is the custom shrinking function we have to provide +# to qc. It has to decide the logic by which we want the +# test case to be shrunk. In this case, we just shrink +# the list of books, using qc default's shrink function. +# Note that we are not shrinking the number of shelves. +# This is the common pattern to create custom shrinker: +# decide what has to change and what has not. Call +# qc's shrinker on the relevant objects (if they are +# supported, otherwise write your own shrinker for those) +def shrink_bookcase(bookcase): + if isinstance(bookcase, Bookcase): + for shrinked_books in qc_shrink(bookcase.books): + yield Bookcase(bookcase.num_shelves, shrinked_books) + # else we do not shrink + # This is the PyUnit test case class TestBookcase(TestCase): @forall(bc=bookcases(), book=integers()) @@ -37,6 +52,11 @@ def testPutAndTake_noshrink(self, bc, book): bc.put(book) self.assertEqual(book, bc.take(book)) + @forall(bc=bookcases(), book=integers(), custom_shrink=shrink_bookcase) + def testPutAndTake_shrink(self, bc, book): + bc.put(book) + self.assertEqual(book, bc.take(book)) + main() From 65fda0e22c10258cc788207076b2239787c4b093 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 27 Aug 2012 09:52:21 -0600 Subject: [PATCH 19/53] Simplified the name of two examples, and updated the README accordingly (added also a TODO item in README) --- README | 7 ++++++- examples/{example1_nose.py => ex1_nose.py} | 0 examples/{example1_unittest.py => ex1_unittest.py} | 0 3 files changed, 6 insertions(+), 1 deletion(-) rename examples/{example1_nose.py => ex1_nose.py} (100%) rename examples/{example1_unittest.py => ex1_unittest.py} (100%) diff --git a/README b/README index 7253382..511a0df 100644 --- a/README +++ b/README @@ -55,7 +55,8 @@ a subdirectory of the tree where you are running. Examples ============ -examples/example1_unittest.py and examples/example1_nose.py +examples/ex1_unittest.py and +examples/ex1_nose.py These examples provide a simple example (borrowed from scalacheck) on how to use this framework with python native PyUnit framework @@ -77,6 +78,10 @@ examples/ex4_bookcase.py TODO ============ +* print a better summary, e.g. including the total number of tries +per method and other infos (need to investigate how to do it cleanly +and in a way that works both in unittest and nose) + * provide the option to not stop in case of failures, and instead logging and continue (or better handled by the underlying test framework?? see http://stackoverflow.com/questions/4732827/ for a diff --git a/examples/example1_nose.py b/examples/ex1_nose.py similarity index 100% rename from examples/example1_nose.py rename to examples/ex1_nose.py diff --git a/examples/example1_unittest.py b/examples/ex1_unittest.py similarity index 100% rename from examples/example1_unittest.py rename to examples/ex1_unittest.py From 4c66718337e037658477726bf0f14e308bf2c267 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 27 Aug 2012 10:20:16 -0600 Subject: [PATCH 20/53] Added some utterance in the README about the possibility of maximum recursion depth exceeded in the stack for too much recursion. --- README | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README b/README index 511a0df..eb9ef47 100644 --- a/README +++ b/README @@ -74,6 +74,28 @@ examples/ex4_bookcase.py see how qc automagically finds the root cause of the bug, compare the output with or without shrinking! +============ + Known bugs +============ + +At present there is not any known bug. If you discover one, please let +us know. One common problem when using automatic shrinking is running +out of stack space in the recursion process (the shrink algorithm +call itself several times to produce a smaller test case). This may +happen either if there is a bug in qc itself, or if there is a problem +in your test code. You will see an error like: + +RuntimeError: maximum recursion depth exceeded while calling a Python object + +with a stack trace that shows the recursion tree of the shrinking +method calling itself. To understand what is happening, it is usually +useful to add the shrink=False option to the @forall decorator of the +affected test method. In very rare cases it may be necessary to increase +the stack depth with a call to sys.setrecursionlimit(NEWDEPTH), but +do not do it until you understand that it is really the case for your +test. More often than not, there will be a bug in your code or in qc. +Please report the latter. + ============ TODO ============ From 96351777171d5d601b3788f04a0a0db961518c75 Mon Sep 17 00:00:00 2001 From: Sean Fisk Date: Wed, 5 Jun 2013 14:42:51 -0600 Subject: [PATCH 21/53] Add to README on running examples with pytest. --- README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README b/README index eb9ef47..511f81d 100644 --- a/README +++ b/README @@ -62,7 +62,8 @@ examples/ex1_nose.py on how to use this framework with python native PyUnit framework (aka unittest module) and with the popular nose framework. Just run "python examples/example1_unittest.py" - or "nosetests examples/example1_nose.py' + or "nosetests examples/example1_nose.py". Pytest can also run + the nose tests using "py.test examples/example1_nose.py". examples/ex3_airplane.py Simple example on how to have qc generate your own custom (random) From a66a0d63c5aae9b83388a1d8834300ec69d6dcac Mon Sep 17 00:00:00 2001 From: Sean Fisk Date: Wed, 5 Jun 2013 15:20:56 -0600 Subject: [PATCH 22/53] Reformat README as reStructuredText. --- README | 124 -------------------------------------------------- README.rst | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 124 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/README b/README deleted file mode 100644 index 511f81d..0000000 --- a/README +++ /dev/null @@ -1,124 +0,0 @@ -============ -Introduction -============ -This framework does Random and Combinatorial Testing. Before you leave -horrified, please have a look at the following videos from professor -John Regehr, University of Utah (less than 15min total): -http://www.youtube.com/watch?v=cwhC19Fa_84 - introduction to random testing -http://www.youtube.com/watch?v=PrJZ6144eeM - why random testing is good (1) -http://www.youtube.com/watch?v=btlfWwyzSXQ - why random testing is good (2) -http://www.youtube.com/watch?v=iw6BtJxPT8A - why random testing is good (3) -http://www.youtube.com/watch?v=QrLtkSdMDgw - why random testing is good (4) - -Random testing, is not just randomly feeding your software a random -stream of bytes. It requires more thoughts. The Udacity course on -testing (http://www.udacity.com/overview/Course/cs258/CourseRev/1) has -units 3 and 4 entirely dedicated to Random Testing, describing many -things you need to know about Random Testing: how to create valid, -good, random test cases (3.25-26 and 4.5-15), mutators (3.29), oracles -(3.30-34), test case reduction (4.5-6), tradeoffs (4.18-20), and more. -Random Testing is also mentioned elsewhere in the class (and expected -to be know in the final exam). Very worth watching (the introductory -videos linked above are from this class) - -Ok, so you're convinced that Random Testing may be worth exploring. -Why this framework? It is a python framework inspired by Haskell's -http://hackage.haskell.org/package/QuickCheck and Scala's -https://github.com/rickynils/scalacheck - -It does many things for you, including automatic test case reduction -(currently only with bisection techniques, and thus it reduce the test case -in logarithmic time, which means it is very fast although it may reduce to -a case potentially larger than the absolute smallest one) - -============ -Installation -============ - -The easy, system-wide way (requires administrative privileges): - -sudo easy_install pip -sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc -(I hope to get it merged upstream, so you will install with -sudo pip install -e git://github.com/dbravender/qc.git#egg=qc -instead) - -If you don't want to pollute the whole system with this library, -or would like to test it before committing, or simply don't have -root access on your machine, just copy the qc directory and its -content (a mere __init__.py file) into the location of your choice. -To have qc available to your programs you will have to set the -PYTHONPATH environmental variable or have the qc directory as -a subdirectory of the tree where you are running. - -============ - Examples -============ - -examples/ex1_unittest.py and -examples/ex1_nose.py - - These examples provide a simple example (borrowed from scalacheck) - on how to use this framework with python native PyUnit framework - (aka unittest module) and with the popular nose framework. - Just run "python examples/example1_unittest.py" - or "nosetests examples/example1_nose.py". Pytest can also run - the nose tests using "py.test examples/example1_nose.py". - -examples/ex3_airplane.py - Simple example on how to have qc generate your own custom (random) - objects and how to use them in practice - -examples/ex4_bookcase.py - A more elaborate example, showing the power of automatic shrinking - and showing how to write your own shrinker. In this example you can - see how qc automagically finds the root cause of the bug, compare - the output with or without shrinking! - -============ - Known bugs -============ - -At present there is not any known bug. If you discover one, please let -us know. One common problem when using automatic shrinking is running -out of stack space in the recursion process (the shrink algorithm -call itself several times to produce a smaller test case). This may -happen either if there is a bug in qc itself, or if there is a problem -in your test code. You will see an error like: - -RuntimeError: maximum recursion depth exceeded while calling a Python object - -with a stack trace that shows the recursion tree of the shrinking -method calling itself. To understand what is happening, it is usually -useful to add the shrink=False option to the @forall decorator of the -affected test method. In very rare cases it may be necessary to increase -the stack depth with a call to sys.setrecursionlimit(NEWDEPTH), but -do not do it until you understand that it is really the case for your -test. More often than not, there will be a bug in your code or in qc. -Please report the latter. - -============ - TODO -============ - -* print a better summary, e.g. including the total number of tries -per method and other infos (need to investigate how to do it cleanly -and in a way that works both in unittest and nose) - -* provide the option to not stop in case of failures, and instead -logging and continue (or better handled by the underlying test -framework?? see http://stackoverflow.com/questions/4732827/ for a -discussion) - -* improve the current test case reduction from bisection only to -delta-debugging (see http://www.st.cs.uni-saarland.de/dd/ -http://delta.tigris.org/ -http://classes.soe.ucsc.edu/cmps290g/Winter04/lectures/flanagan-290g-8.pdf -for details) - -* integration with git-bisect (maybe) - -* better edge cases coverage: e.g. for integers use a range of -[-sys.maxint-1, sys.maxint] instead of [0, 100] or at the very -least some negative cases... Note that the range needs to be different -for long integers. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..73152bc --- /dev/null +++ b/README.rst @@ -0,0 +1,129 @@ +============== + Introduction +============== + +This framework does Random and Combinatorial Testing. Before you leave +horrified, please have a look at the following videos from Professor +John Regehr, University of Utah (less than 15 minutes total): + +* `Introduction to random testing `_ +* `Why random testing is good (1) `_ +* `Why random testing is good (2) `_ +* `Why random testing is good (3) `_ +* `Why random testing is good (4) `_ + +Random Testing is not just randomly feeding your software a random +stream of bytes. It requires more thoughts. The `Udacity course on +testing`_ has units 3 and 4 entirely dedicated to Random Testing, +describing many things you need to know about Random Testing: how to +create valid, good, random test cases (3.25-26 and 4.5-15), mutators +(3.29), oracles (3.30-34), test case reduction (4.5-6), tradeoffs +(4.18-20), and more. Random Testing is also mentioned elsewhere in +the class (and expected to be known in the final exam). It is very +worth watching (the introductory videos linked above are from this +class). + +Ok, so you're convinced that Random Testing may be worth exploring. +Why this framework? It is a Python framework inspired by Haskell's +QuickCheck_ and Scala's scalacheck_. + +It does many things for you, including automatic test case reduction +(currently only with bisection techniques, and thus it reduce the test case +in logarithmic time, which means it is very fast although it may reduce to +a case potentially larger than the absolute smallest one). + +.. _Udacity course on testing: http://www.udacity.com/overview/Course/cs258/CourseRev/1 +.. _QuickCheck: http://hackage.haskell.org/package/QuickCheck +.. _scalacheck: https://github.com/rickynils/scalacheck + +============== + Installation +============== + +The easy, system-wide way (requires administrative privileges):: + + sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc + +(I hope to get it merged upstream, so you will install with ``sudo pip +install -e git://github.com/dbravender/qc.git#egg=qc`` instead). + +If you don't want to pollute the whole system with this library, or +would like to test it before committing, or simply don't have root +access on your machine, just copy the ``qc`` directory and its content +(a mere ``__init__.py`` file) into the location of your choice. To +have qc available to your programs you will have to set the +``PYTHONPATH`` environmental variable or have the ``qc`` directory as +a subdirectory of the tree where you are running. + +========== + Examples +========== + +``examples/ex1_unittest.py`` and ``examples/ex1_nose.py`` + These examples provide a simple example (borrowed from scalacheck) + on how to use this framework with Python native PyUnit framework + (aka unittest module) and with the popular nose framework. Just + run ``python examples/example1_unittest.py`` or ``nosetests + examples/example1_nose.py``. Pytest can also run the nose tests + using ``py.test examples/example1_nose.py``. + +``examples/ex3_airplane.py`` + Simple example on how to have qc generate your own custom (random) + objects and how to use them in practice. + +``examples/ex4_bookcase.py`` + A more elaborate example, showing the power of automatic shrinking + and showing how to write your own shrinker. In this example you can + see how qc automagically finds the root cause of the bug, compare + the output with or without shrinking! + +============ + Known bugs +============ + +At present there are no known bugs. If you discover one, please let us +know. One common problem when using automatic shrinking is running out +of stack space in the recursion process (the shrink algorithm call +itself several times to produce a smaller test case). This may happen +either if there is a bug in qc itself, or if there is a problem in +your test code. You will see an error like:: + + RuntimeError: maximum recursion depth exceeded while calling a Python object + +with a stack trace that shows the recursion tree of the shrinking +method calling itself. To understand what is happening, it is usually +useful to add the ``shrink=False`` option to the ``@forall`` decorator +of the affected test method. In very rare cases it may be necessary to +increase the stack depth with a call to +``sys.setrecursionlimit(NEWDEPTH)``, but do not do it until you +understand that it is really the case for your test. More often than +not, there will be a bug in your code or in qc. Please report the +latter. + +====== + TODO +====== + +* Print a better summary, e.g. including the total number of tries per + method and other infos (need to investigate how to do it cleanly and + in a way that works both in unittest and nose). + +* Provide the option to not stop in case of failures, and instead + logging and continue (or better handled by the underlying test + framework?? See for a `this StackOverflow question`_ discussion). + +.. _this StackOverflow question: http://stackoverflow.com/questions/4732827/ + +* Improve the current test case reduction from bisection only to + delta-debugging. See the following links for details: + + * http://www.st.cs.uni-saarland.de/dd/ + * http://delta.tigris.org/ + * http://classes.soe.ucsc.edu/cmps290g/Winter04/lectures/flanagan-290g-8.pdf + +* Integration with ``git-bisect`` (maybe). + +* Better edge cases coverage: e.g. for integers use a range of + ``[-sys.maxint-1, sys.maxint]`` instead of ``[0, 100]`` or at the + very least some negative cases... Note that the range needs to be + different for long integers. From 11a39507b6579d32db23fdfee4e732f22e9f5696 Mon Sep 17 00:00:00 2001 From: Sean Fisk Date: Wed, 5 Jun 2013 15:43:23 -0600 Subject: [PATCH 23/53] Add Travis-CI support. --- .travis.yml | 7 +++++++ requirements-test.txt | 1 + 2 files changed, 8 insertions(+) create mode 100644 .travis.yml create mode 100644 requirements-test.txt diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ae92d95 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + - "pypy" +install: "pip install --requirement requirements-test.txt --use-mirrors" +script: nosetests \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..f3c7e8e --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +nose From 3b2ce085b4a8ee706a68258b61ecc0841a66905f Mon Sep 17 00:00:00 2001 From: davidedelvento Date: Thu, 6 Jun 2013 10:32:25 -0500 Subject: [PATCH 24/53] Added Travis image to the README --- README.rst | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 73152bc..ce95c0e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,12 @@ -============== - Introduction -============== +============ + QuickCheck +============ + +.. image:: https://travis-ci.org/davidedelvento/qc.png + :target: https://travis-ci.org/davidedelvento/qc + +Introduction +============ This framework does Random and Combinatorial Testing. Before you leave horrified, please have a look at the following videos from Professor @@ -36,9 +42,9 @@ a case potentially larger than the absolute smallest one). .. _QuickCheck: http://hackage.haskell.org/package/QuickCheck .. _scalacheck: https://github.com/rickynils/scalacheck -============== - Installation -============== + +Installation +============ The easy, system-wide way (requires administrative privileges):: @@ -55,9 +61,9 @@ have qc available to your programs you will have to set the ``PYTHONPATH`` environmental variable or have the ``qc`` directory as a subdirectory of the tree where you are running. -========== - Examples -========== + +Examples +======== ``examples/ex1_unittest.py`` and ``examples/ex1_nose.py`` These examples provide a simple example (borrowed from scalacheck) @@ -77,9 +83,9 @@ a subdirectory of the tree where you are running. see how qc automagically finds the root cause of the bug, compare the output with or without shrinking! -============ - Known bugs -============ + +Known bugs +========== At present there are no known bugs. If you discover one, please let us know. One common problem when using automatic shrinking is running out @@ -100,9 +106,9 @@ understand that it is really the case for your test. More often than not, there will be a bug in your code or in qc. Please report the latter. -====== - TODO -====== + +TODO +==== * Print a better summary, e.g. including the total number of tries per method and other infos (need to investigate how to do it cleanly and From 1fbd07a49ab37008800a8fb1376a2a1ab6c9eb00 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 18 Sep 2014 11:27:06 -0600 Subject: [PATCH 25/53] It looks like Dan Bravender is not anymore interested in maintaining qc, removing the explicit references to his repo (implicit references from github will stay) --- README.rst | 28 ++++++++++++---------------- qc/__init__.py | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index ce95c0e..d2fd673 100644 --- a/README.rst +++ b/README.rst @@ -50,16 +50,16 @@ The easy, system-wide way (requires administrative privileges):: sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc -(I hope to get it merged upstream, so you will install with ``sudo pip -install -e git://github.com/dbravender/qc.git#egg=qc`` instead). - -If you don't want to pollute the whole system with this library, or -would like to test it before committing, or simply don't have root -access on your machine, just copy the ``qc`` directory and its content -(a mere ``__init__.py`` file) into the location of your choice. To -have qc available to your programs you will have to set the +If you don't feel ready to commit for a whole system install of this library, or +simply don't have root access on your machine, just copy the ``qc`` directory +and its content (as seen in https://github.com/davidedelvento/qc/tree/master/qc +at the moment the content is a mere ``__init__.py`` file) into the location of your choice. +To make `qc` available to your programs you will have to set the ``PYTHONPATH`` environmental variable or have the ``qc`` directory as -a subdirectory of the tree where you are running. +a subdirectory of the tree where you are running (for details, see +https://docs.python.org/2/tutorial/modules.html#the-module-search-path if you +use python 2 or https://docs.python.org/3/tutorial/modules.html#the-module-search-path +if you use python 3) Examples @@ -103,8 +103,9 @@ of the affected test method. In very rare cases it may be necessary to increase the stack depth with a call to ``sys.setrecursionlimit(NEWDEPTH)``, but do not do it until you understand that it is really the case for your test. More often than -not, there will be a bug in your code or in qc. Please report the -latter. +not, there will be a bug in your code (especially likely if you are +writing your first shrinker) or in qc. Please report the +latter to https://github.com/davidedelvento/qc/issues TODO @@ -128,8 +129,3 @@ TODO * http://classes.soe.ucsc.edu/cmps290g/Winter04/lectures/flanagan-290g-8.pdf * Integration with ``git-bisect`` (maybe). - -* Better edge cases coverage: e.g. for integers use a range of - ``[-sys.maxint-1, sys.maxint]`` instead of ``[0, 100]`` or at the - very least some negative cases... Note that the range needs to be - different for long integers. diff --git a/qc/__init__.py b/qc/__init__.py index 35f903b..8a29d4c 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) 2009-2011, Dan Bravender +# Copyright (c) 2012-2014, Davide Del Vento import random import os, sys From 9d59ddab60df17c84c17369e5deed3982540a540 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 18 Sep 2014 15:08:47 -0600 Subject: [PATCH 26/53] clarified the verbiage about bugs --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d2fd673..53ade5b 100644 --- a/README.rst +++ b/README.rst @@ -87,8 +87,10 @@ Examples Known bugs ========== -At present there are no known bugs. If you discover one, please let us -know. One common problem when using automatic shrinking is running out +See https://github.com/davidedelvento/qc/issues for a list of known +issues. The Python 3 support seems to be broken at the moment. + +One common problem when using automatic shrinking is running out of stack space in the recursion process (the shrink algorithm call itself several times to produce a smaller test case). This may happen either if there is a bug in qc itself, or if there is a problem in From 9e7a64857f657a83a702587e3e48d49da8784dc7 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 18 Sep 2014 15:10:07 -0600 Subject: [PATCH 27/53] using a custom class instead of re-raising the previous exception, tossing away information possibly embedded into it --- qc/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 8a29d4c..c394b57 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -99,6 +99,14 @@ def qc_shrink(something): except TypeError: pass +class QCAssertionError(AssertionError): + def __init__(self, error, *args, **kwargs): + super(QCAssertionError, self).__init__(*args, **kwargs) + self.parent_error=error + + def __str__(self): + return self.message + str(self.parent_error) + def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs): try: f(*inargs, **random_kwargs) @@ -110,10 +118,10 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) shrinked_kwargs[k] = s call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **shrinked_kwargs) if sys.version_info[0] < 3: - raise e.__class__("%s, generated with seed %s, caused a FAIL\n%s" % - (random_kwargs, seed, e)), None, sys.exc_traceback + raise QCAssertionError(e, str(random_kwargs) + + " (from seed " +str(seed) + ") caused a FAIL: "), None, sys.exc_traceback else: - raise e.__class__("{0}, generated with seed {1}, caused a FAIL\n".format( + raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) def forall(tries=100, shrink=True, seed=None, custom_shrink=qc_shrink, **kwargs): From 6e6760392731965e7696d2b1b2a3d7b8d803a796 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 18 Sep 2014 15:16:02 -0600 Subject: [PATCH 28/53] slowly introducing full Python3 syntax: here two exceptions --- qc/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index c394b57..bac44d1 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -110,7 +110,7 @@ def __str__(self): def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs): try: f(*inargs, **random_kwargs) - except AssertionError, e: # shrink only when there is AssertionErrors, in other cases is ad infinitum recursion + except AssertionError as e: # shrink only when there is AssertionErrors, in other cases is ad infinitum recursion if tryshrink: for k in random_kwargs: for s in custom_shrink(random_kwargs[k]): @@ -128,7 +128,7 @@ def forall(tries=100, shrink=True, seed=None, custom_shrink=qc_shrink, **kwargs) if seed is None: try: seed = hash(os.urandom(16)) - except NotImplementedError, e: + except NotImplementedError: seed = random.random() random.seed(seed) def wrap(f): From 253090290e1d45b0fb5d02bfe8aec2e74c03885f Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 13:50:38 -0600 Subject: [PATCH 29/53] verbose enables the pretty-printing of the whole shrink history --- qc/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qc/__init__.py b/qc/__init__.py index bac44d1..f2eb954 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -116,6 +116,9 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) for s in custom_shrink(random_kwargs[k]): shrinked_kwargs = random_kwargs.copy() shrinked_kwargs[k] = s + if forall.verbose or os.environ.has_key('QC_VERBOSE'): + from pprint import pprint + pprint(random_kwargs) call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **shrinked_kwargs) if sys.version_info[0] < 3: raise QCAssertionError(e, str(random_kwargs) + @@ -139,7 +142,7 @@ def wrapped(*inargs, **inkwargs): for (name, gen) in kwargs.iteritems())) if forall.verbose or os.environ.has_key('QC_VERBOSE'): from pprint import pprint - pprint(random_kwargs) + pprint("Shrink history:") random_kwargs.update(**inkwargs) call_and_shrink(f, shrink, seed, custom_shrink, *inargs, **random_kwargs) if forall.printsummary: From d010f87ea7fb161a2d1dc359913de6c4b8f9833a Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 14:12:40 -0600 Subject: [PATCH 30/53] better default values for integers and their tests --- qc/__init__.py | 5 ++++- tests/test_qc.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index f2eb954..5266d78 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -5,12 +5,15 @@ import os, sys import functools -def integers(low=0, high=100): +def integers(low=-sys.maxint-1, high=sys.maxint): '''Endlessly yields random integers between (inclusively) low and high. Yields low then high first to test boundary conditions. ''' yield low yield high + for i in (-1,0,1): + if low < i < high: + yield i while True: yield random.randint(low, high) diff --git a/tests/test_qc.py b/tests/test_qc.py index 02cecb3..dc5f430 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -2,8 +2,17 @@ @forall(tries=10, i=integers()) def test_integers(i): - assert type(i) == int - assert i >= 0 and i <= 100 + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + +@forall(tries=10, i=integers(low=0, high=100)) +def test__nonnegative_integers(i): + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + assert 0 <= i <= 100, "not the expected range" + +@forall(tries=10, i=integers(low=-100, high=0)) +def test__nonpositive_integers(i): + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + assert -100 <= i <= 0, "not the expected range" @forall(tries=10, l=lists(items=integers())) def test_a_int_list(l): From 4705e2f59c0562582a5444fb29fd3e2656157935 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 14:39:49 -0600 Subject: [PATCH 31/53] better default values for floats and their tests --- qc/__init__.py | 13 ++++++++++--- tests/test_qc.py | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 5266d78..1ffb70d 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -17,12 +17,19 @@ def integers(low=-sys.maxint-1, high=sys.maxint): while True: yield random.randint(low, high) -def floats(low=0.0, high=100.0): +def floats(low=-sys.float_info.max, high=sys.float_info.max, special=True): '''Endlessly yields random floats between (inclusively) low and high. Yields low then high first to test boundary conditions. ''' - yield low - yield high + yield float(low) + yield float(high) + for i in (-1.0, -sys.float_info.min, 0.0, sys.float_info.min, 1.0): + if low < i < high: + yield i + if special: + yield float('nan') + yield float('inf') + yield float('-inf') while True: yield random.uniform(low, high) diff --git a/tests/test_qc.py b/tests/test_qc.py index dc5f430..ba36e0a 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,4 +1,5 @@ from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink +import math @forall(tries=10, i=integers()) def test_integers(i): @@ -24,8 +25,23 @@ def test_a_int_tuple(l): @forall(tries=10, i=floats()) def test_floats(i): - assert type(i) == float - assert i >= 0.0 and i <= 100.0 + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + +@forall(tries=10, i=floats(low=0, high=100)) +def test_nonnegative_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert 0 <= i <= 100 or math.isnan(i) or math.isinf(i), "not the expected range" + +@forall(tries=10, i=floats(low=-100, high=0)) +def test_nonpositive_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert -100 <= i <= 0 or math.isnan(i) or math.isinf(i), "not the expected range" + +@forall(tries=10, i=floats(low=-100, high=-100, special=False)) +def test_nonspecial_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert -100 <= i <= 100, "not the expected range" + @forall(tries=10, l=lists(items=floats())) def test_a_float_list(l): From 9cc81d5633e6dfb69263ced740a7f12c5e8b726c Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 14:42:45 -0600 Subject: [PATCH 32/53] do not shrink INFs --- qc/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 1ffb70d..175a341 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) 2009-2011, Dan Bravender # Copyright (c) 2012-2014, Davide Del Vento -import random +import random, math import os, sys import functools @@ -104,7 +104,7 @@ def qc_shrink(something): except TypeError: pass try: - if abs(something) >= 2: + if abs(something) >= 2 and not math.isinf(something): yield something/2 except TypeError: pass From 5a8a4f4e084da71bb6be365a22b03a71852d2c94 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 15:13:24 -0600 Subject: [PATCH 33/53] Lists and tuples tests --- tests/test_qc.py | 51 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index ba36e0a..d0a353e 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,6 +1,8 @@ from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink import math +# Integers + @forall(tries=10, i=integers()) def test_integers(i): assert type(i) == int, "expected an int, instead got a " + str(type(i)) @@ -15,13 +17,7 @@ def test__nonpositive_integers(i): assert type(i) == int, "expected an int, instead got a " + str(type(i)) assert -100 <= i <= 0, "not the expected range" -@forall(tries=10, l=lists(items=integers())) -def test_a_int_list(l): - assert type(l) == list - -@forall(tries=10, l=tuples(items=integers())) -def test_a_int_tuple(l): - assert type(l) == tuple +# Floats @forall(tries=10, i=floats()) def test_floats(i): @@ -42,26 +38,47 @@ def test_nonspecial_floats(i): assert type(i) == float, "expected a float, instead got a " + str(type(i)) assert -100 <= i <= 100, "not the expected range" +# Lists and tuples + +@forall(tries=10, l=lists(items=integers())) +def test_a_int_list(l): + assert type(l) == list, "expected a list, instead got a " + str(type(l)) + for i in l: + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + +@forall(tries=10, t=tuples(items=integers())) +def test_a_int_tuple(t): + assert type(t) == tuple, "expected a tuple, instead got a " + str(type(t)) + for i in t: + assert type(i) == int, "expected an int, instead got a " + str(type(i)) @forall(tries=10, l=lists(items=floats())) def test_a_float_list(l): - assert type(l) == list - assert reduce(lambda x,y: x and type(y) == float, l, True) + assert type(l) == list, "expected a list, instead got a " + str(type(l)) + for i in l: + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + +@forall(tries=10, t=tuples(items=floats())) +def test_a_float_tuple(t): + assert type(t) == tuple, "expected a tuple, instead got a " + str(type(t)) + for i in t: + assert type(i) == float, "expected a float, instead got a " + str(type(i)) +@forall(tries=10, l=lists(items=integers(), size=(10, 50))) +def test_lists_size(l): + assert 10 <= len(l) <= 50, "list of unexpected size: " + str(len(l)) + +@forall(tries=10, t=tuples(items=integers(), size=(10, 50))) +def test_tuples_size(t): + assert 10 <= len(t) <= 50, "tuple of unexpected size: " + str(len(t)) + +# @forall(tries=10, ul=lists(items=unicodes())) def test_unicodes_list(ul): assert type(ul) == list if len(ul): assert type(ul[0]) == unicode -@forall(tries=10, l=lists(items=integers(), size=(10, 50))) -def test_lists_size(l): - assert len(l) <= 50 and len(l) >= 10 - -@forall(tries=10, l=tuples(items=integers(), size=(10, 50))) -def test_tuples_size(l): - assert len(l) <= 50 and len(l) >= 10 - @forall(tries=10, u=unicodes()) def test_unicodes(u): assert type(u) == unicode From cb168ff3bd82558fdb5b820764a0abbe15dc763d Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 15:28:45 -0600 Subject: [PATCH 34/53] make sure correct behavior when requesting incorrect list/tuple size --- qc/__init__.py | 2 ++ tests/test_qc.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/qc/__init__.py b/qc/__init__.py index 175a341..45fa2ad 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -38,6 +38,7 @@ def lists(items=integers(), size=(0, 100)): and size[1]. Yields a list of the low size and the high size first to test boundary conditions. ''' + assert size[0] >=0, "list size must be non-negative" yield [items.next() for _ in xrange(size[0])] yield [items.next() for _ in xrange(size[1])] while True: @@ -48,6 +49,7 @@ def tuples(items=integers(), size=(0, 100)): and size[1]. Yields a tuple of the low size and the high size first to test boundary conditions. ''' + assert size[0] >=0, "tuple size must be non-negative" yield tuple([items.next() for _ in xrange(size[0])]) yield tuple([items.next() for _ in xrange(size[1])]) while True: diff --git a/tests/test_qc.py b/tests/test_qc.py index d0a353e..0129edf 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,5 +1,6 @@ from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink import math +from nose.tools import raises # Integers @@ -72,6 +73,27 @@ def test_lists_size(l): def test_tuples_size(t): assert 10 <= len(t) <= 50, "tuple of unexpected size: " + str(len(t)) +@raises(AssertionError) +@forall(tries=10, t=tuples(size=(-10, 10))) +def test_negative_tuples_size(t): + pass # must raise an exception for negative sizes + +@raises(AssertionError) +@forall(tries=10, t=lists(size=(-10, 10))) +def test_negative_lists_size(t): + pass # must raise an exception for negative sizes + +@raises(Exception) +@forall(tries=10, t=tuples(size=(20, 10))) +def test_wrong_tuples_size(t): + pass # must raise an exception for min > max + +@raises(Exception) +@forall(tries=10, t=lists(size=(20, 10))) +def test_wrong_lists_size(t): + pass # must raise an exception for min > max + + # @forall(tries=10, ul=lists(items=unicodes())) def test_unicodes_list(ul): From ab9f8c24f0ab84eebc13db30d74f94b8f7544b6f Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 19 Sep 2014 15:47:21 -0600 Subject: [PATCH 35/53] moving the order of tests around --- tests/test_qc.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index 0129edf..edd2941 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -93,8 +93,12 @@ def test_wrong_tuples_size(t): def test_wrong_lists_size(t): pass # must raise an exception for min > max +# Unicode and characters + +@forall(tries=10, c=characters()) +def test_characters(c): + assert len(c) == 1 -# @forall(tries=10, ul=lists(items=unicodes())) def test_unicodes_list(ul): assert type(ul) == list @@ -120,17 +124,17 @@ def test_a_tupled_list(l): for x in l: assert type(x[0]) == int and type(x[1]) == unicode +# Other tests + @forall(tries=10, x=integers(), y=integers()) -def test_addition_commutative(x, y): +def test_integer_addition_commutative(x, y): assert x + y == y + x @forall(tries=10, l=lists()) def test_reverse_reverse(l): assert list(reversed(list(reversed(l)))) == l -@forall(tries=10, c=characters()) -def test_characters(c): - assert len(c) == 1 +# Dictionaries def kv_unicode_integers(): u = unicodes() @@ -157,6 +161,8 @@ def test_dicts_size(d): assert type(x) == unicode assert type(y) == list +# Shrinking + def test_shrink_empty_list(): empty_list_has_been_shrunk = False for x in qc_shrink([]): From d823c9387ed6d0e145affe127b27137b954e00bc Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 22 Sep 2014 14:21:48 -0600 Subject: [PATCH 36/53] comments about testing dicts --- tests/test_qc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index edd2941..376b8c9 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -136,7 +136,7 @@ def test_reverse_reverse(l): # Dictionaries -def kv_unicode_integers(): +def kv_unicode_integers(): # key-value helper u = unicodes() i = integers() while True: @@ -148,7 +148,7 @@ def test_dicts(d): assert type(x) == unicode assert type(y) == int -def kv_unicodes_lists(): +def kv_unicodes_lists(): # key-value helper u = unicodes() l = lists() while True: From 8856896b0295255b85b3d054783d91c41ba7eeff Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 22 Sep 2014 14:45:19 -0600 Subject: [PATCH 37/53] clarified some shrinking tests --- tests/test_qc.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index 376b8c9..cc84a58 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -163,11 +163,9 @@ def test_dicts_size(d): # Shrinking +@raises(StopIteration) def test_shrink_empty_list(): - empty_list_has_been_shrunk = False - for x in qc_shrink([]): - empty_list_has_been_shrunk = True - assert empty_list_has_been_shrunk == False, "Empty lists must not be shrunk" + qc_shrink([]).next() # there must be no next, an Exception must be raised def test_shrink_single_element_list(): repeated = False @@ -180,24 +178,21 @@ def test_shrink_single_element_list(): @forall(full_l=lists()) def test_shrink_lists(full_l): for sub_l in qc_shrink(full_l): - assert len(sub_l) <= len(full_l)/2 + 1 + assert len(sub_l) <= len(full_l)/2 + 1, "list must be shrunk in half" -@forall(full_i=integers(low=-100)) +@forall(tries=100, full_i=integers()) def test_shrink_integers(full_i): for i in qc_shrink(full_i): - assert abs(i) <= abs(full_i)/2 + 1 - assert cmp(i,0) == cmp(full_i, 0) # shrink shall not change sign - assert isinstance(i, int) + assert isinstance(i, int), "ints must be shrunk to ints" + assert abs(i) <= abs(full_i)/2 + 1, "ints must be shrunk to smaller ints" + assert cmp(i,0) == cmp(full_i, 0), "shrink must not change sign of ints" -@forall(full_f=floats(low=-5.0, high=5.0)) +@forall(tries=1000, full_f=floats()) def test_shrink_floats(full_f): for f in qc_shrink(full_f): - if abs(full_f) > 1: - assert abs(f) <= abs(full_f)/2 + 1 - else: - assert abs(f) >= abs(full_f)/2 + 1 - assert cmp(f,0) == cmp(full_f, 0) # shrink shall not change sign - assert isinstance(f, float) + assert isinstance(f, float), "floats must be shrunk to floats" + assert abs(f) <= abs(full_f)/2 + 1 # TODO, check if this is really a must + assert cmp(f,0) == cmp(full_f, 0), "shrink must not change sign of floats" @forall(tries=10, i=integers(low=0, high=10)) def each_integer_from_0_to_10(i, target_low, target_high): From 3d1ed70a97cdb4a9e6d757a3a9c6d723c0dc0e84 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 22 Sep 2014 15:42:31 -0600 Subject: [PATCH 38/53] testing that all the cases of floats are covered -- and making sure they are --- qc/__init__.py | 9 +++++-- tests/test_qc.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 45fa2ad..91d95de 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -30,8 +30,13 @@ def floats(low=-sys.float_info.max, high=sys.float_info.max, special=True): yield float('nan') yield float('inf') yield float('-inf') - while True: - yield random.uniform(low, high) + + if low == -sys.float_info.max and high == sys.float_info.max: + while True: # uniform does not work for such a large range + yield random.uniform(-2, 2) * 10 ** random.randint(-sys.float_info.max_10_exp+1, sys.float_info.max_10_exp-1) + else: + while True: # TODO it should probably detect a large range and use an approach similar to the previous one + yield random.uniform(low, high) def lists(items=integers(), size=(0, 100)): '''Endlessly yields random lists varying in size between size[0] diff --git a/tests/test_qc.py b/tests/test_qc.py index cc84a58..e6112b5 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,5 +1,5 @@ from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink -import math +import math, sys from nose.tools import raises # Integers @@ -39,6 +39,70 @@ def test_nonspecial_floats(i): assert type(i) == float, "expected a float, instead got a " + str(type(i)) assert -100 <= i <= 100, "not the expected range" +def test_floats_coverage(): + negative_inf = False + negative_max = False + negative_large = False + negative_one = False + negative_small = False + negative_min = False + zero = False + positive_min = False + positive_small = False + positive_one = False + positive_large = False + positive_max = False + positive_inf = False + nan = False + + gen = floats() + for i in range(100): # 100 opportunities should be plenty + f = gen.next() + print "value = ", f + if math.isinf(f) and f < 0: + negative_inf = True + elif f == -sys.float_info.max: + negative_max = True + elif -sys.float_info.max / 2 < f < -2: + negative_large = True + elif f == -1.0: + negative_one = True + elif -0.5 < f < -sys.float_info.min * 2: + negative_small = True + elif f == -sys.float_info.min: + negative_min = True + elif f == 0: + zero = True + elif f == sys.float_info.min: + positive_min = True + elif sys.float_info.min * 2 < f < 0.5: + positive_small = True + elif f == 1.0: + positive_one = True + elif 2 < f < sys.float_info.max / 2: + positive_large = True + elif f == sys.float_info.max: + positive_max = True + elif math.isinf(f) and f > 0: + positive_inf = True + elif math.isnan(f): + nan = True + + assert negative_inf == True + assert negative_max == True + assert negative_large == True + assert negative_one == True + assert negative_small == True + assert negative_min == True + assert zero == True + assert positive_min == True + assert positive_small == True + assert positive_one == True + assert positive_large == True + assert positive_max == True + assert positive_inf == True + assert nan == True + # Lists and tuples @forall(tries=10, l=lists(items=integers())) From b99e4cf94a023fcd54ae61511281397feb2b55e5 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 22 Sep 2014 16:01:16 -0600 Subject: [PATCH 39/53] clarified the shrinking of floats applies to large one -- comment on small ones --- tests/test_qc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index e6112b5..ecd8b71 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -255,7 +255,8 @@ def test_shrink_integers(full_i): def test_shrink_floats(full_f): for f in qc_shrink(full_f): assert isinstance(f, float), "floats must be shrunk to floats" - assert abs(f) <= abs(full_f)/2 + 1 # TODO, check if this is really a must + assert abs(full_f) > 1 # TODO, check if something is appropriate for smaller number + assert abs(f) <= abs(full_f)/2 + 1, "shrunk fload must be smaller than parent" assert cmp(f,0) == cmp(full_f, 0), "shrink must not change sign of floats" @forall(tries=10, i=integers(low=0, high=10)) From efdc3924fd2671b63c39847807198856bacf11ec Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 26 Sep 2014 15:42:01 -0600 Subject: [PATCH 40/53] This does not seem to be printed as desired, it gets repeated inappropriately (and wasn't very useful to begin with), so removing --- qc/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 91d95de..054a8dd 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -157,9 +157,6 @@ def wrapped(*inargs, **inkwargs): for _ in xrange(tries): random_kwargs = (dict((name, gen.next()) for (name, gen) in kwargs.iteritems())) - if forall.verbose or os.environ.has_key('QC_VERBOSE'): - from pprint import pprint - pprint("Shrink history:") random_kwargs.update(**inkwargs) call_and_shrink(f, shrink, seed, custom_shrink, *inargs, **random_kwargs) if forall.printsummary: From 3ecb3e13cb7d935f656b549c4e743eb4b046d41d Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 26 Sep 2014 15:55:42 -0600 Subject: [PATCH 41/53] some tests for the forall wrapper --- tests/test_qc.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_qc.py b/tests/test_qc.py index ecd8b71..1652311 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -271,6 +271,26 @@ class TestClass(object): def __init__(self, arg): self.arg_from_init = arg +# forall wrapper + +def constant(value): + while True: + yield value + +def test_forall(): + @forall(tries=11, a=constant(7.5)) + def myf(arr, a): + arr.append(a) + arr = [] + myf(arr) + assert arr == [7.5]*11, "arr is" + str(arr) + "instead of the expected" + str([7.5]*11) + @forall(tries=11, a=constant(7.5)) + def myf(a, myarr=None): + myarr.append(a) + arr = [] + myf(myarr=arr) + assert arr == [7.5]*11, "arr is" + str(arr) + "instead of the expected" + str([7.5]*11) + @forall(tries=10, obj=objects(TestClass, {'an_int': integers(), 'a_float': floats()}, unicodes())) def test_objects(obj): assert type(obj) == TestClass From a705d15b7145078bd1069455777ed07ad544add4 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Fri, 3 Oct 2014 15:05:07 -0600 Subject: [PATCH 42/53] All choice combinatorial testing --- qc/__init__.py | 18 ++++++++++++++++-- tests/test_qc.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 054a8dd..9012c9a 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -2,8 +2,8 @@ # Copyright (c) 2012-2014, Davide Del Vento import random, math -import os, sys -import functools +import os, sys, warnings +import itertools, functools def integers(low=-sys.maxint-1, high=sys.maxint): '''Endlessly yields random integers between (inclusively) low and high. @@ -144,6 +144,20 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) +def allchoices(**kwargs): + warnings.warn("Testing all choices may take very long time, because there may be too many", RuntimeWarning) + def wrap(f): + @functools.wraps(f) + def wrapped(*inargs, **inkwargs): + for values in itertools.product(*tuple(kwargs.values())): + new_kwargs={} + for i,k in enumerate(kwargs.keys()): + new_kwargs[k]=values[i] + new_kwargs.update(**inkwargs) + f(*inargs, **new_kwargs) + return wrapped + return wrap + def forall(tries=100, shrink=True, seed=None, custom_shrink=qc_shrink, **kwargs): if seed is None: try: diff --git a/tests/test_qc.py b/tests/test_qc.py index 1652311..5785ef2 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,4 +1,4 @@ -from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, qc_shrink, call_and_shrink +from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, allchoices, qc_shrink, call_and_shrink import math, sys from nose.tools import raises @@ -298,3 +298,33 @@ def test_objects(obj): assert type(obj.a_float) == float assert type(obj.arg_from_init) == unicode +# All choices + +def test_allchoices(): + arr = [] + @allchoices(a=(-1, 0, 1), b=('x', 'y', 'z'), c=(True, False)) + def myf(arr, a,b,c): + arr.append((a,b,c)) + myf(arr) + # explicit is better than implicit, avoid logic in test, please bear the following + arr.remove((-1, 'x', True)) + arr.remove((-1, 'y', True)) + arr.remove((-1, 'z', True)) + arr.remove((0, 'x', True)) + arr.remove((0, 'y', True)) + arr.remove((0, 'z', True)) + arr.remove((1, 'x', True)) + arr.remove((1, 'y', True)) + arr.remove((1, 'z', True)) + # + arr.remove((-1, 'x', False)) + arr.remove((-1, 'y', False)) + arr.remove((-1, 'z', False )) + arr.remove((0, 'x', False)) + arr.remove((0, 'y', False)) + arr.remove((0, 'z', False)) + arr.remove((1, 'x', False)) + arr.remove((1, 'y', False)) + arr.remove((1, 'z', False)) + assert len(arr) == 0, "arr is" +str( arr) + From ce52be7f22d74b22f519e9ba9bc59c280d34dd32 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 6 Oct 2014 15:38:28 -0600 Subject: [PATCH 43/53] suppressing the warning for the test, since the test case is small enough --- tests/test_qc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_qc.py b/tests/test_qc.py index 5785ef2..1d44509 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,5 +1,5 @@ from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, allchoices, qc_shrink, call_and_shrink -import math, sys +import math, sys, warnings from nose.tools import raises # Integers @@ -302,9 +302,11 @@ def test_objects(obj): def test_allchoices(): arr = [] - @allchoices(a=(-1, 0, 1), b=('x', 'y', 'z'), c=(True, False)) - def myf(arr, a,b,c): - arr.append((a,b,c)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # There are only 3 * 3 * 2 = 18 choices + @allchoices(a=(-1, 0, 1), b=('x', 'y', 'z'), c=(True, False)) + def myf(arr, a,b,c): + arr.append((a,b,c)) myf(arr) # explicit is better than implicit, avoid logic in test, please bear the following arr.remove((-1, 'x', True)) From dc5498e02f9f755ed17ef0cd2bf2196281018ad8 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 6 Oct 2014 15:40:41 -0600 Subject: [PATCH 44/53] TDD simple (binary-only) all pairs test and a dummy (test-failing) implementation --- qc/__init__.py | 12 ++++++++++++ tests/test_qc.py | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/qc/__init__.py b/qc/__init__.py index 9012c9a..8eea4ef 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -144,6 +144,18 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) +def allpairs(**kwargs): + def wrap(f): + @functools.wraps(f) + def wrapped(*inargs, **inkwargs): + new_kwargs = {} + for qqq in (0, -1): + for key, values in zip(kwargs.keys(), kwargs.values()): + new_kwargs[key] = values[qqq] + f(*inargs, **new_kwargs) + return wrapped + return wrap + def allchoices(**kwargs): warnings.warn("Testing all choices may take very long time, because there may be too many", RuntimeWarning) def wrap(f): diff --git a/tests/test_qc.py b/tests/test_qc.py index 1d44509..07d55e5 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,4 +1,4 @@ -from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, allchoices, qc_shrink, call_and_shrink +from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, allchoices, allpairs, qc_shrink, call_and_shrink import math, sys, warnings from nose.tools import raises @@ -330,3 +330,29 @@ def myf(arr, a,b,c): arr.remove((1, 'z', False)) assert len(arr) == 0, "arr is" +str( arr) +# All pairs + +def pairs_search(pairs, a, b, expected_values): + for av,bv in expected_values: + found = False + for case in pairs: + if case[a] == av and case[b] == bv: + found = True + assert found, "Pair (" + a + "=" + str(av) + ", " + b + "=" + str(bv) + ") not found in" + str(pairs) + +def test_all_binary_pairs(): + pairs=[] + @allpairs(a=(-1,1), b=(True, False), c=('A', 'B'), d=(0.0, 1.0)) + def mypairs(arr, a, b, c, d): + arr.append({'a':a, 'b':b, 'c':c, 'd':d}) + mypairs(pairs) + + # explicit is better than implicit, wanted to make this test more explicit, but here it is at least for now... + pairs_search(pairs, 'a', 'b', ((-1, True), (-1, False), (1, True), (1, False))) + pairs_search(pairs, 'a', 'c', ((-1, "A"), (-1, "B"), (1, "A"), (1, "B"))) + pairs_search(pairs, 'a', 'd', ((-1, 0.0), (-1, 1.0), (1, 0.0), (1, 1.0))) + + pairs_search(pairs, 'b', 'c', ((True, 'A'), (True, 'B'), (False, 'A') (False, 'B'))) + pairs_search(pairs, 'b', 'd', ((True, 0.0), (True, 1.0), (False, 0.0) (False, 1.0))) + + pairs_search(pairs, 'c', 'd', (('A', 0.0), ('A', 1.0), ('B', 0.0), ('B', 1.0))) From a4d82f9918dd97e8fdf89e3f01168f3dd3dc09b2 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Tue, 7 Oct 2014 11:01:54 -0600 Subject: [PATCH 45/53] Updated and cleaned up the README --- README.rst | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 53ade5b..5f1b385 100644 --- a/README.rst +++ b/README.rst @@ -8,9 +8,33 @@ Introduction ============ -This framework does Random and Combinatorial Testing. Before you leave -horrified, please have a look at the following videos from Professor -John Regehr, University of Utah (less than 15 minutes total): +This framework does Random and Combinatorial Testing. It is a Python framework +inspired by Haskell's QuickCheck_ and Scala's scalacheck_ (not a port to Python +of those frameworks). + +In Combinatorial Testing, `qc` provides a deterministic implementation of all-choices +(simply picks all the combinatorial choices of parameters) and all-pairs_ +algorithms. All-choices simply tests all possible choice of input parameters, so it's +an exhaustive, but very slow technique when there are more than a few parameters with +more than a handful of possible values each. The all-pairs_ algorithm is very +clever and exponentially reduces the running time, while still guaranteeing that each +pair of input parameter is tested for each possible combination of values. + +In Random Testing, `qc` provides many convenient ways of generating test cases, and +a very useful automatic test case reduction. The test case reduction uses binary search +to find a small, failing test case, very quickly. + +.. _QuickCheck: http://hackage.haskell.org/package/QuickCheck +.. _scalacheck: https://github.com/rickynils/scalacheck +.. _all-pairs: https://en.wikipedia.org/wiki/All-pairs_testing + +More on Random Testing +====================== + +Since Random Testing is not popular, you may want to +have a look at the following videos from Professor +John Regehr, University of Utah (less than 15 minutes total) if you are not +familiar with the technique: * `Introduction to random testing `_ * `Why random testing is good (1) `_ @@ -18,7 +42,7 @@ John Regehr, University of Utah (less than 15 minutes total): * `Why random testing is good (3) `_ * `Why random testing is good (4) `_ -Random Testing is not just randomly feeding your software a random +As you can see, Random Testing is not just randomly feeding your software a random stream of bytes. It requires more thoughts. The `Udacity course on testing`_ has units 3 and 4 entirely dedicated to Random Testing, describing many things you need to know about Random Testing: how to @@ -29,19 +53,7 @@ the class (and expected to be known in the final exam). It is very worth watching (the introductory videos linked above are from this class). -Ok, so you're convinced that Random Testing may be worth exploring. -Why this framework? It is a Python framework inspired by Haskell's -QuickCheck_ and Scala's scalacheck_. - -It does many things for you, including automatic test case reduction -(currently only with bisection techniques, and thus it reduce the test case -in logarithmic time, which means it is very fast although it may reduce to -a case potentially larger than the absolute smallest one). - .. _Udacity course on testing: http://www.udacity.com/overview/Course/cs258/CourseRev/1 -.. _QuickCheck: http://hackage.haskell.org/package/QuickCheck -.. _scalacheck: https://github.com/rickynils/scalacheck - Installation ============ @@ -73,6 +85,8 @@ Examples examples/example1_nose.py``. Pytest can also run the nose tests using ``py.test examples/example1_nose.py``. +``examples/ex2_choices_pairs.py`` TBD + ``examples/ex3_airplane.py`` Simple example on how to have qc generate your own custom (random) objects and how to use them in practice. From b7cfa4e0659fba15e26bb84aa708799a7f62dc3b Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Tue, 7 Oct 2014 12:02:37 -0600 Subject: [PATCH 46/53] TDD: simple all pair implementation with very limited functionality --- qc/__init__.py | 25 +++++++++++++++++++++---- tests/test_qc.py | 14 ++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index 8eea4ef..c17a023 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -144,14 +144,31 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) +# TODO: I don't like the following implementation, but I'm putting it out there for now + +magic={} +magic[1]=['0','1'] +magic[3]=['000','011','100','110'] +magic[10]=['0000000000','0000111111','0111000111','1011011001','1101101010','1110110100'] + def allpairs(**kwargs): + mkeys=magic.keys() + def wrap(f): @functools.wraps(f) def wrapped(*inargs, **inkwargs): - new_kwargs = {} - for qqq in (0, -1): - for key, values in zip(kwargs.keys(), kwargs.values()): - new_kwargs[key] = values[qqq] + for values in kwargs.values(): + if len(values) != 2: + raise NotImplementedError("At the moment only binary arguments are supported") + for k in mkeys: + if k < len(kwargs.keys()): + mkeys.remove(k) + if len(mkeys) == 0: + raise NotImplementedError("At the moment only up to 10 arguments are supported") + for curr_test in magic[min(mkeys)]: + new_kwargs = {} + for i, (key, values) in enumerate(zip(kwargs.keys(), kwargs.values())): + new_kwargs[key] = values[int(curr_test[i])] f(*inargs, **new_kwargs) return wrapped return wrap diff --git a/tests/test_qc.py b/tests/test_qc.py index 07d55e5..127e20c 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -352,7 +352,17 @@ def mypairs(arr, a, b, c, d): pairs_search(pairs, 'a', 'c', ((-1, "A"), (-1, "B"), (1, "A"), (1, "B"))) pairs_search(pairs, 'a', 'd', ((-1, 0.0), (-1, 1.0), (1, 0.0), (1, 1.0))) - pairs_search(pairs, 'b', 'c', ((True, 'A'), (True, 'B'), (False, 'A') (False, 'B'))) - pairs_search(pairs, 'b', 'd', ((True, 0.0), (True, 1.0), (False, 0.0) (False, 1.0))) + pairs_search(pairs, 'b', 'c', ((True, 'A'), (True, 'B'), (False, 'A'), (False, 'B'))) + pairs_search(pairs, 'b', 'd', ((True, 0.0), (True, 1.0), (False, 0.0), (False, 1.0))) pairs_search(pairs, 'c', 'd', (('A', 0.0), ('A', 1.0), ('B', 0.0), ('B', 1.0))) + +def test_allpairs(): + pairs = [] + @allpairs(x=('x', 'y', 'z'), p=(0,1), q=(True, False)) + def mypairs(arr, x, p, q): + arr.append({'x':x, 'p':p, 'q':q}) + mypairs(pairs) + + pairs_search(pairs, 'x', 'q', (('x', True), ('y', True), ('z', True), ('x', False), ('y', False), ('z', False))) + # and other pairs From c5ed801d763642373f5f5947b7f8f499cfb8a25b Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Tue, 7 Oct 2014 13:35:00 -0600 Subject: [PATCH 47/53] marking the fully all-pairs test as skipped instead of failed --- tests/test_qc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_qc.py b/tests/test_qc.py index 127e20c..54d7726 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -358,6 +358,8 @@ def mypairs(arr, a, b, c, d): pairs_search(pairs, 'c', 'd', (('A', 0.0), ('A', 1.0), ('B', 0.0), ('B', 1.0))) def test_allpairs(): + from nose.plugins.skip import SkipTest + raise SkipTest("Test is skipped") pairs = [] @allpairs(x=('x', 'y', 'z'), p=(0,1), q=(True, False)) def mypairs(arr, x, p, q): From 8a90d3c151e76c47c8e8a6c97632ccedbdb203d5 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 9 Oct 2014 15:49:24 -0600 Subject: [PATCH 48/53] nicer, more generic implementation to generate all pairs for binary variables --- qc/__init__.py | 45 +++++++++++++++++++++++++++++++-------------- tests/test_qc.py | 9 ++++++++- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/qc/__init__.py b/qc/__init__.py index c17a023..3ae7587 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -144,28 +144,45 @@ def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs) raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( random_kwargs, seed)).with_traceback(e.__traceback__) -# TODO: I don't like the following implementation, but I'm putting it out there for now - -magic={} -magic[1]=['0','1'] -magic[3]=['000','011','100','110'] -magic[10]=['0000000000','0000111111','0111000111','1011011001','1101101010','1110110100'] +def list_of_tests(num_of_vars): + if num_of_vars == 1: + return ['0','1'] + elif num_of_vars <= 3: + return ['000','011','100','110'] + else: + from math import factorial + coverage = -1 # how many variables we can cover with not tests + num_of_tests = 4 # how many tests we need + while coverage < num_of_vars: # we want to cover all the ones we have + num_of_tests += 2 + coverage = factorial(num_of_tests) / ( 2 * (factorial(num_of_tests/2) ** 2) ) + + # at this point we know that with num_of_tests tests we can cover the num_of_vars we have + + # To build the test cases, we use the following which is one way to build the minimum + # necessary and sufficient set to to cover coverage variable. + # The proof is simple, but too large for this margin + list_of_cases = [] + for t in range(2**(num_of_tests-1)): + case=bin(t)[2:].zfill(num_of_tests) + if case.count('0') == num_of_tests / 2: # == case.count('1'): + list_of_cases.append(case) + + # For going from cases to tests, we take the transpose + lot = [] # list of tests + for i in range(num_of_tests): + lot.append(''.join([case[i] for case in list_of_cases])) + + return lot def allpairs(**kwargs): - mkeys=magic.keys() - def wrap(f): @functools.wraps(f) def wrapped(*inargs, **inkwargs): for values in kwargs.values(): if len(values) != 2: raise NotImplementedError("At the moment only binary arguments are supported") - for k in mkeys: - if k < len(kwargs.keys()): - mkeys.remove(k) - if len(mkeys) == 0: - raise NotImplementedError("At the moment only up to 10 arguments are supported") - for curr_test in magic[min(mkeys)]: + for curr_test in list_of_tests(len(kwargs.keys())): new_kwargs = {} for i, (key, values) in enumerate(zip(kwargs.keys(), kwargs.values())): new_kwargs[key] = values[int(curr_test[i])] diff --git a/tests/test_qc.py b/tests/test_qc.py index 54d7726..43a11bb 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -347,7 +347,11 @@ def mypairs(arr, a, b, c, d): arr.append({'a':a, 'b':b, 'c':c, 'd':d}) mypairs(pairs) - # explicit is better than implicit, wanted to make this test more explicit, but here it is at least for now... + # The following it should be implicitly covered by the pairs_search below, but explicit is better than implicit... + assert len(pairs) > 4, "Cannot provide all pairs of four binary variables with four or less tests (" + str(len(pairs)) + " given)" + + # explicit is better than implicit, wanted to make this test more explicit, but here it is at least for now and + # it does test correctness pairs_search(pairs, 'a', 'b', ((-1, True), (-1, False), (1, True), (1, False))) pairs_search(pairs, 'a', 'c', ((-1, "A"), (-1, "B"), (1, "A"), (1, "B"))) pairs_search(pairs, 'a', 'd', ((-1, 0.0), (-1, 1.0), (1, 0.0), (1, 1.0))) @@ -357,6 +361,9 @@ def mypairs(arr, a, b, c, d): pairs_search(pairs, 'c', 'd', (('A', 0.0), ('A', 1.0), ('B', 0.0), ('B', 1.0))) + # The following tests that the implementation is efficient + assert len(pairs) <= 6, "Six tests suffice to provide all pairs of four binary variables (" + str(len(pairs)) + " given)" + def test_allpairs(): from nose.plugins.skip import SkipTest raise SkipTest("Test is skipped") From a8f70e26b4fb8b74f9be93e42ecec1d456257dc1 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 13 Oct 2014 15:46:17 -0600 Subject: [PATCH 49/53] Removed the TODO section, moved to github issues --- README.rst | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 5f1b385..e8414ba 100644 --- a/README.rst +++ b/README.rst @@ -102,7 +102,7 @@ Known bugs ========== See https://github.com/davidedelvento/qc/issues for a list of known -issues. The Python 3 support seems to be broken at the moment. +issues. One common problem when using automatic shrinking is running out of stack space in the recursion process (the shrink algorithm call @@ -124,24 +124,3 @@ writing your first shrinker) or in qc. Please report the latter to https://github.com/davidedelvento/qc/issues -TODO -==== - -* Print a better summary, e.g. including the total number of tries per - method and other infos (need to investigate how to do it cleanly and - in a way that works both in unittest and nose). - -* Provide the option to not stop in case of failures, and instead - logging and continue (or better handled by the underlying test - framework?? See for a `this StackOverflow question`_ discussion). - -.. _this StackOverflow question: http://stackoverflow.com/questions/4732827/ - -* Improve the current test case reduction from bisection only to - delta-debugging. See the following links for details: - - * http://www.st.cs.uni-saarland.de/dd/ - * http://delta.tigris.org/ - * http://classes.soe.ucsc.edu/cmps290g/Winter04/lectures/flanagan-290g-8.pdf - -* Integration with ``git-bisect`` (maybe). From a45ec7fd2e1a5e9564be803e98e944e69e3ba9d3 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Mon, 13 Oct 2014 15:52:11 -0600 Subject: [PATCH 50/53] Removed another reference to Python 3 (see issue#9 for details) --- README.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.rst b/README.rst index e8414ba..44acf51 100644 --- a/README.rst +++ b/README.rst @@ -69,10 +69,7 @@ at the moment the content is a mere ``__init__.py`` file) into the location of y To make `qc` available to your programs you will have to set the ``PYTHONPATH`` environmental variable or have the ``qc`` directory as a subdirectory of the tree where you are running (for details, see -https://docs.python.org/2/tutorial/modules.html#the-module-search-path if you -use python 2 or https://docs.python.org/3/tutorial/modules.html#the-module-search-path -if you use python 3) - +https://docs.python.org/2/tutorial/modules.html#the-module-search-path Examples ======== From 657a3c05d666488b5959e3e0e6caf17bd5a125a4 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Tue, 14 Oct 2014 10:09:58 -0600 Subject: [PATCH 51/53] mentioning virtualenv in the README --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 44acf51..20afa06 100644 --- a/README.rst +++ b/README.rst @@ -63,13 +63,17 @@ The easy, system-wide way (requires administrative privileges):: sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc If you don't feel ready to commit for a whole system install of this library, or -simply don't have root access on your machine, just copy the ``qc`` directory +simply don't have root access on your machine (and don't want to use virtualenv_), +just copy the ``qc`` directory and its content (as seen in https://github.com/davidedelvento/qc/tree/master/qc at the moment the content is a mere ``__init__.py`` file) into the location of your choice. To make `qc` available to your programs you will have to set the ``PYTHONPATH`` environmental variable or have the ``qc`` directory as a subdirectory of the tree where you are running (for details, see -https://docs.python.org/2/tutorial/modules.html#the-module-search-path +https://docs.python.org/2/tutorial/modules.html#the-module-search-path ) + +.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/virtualenv.html + Examples ======== From 4ed5e2356abfe90b74f8cf345634a0eb7fe82039 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Oct 2014 13:32:16 -0600 Subject: [PATCH 52/53] updating examples and README for any test framework --- README.rst | 24 ++++++++++++++++++------ examples/ex1_unittest.py | 3 ++- examples/ex3_airplane.py | 3 ++- examples/ex4_bookcase.py | 3 ++- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 20afa06..47de614 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,9 @@ Introduction This framework does Random and Combinatorial Testing. It is a Python framework inspired by Haskell's QuickCheck_ and Scala's scalacheck_ (not a port to Python -of those frameworks). +of those frameworks). The framework is not standalone, but works in your own +favorite testing infrastructure: PyUnit (a.k.a. unittest_) nose_ and py.test_ are +all supported. In Combinatorial Testing, `qc` provides a deterministic implementation of all-choices (simply picks all the combinatorial choices of parameters) and all-pairs_ @@ -27,6 +29,9 @@ to find a small, failing test case, very quickly. .. _QuickCheck: http://hackage.haskell.org/package/QuickCheck .. _scalacheck: https://github.com/rickynils/scalacheck .. _all-pairs: https://en.wikipedia.org/wiki/All-pairs_testing +.. _unittest: https://docs.python.org/2/library/unittest.html +.. _nose: https://nose.readthedocs.org/en/latest/ +.. _py.test: http://pytest.org/latest/ More on Random Testing ====================== @@ -78,25 +83,32 @@ https://docs.python.org/2/tutorial/modules.html#the-module-search-path ) Examples ======== +These examples are more to be read than to be run, but of course you want to +run them to see the framework in action (and of course all the failures are +there on purpose...) + ``examples/ex1_unittest.py`` and ``examples/ex1_nose.py`` - These examples provide a simple example (borrowed from scalacheck) + These files provide a simple example (borrowed from scalacheck) on how to use this framework with Python native PyUnit framework (aka unittest module) and with the popular nose framework. Just run ``python examples/example1_unittest.py`` or ``nosetests - examples/example1_nose.py``. Pytest can also run the nose tests - using ``py.test examples/example1_nose.py``. + examples/example1_nose.py``. Py.test can run both the nose test or the unittest + using ``py.test examples/example1_nose.py`` or ``py.test examples/ex1_nose.py`` ``examples/ex2_choices_pairs.py`` TBD ``examples/ex3_airplane.py`` Simple example on how to have qc generate your own custom (random) - objects and how to use them in practice. + objects and how to use them in practice. Run it with either + ``nosetests examples/ex3_airplane.py`` or ``py.test examples/ex3_airplane.py`` ``examples/ex4_bookcase.py`` A more elaborate example, showing the power of automatic shrinking and showing how to write your own shrinker. In this example you can see how qc automagically finds the root cause of the bug, compare - the output with or without shrinking! + the output with or without shrinking! Run it with either + ``nosetests examples/ex4_bookcase.py`` or ``py.test examples/ex4_bookcase.py`` + Known bugs diff --git a/examples/ex1_unittest.py b/examples/ex1_unittest.py index 1fafda5..babeff8 100644 --- a/examples/ex1_unittest.py +++ b/examples/ex1_unittest.py @@ -26,5 +26,6 @@ def testSubstring(self, a, b, c): stop = len(a) + len(b) self.assertEqual(concat[start: stop], b) -main() +if __name__ == "__main__": + main() diff --git a/examples/ex3_airplane.py b/examples/ex3_airplane.py index b35c7c3..a677c5d 100644 --- a/examples/ex3_airplane.py +++ b/examples/ex3_airplane.py @@ -56,5 +56,6 @@ def testFly(self, l): self.cessna172.fly_to(l) self.assertTrue(self.cessna172.has_gas()) -main() +if __name__ == '__main__': + main() diff --git a/examples/ex4_bookcase.py b/examples/ex4_bookcase.py index bf58e4a..3f44d43 100644 --- a/examples/ex4_bookcase.py +++ b/examples/ex4_bookcase.py @@ -57,6 +57,7 @@ def testPutAndTake_shrink(self, bc, book): bc.put(book) self.assertEqual(book, bc.take(book)) -main() +if __name__ == '__main__': + main() From 6052be0428578ce6d56d0bd876ba1d5447d4bcc7 Mon Sep 17 00:00:00 2001 From: Davide Del Vento Date: Thu, 23 Oct 2014 13:35:47 -0600 Subject: [PATCH 53/53] Commit 253090290e1d45b0fb5d02bfe8aec2e74c03885f changed the default values in a way that made this test succeed instead of fail. Fixes issue #10 --- examples/ex4_bookcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ex4_bookcase.py b/examples/ex4_bookcase.py index 3f44d43..528b2bd 100644 --- a/examples/ex4_bookcase.py +++ b/examples/ex4_bookcase.py @@ -26,7 +26,7 @@ def __repr__(self): # This is the qc-related auxiliary method # used to create random objects (bookcases) -def bookcases(nshelves = integers(1,10), book_set=lists()): +def bookcases(nshelves = integers(1,10), book_set=lists(items=integers(1,20))): while True: yield Bookcase(nshelves.next(), book_set.next())