diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ae92d95 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + - "pypy" +install: "pip install --requirement requirements-test.txt --use-mirrors" +script: nosetests \ No newline at end of file diff --git a/README b/README deleted file mode 100644 index 32a2077..0000000 --- a/README +++ /dev/null @@ -1,6 +0,0 @@ -============ -Installation -============ - -sudo easy_install pip -sudo pip install -e git://github.com/dbravender/qc.git#egg=qc diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..47de614 --- /dev/null +++ b/README.rst @@ -0,0 +1,139 @@ +============ + QuickCheck +============ + +.. image:: https://travis-ci.org/davidedelvento/qc.png + :target: https://travis-ci.org/davidedelvento/qc + +Introduction +============ + +This framework does Random and Combinatorial Testing. It is a Python framework +inspired by Haskell's QuickCheck_ and Scala's scalacheck_ (not a port to Python +of those frameworks). The framework is not standalone, but works in your own +favorite testing infrastructure: PyUnit (a.k.a. unittest_) nose_ and py.test_ are +all supported. + +In Combinatorial Testing, `qc` provides a deterministic implementation of all-choices +(simply picks all the combinatorial choices of parameters) and all-pairs_ +algorithms. All-choices simply tests all possible choice of input parameters, so it's +an exhaustive, but very slow technique when there are more than a few parameters with +more than a handful of possible values each. The all-pairs_ algorithm is very +clever and exponentially reduces the running time, while still guaranteeing that each +pair of input parameter is tested for each possible combination of values. + +In Random Testing, `qc` provides many convenient ways of generating test cases, and +a very useful automatic test case reduction. The test case reduction uses binary search +to find a small, failing test case, very quickly. + +.. _QuickCheck: http://hackage.haskell.org/package/QuickCheck +.. _scalacheck: https://github.com/rickynils/scalacheck +.. _all-pairs: https://en.wikipedia.org/wiki/All-pairs_testing +.. _unittest: https://docs.python.org/2/library/unittest.html +.. _nose: https://nose.readthedocs.org/en/latest/ +.. _py.test: http://pytest.org/latest/ + +More on Random Testing +====================== + +Since Random Testing is not popular, you may want to +have a look at the following videos from Professor +John Regehr, University of Utah (less than 15 minutes total) if you are not +familiar with the technique: + +* `Introduction to random testing `_ +* `Why random testing is good (1) `_ +* `Why random testing is good (2) `_ +* `Why random testing is good (3) `_ +* `Why random testing is good (4) `_ + +As you can see, Random Testing is not just randomly feeding your software a random +stream of bytes. It requires more thoughts. The `Udacity course on +testing`_ has units 3 and 4 entirely dedicated to Random Testing, +describing many things you need to know about Random Testing: how to +create valid, good, random test cases (3.25-26 and 4.5-15), mutators +(3.29), oracles (3.30-34), test case reduction (4.5-6), tradeoffs +(4.18-20), and more. Random Testing is also mentioned elsewhere in +the class (and expected to be known in the final exam). It is very +worth watching (the introductory videos linked above are from this +class). + +.. _Udacity course on testing: http://www.udacity.com/overview/Course/cs258/CourseRev/1 + +Installation +============ + +The easy, system-wide way (requires administrative privileges):: + + sudo pip install -e git://github.com/davidedelvento/qc.git#egg=qc + +If you don't feel ready to commit for a whole system install of this library, or +simply don't have root access on your machine (and don't want to use virtualenv_), +just copy the ``qc`` directory +and its content (as seen in https://github.com/davidedelvento/qc/tree/master/qc +at the moment the content is a mere ``__init__.py`` file) into the location of your choice. +To make `qc` available to your programs you will have to set the +``PYTHONPATH`` environmental variable or have the ``qc`` directory as +a subdirectory of the tree where you are running (for details, see +https://docs.python.org/2/tutorial/modules.html#the-module-search-path ) + +.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/virtualenv.html + + +Examples +======== + +These examples are more to be read than to be run, but of course you want to +run them to see the framework in action (and of course all the failures are +there on purpose...) + +``examples/ex1_unittest.py`` and ``examples/ex1_nose.py`` + These files provide a simple example (borrowed from scalacheck) + on how to use this framework with Python native PyUnit framework + (aka unittest module) and with the popular nose framework. Just + run ``python examples/example1_unittest.py`` or ``nosetests + examples/example1_nose.py``. Py.test can run both the nose test or the unittest + using ``py.test examples/example1_nose.py`` or ``py.test examples/ex1_nose.py`` + +``examples/ex2_choices_pairs.py`` TBD + +``examples/ex3_airplane.py`` + Simple example on how to have qc generate your own custom (random) + objects and how to use them in practice. Run it with either + ``nosetests examples/ex3_airplane.py`` or ``py.test examples/ex3_airplane.py`` + +``examples/ex4_bookcase.py`` + A more elaborate example, showing the power of automatic shrinking + and showing how to write your own shrinker. In this example you can + see how qc automagically finds the root cause of the bug, compare + the output with or without shrinking! Run it with either + ``nosetests examples/ex4_bookcase.py`` or ``py.test examples/ex4_bookcase.py`` + + + +Known bugs +========== + +See https://github.com/davidedelvento/qc/issues for a list of known +issues. + +One common problem when using automatic shrinking is running out +of stack space in the recursion process (the shrink algorithm call +itself several times to produce a smaller test case). This may happen +either if there is a bug in qc itself, or if there is a problem in +your test code. You will see an error like:: + + RuntimeError: maximum recursion depth exceeded while calling a Python object + +with a stack trace that shows the recursion tree of the shrinking +method calling itself. To understand what is happening, it is usually +useful to add the ``shrink=False`` option to the ``@forall`` decorator +of the affected test method. In very rare cases it may be necessary to +increase the stack depth with a call to +``sys.setrecursionlimit(NEWDEPTH)``, but do not do it until you +understand that it is really the case for your test. More often than +not, there will be a bug in your code (especially likely if you are +writing your first shrinker) or in qc. Please report the +latter to https://github.com/davidedelvento/qc/issues + + diff --git a/examples/ex1_nose.py b/examples/ex1_nose.py new file mode 100644 index 0000000..7bdf314 --- /dev/null +++ b/examples/ex1_nose.py @@ -0,0 +1,27 @@ +from qc import forall, unicodes + +# This example is adapted from Scala's +# https://github.com/rickynils/scalacheck +# and we are pretending to test the string +# concatenation, slicing and the len builtin + +@forall(tries=2000, a=unicodes(), b=unicodes()) +def testStartswith(a, b): + concat = a + b + assert concat.startswith(a) + +@forall(tries=10, a=unicodes(), b=unicodes()) +def testConcatenation(a, b): + concat = a + b + # the following is meant to fail as an example of a failure + assert len(concat) > len(a) + assert len(concat) > len(b) + +@forall(a=unicodes(), b=unicodes(), c=unicodes()) +def testSubstring(a, b, c): + concat = a + b + c + start = len(a) + stop = len(a) + len(b) + assert concat[start: stop] == b + + diff --git a/examples/ex1_unittest.py b/examples/ex1_unittest.py new file mode 100644 index 0000000..babeff8 --- /dev/null +++ b/examples/ex1_unittest.py @@ -0,0 +1,31 @@ +from unittest import TestCase, main +from qc import forall, unicodes + +# This example is adapted from Scala's +# https://github.com/rickynils/scalacheck +# and we are pretending to test the string +# concatenation, slicing and the len builtin + +class TestString(TestCase): + @forall(tries=2000, a=unicodes(), b=unicodes()) + def testStartswith(self, a, b): + concat = a + b + self.assertTrue(concat.startswith(a)) + + @forall(tries=10, a=unicodes(), b=unicodes()) + def testConcatenation(self, a, b): + concat = a + b + # the following is meant to fail as an example of a failure + self.assertTrue(len(concat) > len(a)) + self.assertTrue(len(concat) > len(b)) + + @forall(a=unicodes(), b=unicodes(), c=unicodes()) + def testSubstring(self, a, b, c): + concat = a + b + c + start = len(a) + stop = len(a) + len(b) + self.assertEqual(concat[start: stop], b) + +if __name__ == "__main__": + main() + diff --git a/examples/ex3_airplane.py b/examples/ex3_airplane.py new file mode 100644 index 0000000..a677c5d --- /dev/null +++ b/examples/ex3_airplane.py @@ -0,0 +1,61 @@ +# Location is a (lame) class needed by the +# class under test. In real life it will be +# a non-lame object you really need +class Location(object): + def __init__(self, lat, lon, height): + self.lat=lat + self.lon=lon + self.h=height + +# This is the class under test +class Plane(object): + def __init__(self): + self.location = Location(46.22, -112.1, 4968) + def fly_to(self, loc): + if (-75 < loc.lon < -70 and + 20 < loc.lat < 30 and + loc.h < 6000): + raise Exception("Plane has disappeared in the Bermuda triangle") + else: + self.location = loc + def has_gas(self): + return True + +from unittest import TestCase, main +from qc import forall, floats + +# This is the qc-related auxiliary method +# used to create random objects. In this +# simple example it is overkill, but the +# purpose here is just to show how it is +# done: in real life you may have several +# testing methods requiring random locations +# and defining the method this way will +# allow @forall to be able to inject +# Location's everywhere you need them +def locations(lat = floats(-90,+90), + lon = floats(-180, +180), + height = floats(0, 30000)): + while True: + yield Location(lat.next(), lon.next(), height.next()) + + +# This is the PyUnit test case +# In this example we are just +# flying the plane to random locations +# and assert that it does not run out +# of gas. In real life you would assert +# that it *does* run out of gas, but it +# does *not* fall apart in other ways. +class TestPlane(TestCase): + def setUp(self): + self.cessna172 = Plane() + + @forall(tries=50000, l=locations()) + def testFly(self, l): + self.cessna172.fly_to(l) + self.assertTrue(self.cessna172.has_gas()) + +if __name__ == '__main__': + main() + diff --git a/examples/ex4_bookcase.py b/examples/ex4_bookcase.py new file mode 100644 index 0000000..528b2bd --- /dev/null +++ b/examples/ex4_bookcase.py @@ -0,0 +1,63 @@ +# This is at the same time the class under test, +# and the class that you need to randomly generate +# For simplicity a book is uniquely identified +# by a just a positive integer +class Bookcase(object): + def __init__(self, num_shelves, books): + self.num_shelves = num_shelves + self.books = books + def put(self, book): + if not self.full(): + self.books.append(book) + def take(self, book): + if book in self.books: + self.books.remove(book) + if 13 in self.books: # artificially introduced bug + return 13 + else: + return book + def full(self): + return False # This bookcase has infinite capacity + def __repr__(self): + return "Bookcase with " + str(self.num_shelves) + " shelves and containing books: " + self.books.__repr__() + +from unittest import TestCase, main +from qc import forall, integers, lists, qc_shrink + +# This is the qc-related auxiliary method +# used to create random objects (bookcases) +def bookcases(nshelves = integers(1,10), book_set=lists(items=integers(1,20))): + while True: + yield Bookcase(nshelves.next(), book_set.next()) + +# This is the custom shrinking function we have to provide +# to qc. It has to decide the logic by which we want the +# test case to be shrunk. In this case, we just shrink +# the list of books, using qc default's shrink function. +# Note that we are not shrinking the number of shelves. +# This is the common pattern to create custom shrinker: +# decide what has to change and what has not. Call +# qc's shrinker on the relevant objects (if they are +# supported, otherwise write your own shrinker for those) +def shrink_bookcase(bookcase): + if isinstance(bookcase, Bookcase): + for shrinked_books in qc_shrink(bookcase.books): + yield Bookcase(bookcase.num_shelves, shrinked_books) + # else we do not shrink + +# This is the PyUnit test case +class TestBookcase(TestCase): + @forall(bc=bookcases(), book=integers()) + def testPutAndTake_noshrink(self, bc, book): + bc.put(book) + self.assertEqual(book, bc.take(book)) + + @forall(bc=bookcases(), book=integers(), custom_shrink=shrink_bookcase) + def testPutAndTake_shrink(self, bc, book): + bc.put(book) + self.assertEqual(book, bc.take(book)) + +if __name__ == '__main__': + main() + + diff --git a/qc/__init__.py b/qc/__init__.py index 3713be2..3ae7587 100644 --- a/qc/__init__.py +++ b/qc/__init__.py @@ -1,32 +1,49 @@ # Copyright (c) 2009-2011, Dan Bravender +# Copyright (c) 2012-2014, Davide Del Vento -import random -import os -import functools +import random, math +import os, sys, warnings +import itertools, functools -def integers(low=0, high=100): +def integers(low=-sys.maxint-1, high=sys.maxint): '''Endlessly yields random integers between (inclusively) low and high. Yields low then high first to test boundary conditions. ''' yield low yield high + for i in (-1,0,1): + if low < i < high: + yield i while True: yield random.randint(low, high) -def floats(low=0.0, high=100.0): +def floats(low=-sys.float_info.max, high=sys.float_info.max, special=True): '''Endlessly yields random floats between (inclusively) low and high. Yields low then high first to test boundary conditions. ''' - yield low - yield high - while True: - yield random.uniform(low, high) + yield float(low) + yield float(high) + for i in (-1.0, -sys.float_info.min, 0.0, sys.float_info.min, 1.0): + if low < i < high: + yield i + if special: + yield float('nan') + yield float('inf') + yield float('-inf') + + if low == -sys.float_info.max and high == sys.float_info.max: + while True: # uniform does not work for such a large range + yield random.uniform(-2, 2) * 10 ** random.randint(-sys.float_info.max_10_exp+1, sys.float_info.max_10_exp-1) + else: + while True: # TODO it should probably detect a large range and use an approach similar to the previous one + yield random.uniform(low, high) def lists(items=integers(), size=(0, 100)): '''Endlessly yields random lists varying in size between size[0] and size[1]. Yields a list of the low size and the high size first to test boundary conditions. ''' + assert size[0] >=0, "list size must be non-negative" yield [items.next() for _ in xrange(size[0])] yield [items.next() for _ in xrange(size[1])] while True: @@ -37,6 +54,7 @@ def tuples(items=integers(), size=(0, 100)): and size[1]. Yields a tuple of the low size and the high size first to test boundary conditions. ''' + assert size[0] >=0, "tuple size must be non-negative" yield tuple([items.next() for _ in xrange(size[0])]) yield tuple([items.next() for _ in xrange(size[1])]) while True: @@ -79,20 +97,134 @@ def objects(_object_class, _fields={}, *init_args, **init_kwargs): setattr(obj, k, v.next()) yield obj -def forall(tries=100, **kwargs): +def qc_shrink(something): + try: + if len(something) == 0: # never shrink a zero-len object, since it + return # will lead to infinite recursion + if len(something) == 1: # if single-object collection + yield something[:0] # try the empty collection + return # if it works, no need to try the single-object + # collection again + l = len(something)/2 + yield something[:l] + yield something[l:] + except TypeError: + pass + try: + if abs(something) >= 2 and not math.isinf(something): + yield something/2 + except TypeError: + pass + +class QCAssertionError(AssertionError): + def __init__(self, error, *args, **kwargs): + super(QCAssertionError, self).__init__(*args, **kwargs) + self.parent_error=error + + def __str__(self): + return self.message + str(self.parent_error) + +def call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **random_kwargs): + try: + f(*inargs, **random_kwargs) + except AssertionError as e: # shrink only when there is AssertionErrors, in other cases is ad infinitum recursion + if tryshrink: + for k in random_kwargs: + for s in custom_shrink(random_kwargs[k]): + shrinked_kwargs = random_kwargs.copy() + shrinked_kwargs[k] = s + if forall.verbose or os.environ.has_key('QC_VERBOSE'): + from pprint import pprint + pprint(random_kwargs) + call_and_shrink(f, tryshrink, seed, custom_shrink, *inargs, **shrinked_kwargs) + if sys.version_info[0] < 3: + raise QCAssertionError(e, str(random_kwargs) + + " (from seed " +str(seed) + ") caused a FAIL: "), None, sys.exc_traceback + else: + raise QCAssertionError(e, "{0}, from seed {1}, caused a FAILURE\n".format( + random_kwargs, seed)).with_traceback(e.__traceback__) + +def list_of_tests(num_of_vars): + if num_of_vars == 1: + return ['0','1'] + elif num_of_vars <= 3: + return ['000','011','100','110'] + else: + from math import factorial + coverage = -1 # how many variables we can cover with not tests + num_of_tests = 4 # how many tests we need + while coverage < num_of_vars: # we want to cover all the ones we have + num_of_tests += 2 + coverage = factorial(num_of_tests) / ( 2 * (factorial(num_of_tests/2) ** 2) ) + + # at this point we know that with num_of_tests tests we can cover the num_of_vars we have + + # To build the test cases, we use the following which is one way to build the minimum + # necessary and sufficient set to to cover coverage variable. + # The proof is simple, but too large for this margin + list_of_cases = [] + for t in range(2**(num_of_tests-1)): + case=bin(t)[2:].zfill(num_of_tests) + if case.count('0') == num_of_tests / 2: # == case.count('1'): + list_of_cases.append(case) + + # For going from cases to tests, we take the transpose + lot = [] # list of tests + for i in range(num_of_tests): + lot.append(''.join([case[i] for case in list_of_cases])) + + return lot + +def allpairs(**kwargs): + def wrap(f): + @functools.wraps(f) + def wrapped(*inargs, **inkwargs): + for values in kwargs.values(): + if len(values) != 2: + raise NotImplementedError("At the moment only binary arguments are supported") + for curr_test in list_of_tests(len(kwargs.keys())): + new_kwargs = {} + for i, (key, values) in enumerate(zip(kwargs.keys(), kwargs.values())): + new_kwargs[key] = values[int(curr_test[i])] + f(*inargs, **new_kwargs) + return wrapped + return wrap + +def allchoices(**kwargs): + warnings.warn("Testing all choices may take very long time, because there may be too many", RuntimeWarning) + def wrap(f): + @functools.wraps(f) + def wrapped(*inargs, **inkwargs): + for values in itertools.product(*tuple(kwargs.values())): + new_kwargs={} + for i,k in enumerate(kwargs.keys()): + new_kwargs[k]=values[i] + new_kwargs.update(**inkwargs) + f(*inargs, **new_kwargs) + return wrapped + return wrap + +def forall(tries=100, shrink=True, seed=None, custom_shrink=qc_shrink, **kwargs): + if seed is None: + try: + seed = hash(os.urandom(16)) + except NotImplementedError: + seed = random.random() + random.seed(seed) def wrap(f): @functools.wraps(f) def wrapped(*inargs, **inkwargs): for _ in xrange(tries): random_kwargs = (dict((name, gen.next()) for (name, gen) in kwargs.iteritems())) - if forall.verbose or os.environ.has_key('QC_VERBOSE'): - from pprint import pprint - pprint(random_kwargs) random_kwargs.update(**inkwargs) - f(*inargs, **random_kwargs) + call_and_shrink(f, shrink, seed, custom_shrink, *inargs, **random_kwargs) + if forall.printsummary: + from pprint import pprint + pprint(f.__name__+": passed "+str(tries)+" tests [OK]") return wrapped return wrap +forall.printsummary = False forall.verbose = False # if enabled will print out the random test cases __all__ = ['integers', 'floats', 'lists', 'tuples', 'unicodes', 'characters', 'objects', 'forall'] diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..f3c7e8e --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +nose diff --git a/tests/test_qc.py b/tests/test_qc.py index c3f23e6..43a11bb 100644 --- a/tests/test_qc.py +++ b/tests/test_qc.py @@ -1,27 +1,167 @@ -from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall +from qc import integers, floats, unicodes, characters, lists, tuples, dicts, objects, forall, allchoices, allpairs, qc_shrink, call_and_shrink +import math, sys, warnings +from nose.tools import raises + +# Integers @forall(tries=10, i=integers()) def test_integers(i): - assert type(i) == int - assert i >= 0 and i <= 100 + assert type(i) == int, "expected an int, instead got a " + str(type(i)) -@forall(tries=10, l=lists(items=integers())) -def test_a_int_list(l): - assert type(l) == list +@forall(tries=10, i=integers(low=0, high=100)) +def test__nonnegative_integers(i): + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + assert 0 <= i <= 100, "not the expected range" + +@forall(tries=10, i=integers(low=-100, high=0)) +def test__nonpositive_integers(i): + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + assert -100 <= i <= 0, "not the expected range" -@forall(tries=10, l=tuples(items=integers())) -def test_a_int_tuple(l): - assert type(l) == tuple +# Floats @forall(tries=10, i=floats()) def test_floats(i): - assert type(i) == float - assert i >= 0.0 and i <= 100.0 + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + +@forall(tries=10, i=floats(low=0, high=100)) +def test_nonnegative_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert 0 <= i <= 100 or math.isnan(i) or math.isinf(i), "not the expected range" + +@forall(tries=10, i=floats(low=-100, high=0)) +def test_nonpositive_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert -100 <= i <= 0 or math.isnan(i) or math.isinf(i), "not the expected range" + +@forall(tries=10, i=floats(low=-100, high=-100, special=False)) +def test_nonspecial_floats(i): + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + assert -100 <= i <= 100, "not the expected range" + +def test_floats_coverage(): + negative_inf = False + negative_max = False + negative_large = False + negative_one = False + negative_small = False + negative_min = False + zero = False + positive_min = False + positive_small = False + positive_one = False + positive_large = False + positive_max = False + positive_inf = False + nan = False + + gen = floats() + for i in range(100): # 100 opportunities should be plenty + f = gen.next() + print "value = ", f + if math.isinf(f) and f < 0: + negative_inf = True + elif f == -sys.float_info.max: + negative_max = True + elif -sys.float_info.max / 2 < f < -2: + negative_large = True + elif f == -1.0: + negative_one = True + elif -0.5 < f < -sys.float_info.min * 2: + negative_small = True + elif f == -sys.float_info.min: + negative_min = True + elif f == 0: + zero = True + elif f == sys.float_info.min: + positive_min = True + elif sys.float_info.min * 2 < f < 0.5: + positive_small = True + elif f == 1.0: + positive_one = True + elif 2 < f < sys.float_info.max / 2: + positive_large = True + elif f == sys.float_info.max: + positive_max = True + elif math.isinf(f) and f > 0: + positive_inf = True + elif math.isnan(f): + nan = True + + assert negative_inf == True + assert negative_max == True + assert negative_large == True + assert negative_one == True + assert negative_small == True + assert negative_min == True + assert zero == True + assert positive_min == True + assert positive_small == True + assert positive_one == True + assert positive_large == True + assert positive_max == True + assert positive_inf == True + assert nan == True + +# Lists and tuples + +@forall(tries=10, l=lists(items=integers())) +def test_a_int_list(l): + assert type(l) == list, "expected a list, instead got a " + str(type(l)) + for i in l: + assert type(i) == int, "expected an int, instead got a " + str(type(i)) + +@forall(tries=10, t=tuples(items=integers())) +def test_a_int_tuple(t): + assert type(t) == tuple, "expected a tuple, instead got a " + str(type(t)) + for i in t: + assert type(i) == int, "expected an int, instead got a " + str(type(i)) @forall(tries=10, l=lists(items=floats())) def test_a_float_list(l): - assert type(l) == list - assert reduce(lambda x,y: x and type(y) == float, l, True) + assert type(l) == list, "expected a list, instead got a " + str(type(l)) + for i in l: + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + +@forall(tries=10, t=tuples(items=floats())) +def test_a_float_tuple(t): + assert type(t) == tuple, "expected a tuple, instead got a " + str(type(t)) + for i in t: + assert type(i) == float, "expected a float, instead got a " + str(type(i)) + +@forall(tries=10, l=lists(items=integers(), size=(10, 50))) +def test_lists_size(l): + assert 10 <= len(l) <= 50, "list of unexpected size: " + str(len(l)) + +@forall(tries=10, t=tuples(items=integers(), size=(10, 50))) +def test_tuples_size(t): + assert 10 <= len(t) <= 50, "tuple of unexpected size: " + str(len(t)) + +@raises(AssertionError) +@forall(tries=10, t=tuples(size=(-10, 10))) +def test_negative_tuples_size(t): + pass # must raise an exception for negative sizes + +@raises(AssertionError) +@forall(tries=10, t=lists(size=(-10, 10))) +def test_negative_lists_size(t): + pass # must raise an exception for negative sizes + +@raises(Exception) +@forall(tries=10, t=tuples(size=(20, 10))) +def test_wrong_tuples_size(t): + pass # must raise an exception for min > max + +@raises(Exception) +@forall(tries=10, t=lists(size=(20, 10))) +def test_wrong_lists_size(t): + pass # must raise an exception for min > max + +# Unicode and characters + +@forall(tries=10, c=characters()) +def test_characters(c): + assert len(c) == 1 @forall(tries=10, ul=lists(items=unicodes())) def test_unicodes_list(ul): @@ -29,14 +169,6 @@ def test_unicodes_list(ul): if len(ul): assert type(ul[0]) == unicode -@forall(tries=10, l=lists(items=integers(), size=(10, 50))) -def test_lists_size(l): - assert len(l) <= 50 and len(l) >= 10 - -@forall(tries=10, l=tuples(items=integers(), size=(10, 50))) -def test_tuples_size(l): - assert len(l) <= 50 and len(l) >= 10 - @forall(tries=10, u=unicodes()) def test_unicodes(u): assert type(u) == unicode @@ -56,19 +188,19 @@ def test_a_tupled_list(l): for x in l: assert type(x[0]) == int and type(x[1]) == unicode +# Other tests + @forall(tries=10, x=integers(), y=integers()) -def test_addition_commutative(x, y): +def test_integer_addition_commutative(x, y): assert x + y == y + x @forall(tries=10, l=lists()) def test_reverse_reverse(l): assert list(reversed(list(reversed(l)))) == l -@forall(tries=10, c=characters()) -def test_characters(c): - assert len(c) == 1 +# Dictionaries -def kv_unicode_integers(): +def kv_unicode_integers(): # key-value helper u = unicodes() i = integers() while True: @@ -80,7 +212,7 @@ def test_dicts(d): assert type(x) == unicode assert type(y) == int -def kv_unicodes_lists(): +def kv_unicodes_lists(): # key-value helper u = unicodes() l = lists() while True: @@ -93,6 +225,40 @@ def test_dicts_size(d): assert type(x) == unicode assert type(y) == list +# Shrinking + +@raises(StopIteration) +def test_shrink_empty_list(): + qc_shrink([]).next() # there must be no next, an Exception must be raised + +def test_shrink_single_element_list(): + repeated = False + l = [0] + for x in qc_shrink(l): + if x == l: + repeated = True + assert repeated == False, "Shrink must not repeat things already seen" + +@forall(full_l=lists()) +def test_shrink_lists(full_l): + for sub_l in qc_shrink(full_l): + assert len(sub_l) <= len(full_l)/2 + 1, "list must be shrunk in half" + +@forall(tries=100, full_i=integers()) +def test_shrink_integers(full_i): + for i in qc_shrink(full_i): + assert isinstance(i, int), "ints must be shrunk to ints" + assert abs(i) <= abs(full_i)/2 + 1, "ints must be shrunk to smaller ints" + assert cmp(i,0) == cmp(full_i, 0), "shrink must not change sign of ints" + +@forall(tries=1000, full_f=floats()) +def test_shrink_floats(full_f): + for f in qc_shrink(full_f): + assert isinstance(f, float), "floats must be shrunk to floats" + assert abs(full_f) > 1 # TODO, check if something is appropriate for smaller number + assert abs(f) <= abs(full_f)/2 + 1, "shrunk fload must be smaller than parent" + assert cmp(f,0) == cmp(full_f, 0), "shrink must not change sign of floats" + @forall(tries=10, i=integers(low=0, high=10)) def each_integer_from_0_to_10(i, target_low, target_high): assert i >= target_low and i<= target_high @@ -105,6 +271,26 @@ class TestClass(object): def __init__(self, arg): self.arg_from_init = arg +# forall wrapper + +def constant(value): + while True: + yield value + +def test_forall(): + @forall(tries=11, a=constant(7.5)) + def myf(arr, a): + arr.append(a) + arr = [] + myf(arr) + assert arr == [7.5]*11, "arr is" + str(arr) + "instead of the expected" + str([7.5]*11) + @forall(tries=11, a=constant(7.5)) + def myf(a, myarr=None): + myarr.append(a) + arr = [] + myf(myarr=arr) + assert arr == [7.5]*11, "arr is" + str(arr) + "instead of the expected" + str([7.5]*11) + @forall(tries=10, obj=objects(TestClass, {'an_int': integers(), 'a_float': floats()}, unicodes())) def test_objects(obj): assert type(obj) == TestClass @@ -112,3 +298,80 @@ def test_objects(obj): assert type(obj.a_float) == float assert type(obj.arg_from_init) == unicode +# All choices + +def test_allchoices(): + arr = [] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # There are only 3 * 3 * 2 = 18 choices + @allchoices(a=(-1, 0, 1), b=('x', 'y', 'z'), c=(True, False)) + def myf(arr, a,b,c): + arr.append((a,b,c)) + myf(arr) + # explicit is better than implicit, avoid logic in test, please bear the following + arr.remove((-1, 'x', True)) + arr.remove((-1, 'y', True)) + arr.remove((-1, 'z', True)) + arr.remove((0, 'x', True)) + arr.remove((0, 'y', True)) + arr.remove((0, 'z', True)) + arr.remove((1, 'x', True)) + arr.remove((1, 'y', True)) + arr.remove((1, 'z', True)) + # + arr.remove((-1, 'x', False)) + arr.remove((-1, 'y', False)) + arr.remove((-1, 'z', False )) + arr.remove((0, 'x', False)) + arr.remove((0, 'y', False)) + arr.remove((0, 'z', False)) + arr.remove((1, 'x', False)) + arr.remove((1, 'y', False)) + arr.remove((1, 'z', False)) + assert len(arr) == 0, "arr is" +str( arr) + +# All pairs + +def pairs_search(pairs, a, b, expected_values): + for av,bv in expected_values: + found = False + for case in pairs: + if case[a] == av and case[b] == bv: + found = True + assert found, "Pair (" + a + "=" + str(av) + ", " + b + "=" + str(bv) + ") not found in" + str(pairs) + +def test_all_binary_pairs(): + pairs=[] + @allpairs(a=(-1,1), b=(True, False), c=('A', 'B'), d=(0.0, 1.0)) + def mypairs(arr, a, b, c, d): + arr.append({'a':a, 'b':b, 'c':c, 'd':d}) + mypairs(pairs) + + # The following it should be implicitly covered by the pairs_search below, but explicit is better than implicit... + assert len(pairs) > 4, "Cannot provide all pairs of four binary variables with four or less tests (" + str(len(pairs)) + " given)" + + # explicit is better than implicit, wanted to make this test more explicit, but here it is at least for now and + # it does test correctness + pairs_search(pairs, 'a', 'b', ((-1, True), (-1, False), (1, True), (1, False))) + pairs_search(pairs, 'a', 'c', ((-1, "A"), (-1, "B"), (1, "A"), (1, "B"))) + pairs_search(pairs, 'a', 'd', ((-1, 0.0), (-1, 1.0), (1, 0.0), (1, 1.0))) + + pairs_search(pairs, 'b', 'c', ((True, 'A'), (True, 'B'), (False, 'A'), (False, 'B'))) + pairs_search(pairs, 'b', 'd', ((True, 0.0), (True, 1.0), (False, 0.0), (False, 1.0))) + + pairs_search(pairs, 'c', 'd', (('A', 0.0), ('A', 1.0), ('B', 0.0), ('B', 1.0))) + + # The following tests that the implementation is efficient + assert len(pairs) <= 6, "Six tests suffice to provide all pairs of four binary variables (" + str(len(pairs)) + " given)" + +def test_allpairs(): + from nose.plugins.skip import SkipTest + raise SkipTest("Test is skipped") + pairs = [] + @allpairs(x=('x', 'y', 'z'), p=(0,1), q=(True, False)) + def mypairs(arr, x, p, q): + arr.append({'x':x, 'p':p, 'q':q}) + mypairs(pairs) + + pairs_search(pairs, 'x', 'q', (('x', True), ('y', True), ('z', True), ('x', False), ('y', False), ('z', False))) + # and other pairs