diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ linux-build win32-build test/functional/config.ini +test/util/buildenv.py test/cache/* !src/leveldb*/Makefile diff --git a/Makefile.am b/Makefile.am --- a/Makefile.am +++ b/Makefile.am @@ -201,7 +201,7 @@ $(LCOV) -r $< "/usr/include/*" -o $@ functional_test.info: test_bitcoin_filtered.info - -@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_RPC_TESTS) + -@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) $(LCOV) -c -d $(abs_builddir)/src --t functional-tests -o $@ $(LCOV) -z -d $(abs_builddir)/src $(LCOV) -z -d $(abs_builddir)/src/leveldb @@ -231,8 +231,52 @@ EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST += \ + test/util/bctest.py \ + test/util/bitcoin-util-test.py \ + test/util/data/bitcoin-util-test.json \ + test/util/data/blanktxv1.hex \ + test/util/data/blanktxv1.json \ + test/util/data/blanktxv2.hex \ + test/util/data/blanktxv2.json \ + test/util/data/tt-delin1-out.hex \ + test/util/data/tt-delin1-out.json \ + test/util/data/tt-delout1-out.hex \ + test/util/data/tt-delout1-out.json \ + test/util/data/tt-locktime317000-out.hex \ + test/util/data/tt-locktime317000-out.json \ + test/util/data/tx394b54bb.hex \ + test/util/data/txcreate1.hex \ + test/util/data/txcreate1.json \ + test/util/data/txcreate2.hex \ + test/util/data/txcreate2.json \ + test/util/data/txcreatedata1.hex \ + test/util/data/txcreatedata1.json \ + test/util/data/txcreatedata2.hex \ + test/util/data/txcreatedata2.json \ + test/util/data/txcreatedata_seq0.hex \ + test/util/data/txcreatedata_seq0.json \ + test/util/data/txcreatedata_seq1.hex \ + test/util/data/txcreatedata_seq1.json \ + test/util/data/txcreatemultisig1.hex \ + test/util/data/txcreatemultisig1.json \ + test/util/data/txcreatemultisig2.hex \ + test/util/data/txcreatemultisig2.json \ + test/util/data/txcreateoutpubkey1.hex \ + test/util/data/txcreateoutpubkey1.json \ + test/util/data/txcreatescript1.hex \ + test/util/data/txcreatescript1.json \ + test/util/data/txcreatescript2.hex \ + test/util/data/txcreatescript2.json \ + test/util/data/txcreatesignv1.hex \ + test/util/data/txcreatesignv1.json \ + test/util/data/txcreatesignv2.hex + CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) +# This file is problematic for out-of-tree builds if it exists. +DISTCLEANFILES = test/util/buildenv.pyc + .INTERMEDIATE: $(COVERAGE_INFO) DISTCHECK_CONFIGURE_FLAGS = --enable-man diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -125,10 +125,10 @@ [use_bench=$enableval], [use_bench=yes]) -AC_ARG_ENABLE([extended-rpc-tests], - AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]), - [use_extended_rpc_tests=$enableval], - [use_extended_rpc_tests=no]) +AC_ARG_ENABLE([extended-functional-tests], + AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]), + [use_extended_functional_tests=$enableval], + [use_extended_functional_tests=no]) AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], @@ -530,8 +530,8 @@ ]) fi -if test x$use_extended_rpc_tests != xno; then - AC_SUBST(EXTENDED_RPC_TESTS, -extended) +if test x$use_extended_functional_tests != xno; then + AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi if test x$use_lcov = xyes; then @@ -1277,9 +1277,12 @@ AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) -AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py test/functional/config.ini]) +AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini]) +AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) +AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) +AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -1327,8 +1330,8 @@ dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows case ${OS} in *Windows*) - sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config2.ini - mv test/functional/config2.ini test/functional/config.ini + sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config-2.ini + mv test/functional/config-2.ini test/functional/config.ini ;; esac diff --git a/contrib/rpm/bitcoin.spec b/contrib/rpm/bitcoin.spec --- a/contrib/rpm/bitcoin.spec +++ b/contrib/rpm/bitcoin.spec @@ -311,9 +311,7 @@ %check make check -pushd src -srcdir=. test/bitcoin-util-test.py -popd +srcdir=src test/bitcoin-util-test.py test/functional/test_runner.py --extended %post libs -p /sbin/ldconfig diff --git a/src/Makefile.test.include b/src/Makefile.test.include --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -10,48 +10,6 @@ TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) - -EXTRA_DIST += \ - test/bctest.py \ - test/bitcoin-util-test.py \ - test/data/bitcoin-util-test.json \ - test/data/blanktxv1.hex \ - test/data/blanktxv1.json \ - test/data/blanktxv2.hex \ - test/data/blanktxv2.json \ - test/data/tt-delin1-out.hex \ - test/data/tt-delin1-out.json \ - test/data/tt-delout1-out.hex \ - test/data/tt-delout1-out.json \ - test/data/tt-locktime317000-out.hex \ - test/data/tt-locktime317000-out.json \ - test/data/tx394b54bb.hex \ - test/data/txcreate1.hex \ - test/data/txcreate1.json \ - test/data/txcreate2.hex \ - test/data/txcreate2.json \ - test/data/txcreatedata1.hex \ - test/data/txcreatedata1.json \ - test/data/txcreatedata2.hex \ - test/data/txcreatedata2.json \ - test/data/txcreatedata_seq0.hex \ - test/data/txcreatedata_seq0.json \ - test/data/txcreatedata_seq1.hex \ - test/data/txcreatedata_seq1.json \ - test/data/txcreatemultisig1.hex \ - test/data/txcreatemultisig1.json \ - test/data/txcreatemultisig2.hex \ - test/data/txcreatemultisig2.json \ - test/data/txcreateoutpubkey1.hex \ - test/data/txcreateoutpubkey1.json \ - test/data/txcreatescript1.hex \ - test/data/txcreatescript1.json \ - test/data/txcreatescript2.hex \ - test/data/txcreatescript2.json \ - test/data/txcreatesignv1.hex \ - test/data/txcreatesignv1.json \ - test/data/txcreatesignv2.hex - JSON_TEST_FILES = \ test/data/script_tests.json \ test/data/base58_keys_valid.json \ @@ -196,9 +154,6 @@ CLEANFILES += $(CLEAN_BITCOIN_TEST) -# This file is problematic for out-of-tree builds if it exists. -DISTCLEANFILES += test/buildenv.pyc - bitcoin_test: $(TEST_BINARY) bitcoin_test_check: $(TEST_BINARY) FORCE @@ -208,8 +163,8 @@ rm -f $(CLEAN_BITCOIN_TEST) $(test_test_bitcoin_OBJECTS) $(TEST_BINARY) check-local: - @echo "Running test/bitcoin-util-test.py..." - $(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(PYTHON) $(srcdir)/test/bitcoin-util-test.py + @echo "Running test/util/bitcoin-util-test.py..." + $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check diff --git a/src/test/README.md b/src/test/README.md --- a/src/test/README.md +++ b/src/test/README.md @@ -50,12 +50,3 @@ For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: [http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). - -### bitcoin-util-test.py - -The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just bitcoin-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory: - -``` -test/bitcoin-util-test.py --srcdir=[current directory] - -``` diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py deleted file mode 100755 --- a/src/test/bitcoin-util-test.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division, print_function, unicode_literals -import os -import bctest -import buildenv -import argparse -import logging - -help_text = """Test framework for bitcoin utils. - -Runs automatically during `make check`. - -Can also be run manually from the src directory by specifying the source directory: - -test/bitcoin-util-test.py --srcdir='srcdir' [--verbose] -""" - -if __name__ == '__main__': - # Try to get the source directory from the environment variables. This will - # be set for `make check` automated runs. If environment variable is not set, - # then get the source directory from command line args. - try: - srcdir = os.environ["srcdir"] - verbose = False - except: - parser = argparse.ArgumentParser(description=help_text) - parser.add_argument('-s', '--srcdir') - parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() - srcdir = args.srcdir - verbose = args.verbose - - if verbose: - level = logging.DEBUG - else: - level = logging.ERROR - formatter = '%(asctime)s - %(levelname)s - %(message)s' - # Add the format/level to the logger - logging.basicConfig(format=formatter, level=level) - - bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/buildenv.py.in b/src/test/buildenv.py.in deleted file mode 100644 --- a/src/test/buildenv.py.in +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -exeext="@EXEEXT@" diff --git a/test/README.md b/test/README.md --- a/test/README.md +++ b/test/README.md @@ -1,29 +1,32 @@ -The [functional](/test/functional/) folder contains a script test_runner - to call multiple functional tests from its folder. +This directory contains integration tests that test bitcoind and its +utilities in their entirety. It does not contain unit tests, which +can be found in [/src/test](/src/test), [/src/wallet/test](/src/wallet/test), +etc. -Every pull request to the bitcoin repository is built and run through -the regression test suite. You can also run all or only individual -tests locally. +There are currently two sets of tests in this directory: -Test dependencies -================= -Before running the tests, the following must be installed. +- [functional](/test/functional) which test the functionality of +bitcoind and bitcoin-qt by interacting with them through the RPC and P2P +interfaces. +- [util](test/util) which tests the bitcoin utilities, currently only +bitcoin-tx. -Unix ----- -The python3-zmq library is required. On Ubuntu or Debian it can be installed via: -``` -sudo apt-get install python3-zmq -``` +The util tests are run as part of `make check` target. The functional +tests are run by the travis continuous build process whenever a pull +request is opened. Both sets of tests can also be run locally. -OS X ------- -``` -pip3 install pyzmq -``` +Functional Test dependencies +============================ +The ZMQ functional test requires a python ZMQ library. To install it: -Running tests -============= +- on Unix, run `sudo apt-get install python3-zmq` +- on mac OS, run `pip3 install pyzmq` + +Running tests locally +===================== + +Functional tests +---------------- You can run any single test by calling @@ -90,9 +93,15 @@ killall bitcoind ``` -Writing tests -============= -You are encouraged to write tests for new or existing features. +Util tests +---------- + +Util tests can be run locally by running `test/util/bitcoin-util-test.py`. +Use the `-v` option for verbose output. + +Writing functional tests +======================== -Further information about the test framework and individual functional +You are encouraged to write functional tests for new or existing features. +Further information about the functional test framework and individual tests is found in [test/functional](/test/functional). diff --git a/src/test/bctest.py b/test/util/bctest.py rename from src/test/bctest.py rename to test/util/bctest.py --- a/src/test/bctest.py +++ b/test/util/bctest.py @@ -24,14 +24,14 @@ raise NotImplementedError("Don't know how to compare %s" % fmt) -def bctest(testDir, testObj, exeext): +def bctest(testDir, testObj, buildenv): """Runs a single test, comparing output and RC to expected output and RC. Raises an error if input can't be read, executable fails, or output/RC are not as expected. Error is caught by bctester() and reported. """ # Get the exec names and arguments - execprog = testObj['exec'] + exeext + execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext execargs = testObj['args'] execrun = [execprog] + execargs @@ -130,7 +130,7 @@ for testObj in input_data: try: - bctest(testDir, testObj, buildenv.exeext) + bctest(testDir, testObj, buildenv) logging.info("PASSED: " + testObj["description"]) except: logging.info("FAILED: " + testObj["description"]) diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py new file mode 100755 --- /dev/null +++ b/test/util/bitcoin-util-test.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +from __future__ import division, print_function, unicode_literals +import os +import sys +import argparse +import logging + +help_text = """Test framework for bitcoin utils. + +Runs automatically during `make check`. + +Can also be run manually.""" + +if __name__ == '__main__': + sys.path.append(os.path.dirname(os.path.abspath(__file__))) + import buildenv + import bctest + + parser = argparse.ArgumentParser(description=help_text) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + verbose = args.verbose + + if verbose: + level = logging.DEBUG + else: + level = logging.ERROR + formatter = '%(asctime)s - %(levelname)s - %(message)s' + # Add the format/level to the logger + logging.basicConfig(format=formatter, level=level) + + bctest.bctester(buildenv.SRCDIR + "/test/util/data", + "bitcoin-util-test.json", buildenv) diff --git a/test/util/buildenv.py.in b/test/util/buildenv.py.in new file mode 100644 --- /dev/null +++ b/test/util/buildenv.py.in @@ -0,0 +1,4 @@ +#!/usr/bin/env python +exeext="@EXEEXT@" +SRCDIR="@abs_top_srcdir@" +BUILDDIR="@abs_top_builddir@" diff --git a/src/test/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json rename from src/test/data/bitcoin-util-test.json rename to test/util/data/bitcoin-util-test.json diff --git a/src/test/data/blanktxv1.hex b/test/util/data/blanktxv1.hex rename from src/test/data/blanktxv1.hex rename to test/util/data/blanktxv1.hex diff --git a/src/test/data/blanktxv1.json b/test/util/data/blanktxv1.json rename from src/test/data/blanktxv1.json rename to test/util/data/blanktxv1.json diff --git a/src/test/data/blanktxv2.hex b/test/util/data/blanktxv2.hex rename from src/test/data/blanktxv2.hex rename to test/util/data/blanktxv2.hex diff --git a/src/test/data/blanktxv2.json b/test/util/data/blanktxv2.json rename from src/test/data/blanktxv2.json rename to test/util/data/blanktxv2.json diff --git a/src/test/data/tt-delin1-out.hex b/test/util/data/tt-delin1-out.hex rename from src/test/data/tt-delin1-out.hex rename to test/util/data/tt-delin1-out.hex diff --git a/src/test/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json rename from src/test/data/tt-delin1-out.json rename to test/util/data/tt-delin1-out.json diff --git a/src/test/data/tt-delout1-out.hex b/test/util/data/tt-delout1-out.hex rename from src/test/data/tt-delout1-out.hex rename to test/util/data/tt-delout1-out.hex diff --git a/src/test/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json rename from src/test/data/tt-delout1-out.json rename to test/util/data/tt-delout1-out.json diff --git a/src/test/data/tt-locktime317000-out.hex b/test/util/data/tt-locktime317000-out.hex rename from src/test/data/tt-locktime317000-out.hex rename to test/util/data/tt-locktime317000-out.hex diff --git a/src/test/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json rename from src/test/data/tt-locktime317000-out.json rename to test/util/data/tt-locktime317000-out.json diff --git a/src/test/data/tx394b54bb.hex b/test/util/data/tx394b54bb.hex rename from src/test/data/tx394b54bb.hex rename to test/util/data/tx394b54bb.hex diff --git a/src/test/data/txcreate1.hex b/test/util/data/txcreate1.hex rename from src/test/data/txcreate1.hex rename to test/util/data/txcreate1.hex diff --git a/src/test/data/txcreate1.json b/test/util/data/txcreate1.json rename from src/test/data/txcreate1.json rename to test/util/data/txcreate1.json diff --git a/src/test/data/txcreate2.hex b/test/util/data/txcreate2.hex rename from src/test/data/txcreate2.hex rename to test/util/data/txcreate2.hex diff --git a/src/test/data/txcreate2.json b/test/util/data/txcreate2.json rename from src/test/data/txcreate2.json rename to test/util/data/txcreate2.json diff --git a/src/test/data/txcreatedata1.hex b/test/util/data/txcreatedata1.hex rename from src/test/data/txcreatedata1.hex rename to test/util/data/txcreatedata1.hex diff --git a/src/test/data/txcreatedata1.json b/test/util/data/txcreatedata1.json rename from src/test/data/txcreatedata1.json rename to test/util/data/txcreatedata1.json diff --git a/src/test/data/txcreatedata2.hex b/test/util/data/txcreatedata2.hex rename from src/test/data/txcreatedata2.hex rename to test/util/data/txcreatedata2.hex diff --git a/src/test/data/txcreatedata2.json b/test/util/data/txcreatedata2.json rename from src/test/data/txcreatedata2.json rename to test/util/data/txcreatedata2.json diff --git a/src/test/data/txcreatedata_seq0.hex b/test/util/data/txcreatedata_seq0.hex rename from src/test/data/txcreatedata_seq0.hex rename to test/util/data/txcreatedata_seq0.hex diff --git a/src/test/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json rename from src/test/data/txcreatedata_seq0.json rename to test/util/data/txcreatedata_seq0.json diff --git a/src/test/data/txcreatedata_seq1.hex b/test/util/data/txcreatedata_seq1.hex rename from src/test/data/txcreatedata_seq1.hex rename to test/util/data/txcreatedata_seq1.hex diff --git a/src/test/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json rename from src/test/data/txcreatedata_seq1.json rename to test/util/data/txcreatedata_seq1.json diff --git a/src/test/data/txcreatemultisig1.hex b/test/util/data/txcreatemultisig1.hex rename from src/test/data/txcreatemultisig1.hex rename to test/util/data/txcreatemultisig1.hex diff --git a/src/test/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json rename from src/test/data/txcreatemultisig1.json rename to test/util/data/txcreatemultisig1.json diff --git a/src/test/data/txcreatemultisig2.hex b/test/util/data/txcreatemultisig2.hex rename from src/test/data/txcreatemultisig2.hex rename to test/util/data/txcreatemultisig2.hex diff --git a/src/test/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json rename from src/test/data/txcreatemultisig2.json rename to test/util/data/txcreatemultisig2.json diff --git a/src/test/data/txcreateoutpubkey1.hex b/test/util/data/txcreateoutpubkey1.hex rename from src/test/data/txcreateoutpubkey1.hex rename to test/util/data/txcreateoutpubkey1.hex diff --git a/src/test/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json rename from src/test/data/txcreateoutpubkey1.json rename to test/util/data/txcreateoutpubkey1.json diff --git a/src/test/data/txcreatescript1.hex b/test/util/data/txcreatescript1.hex rename from src/test/data/txcreatescript1.hex rename to test/util/data/txcreatescript1.hex diff --git a/src/test/data/txcreatescript1.json b/test/util/data/txcreatescript1.json rename from src/test/data/txcreatescript1.json rename to test/util/data/txcreatescript1.json diff --git a/src/test/data/txcreatescript2.hex b/test/util/data/txcreatescript2.hex rename from src/test/data/txcreatescript2.hex rename to test/util/data/txcreatescript2.hex diff --git a/src/test/data/txcreatescript2.json b/test/util/data/txcreatescript2.json rename from src/test/data/txcreatescript2.json rename to test/util/data/txcreatescript2.json diff --git a/src/test/data/txcreatesignv1.hex b/test/util/data/txcreatesignv1.hex rename from src/test/data/txcreatesignv1.hex rename to test/util/data/txcreatesignv1.hex diff --git a/src/test/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json rename from src/test/data/txcreatesignv1.json rename to test/util/data/txcreatesignv1.json diff --git a/src/test/data/txcreatesignv2.hex b/test/util/data/txcreatesignv2.hex rename from src/test/data/txcreatesignv2.hex rename to test/util/data/txcreatesignv2.hex