diff --git a/.travis.yml b/.travis.yml
index bb8b278..159d1ca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,11 @@
 sudo: false
 language: python
-
+cache: pip
+python:
+ - 2.7
+ - 3.4
+ - 3.5
+ - 3.6
 matrix:
   include:
     - os: linux
@@ -10,6 +15,17 @@ matrix:
     - os: osx
       language: generic
       python: 2.7
+jobs:
+  include:
+    - stage: release-test
+      python: 2.7
+      script: RELEASE_SKIP=head PYVER=2.7 ./release-test.sh
+    - python: 3.4
+      script: RELEASE_SKIP=head PYVER=3 ./release-test.sh
+    - python: 3.5
+      script: RELEASE_SKIP=head PYVER=3 ./release-test.sh
+    - python: 3.6
+      script: RELEASE_SKIP=head PYVER=3 ./release-test.sh
 
 install:
   - pip2 install tox-travis || pip3 install tox-travis
diff --git a/MANIFEST.in b/MANIFEST.in
index 6c6bdd6..e951ca2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,6 @@
 include Makefile gittaggers.py
-include tests/*
-include tests/test-data/*
+include cwltest/tests/*
+include cwltest/tests/test-data/*
 global-exclude *~
 global-exclude *.pyc
 
diff --git a/Makefile b/Makefile
index 8c566eb..42a11f1 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ VERSION=1.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 ## all         : default task
-all:
+all: FORCE
 	pip install -e .
 
 ## help        : print this help message and exit
@@ -131,7 +131,7 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
 		--html-report diff-cover.html
 
 ## test        : run the ${MODULE} test suite
-test: FORCE
+test: all
 	./setup.py test
 
 sloccount.sc: ${PYSOURCES} Makefile
diff --git a/tests/__init__.py b/cwltest/tests/__init__.py
similarity index 100%
rename from tests/__init__.py
rename to cwltest/tests/__init__.py
diff --git a/tests/mock_cwl_runner.py b/cwltest/tests/mock_cwl_runner.py
similarity index 94%
rename from tests/mock_cwl_runner.py
rename to cwltest/tests/mock_cwl_runner.py
index 1674f22..510864b 100644
--- a/tests/mock_cwl_runner.py
+++ b/cwltest/tests/mock_cwl_runner.py
@@ -5,7 +5,7 @@
 ERROR_TOOL = "return-1.cwl"
 
 
-def main():
+def main():  # type: ()->int
     parser = argparse.ArgumentParser()
     parser.add_argument("processfile")
     parser.add_argument("jobfile")
@@ -24,4 +24,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()
diff --git a/tests/test-data/cores.txt b/cwltest/tests/test-data/cores.txt
similarity index 100%
rename from tests/test-data/cores.txt
rename to cwltest/tests/test-data/cores.txt
diff --git a/tests/test-data/optional-error.yml b/cwltest/tests/test-data/optional-error.yml
similarity index 100%
rename from tests/test-data/optional-error.yml
rename to cwltest/tests/test-data/optional-error.yml
diff --git a/tests/test-data/optional-unsupported.yml b/cwltest/tests/test-data/optional-unsupported.yml
similarity index 100%
rename from tests/test-data/optional-unsupported.yml
rename to cwltest/tests/test-data/optional-unsupported.yml
diff --git a/tests/test-data/required-unsupported.yml b/cwltest/tests/test-data/required-unsupported.yml
similarity index 100%
rename from tests/test-data/required-unsupported.yml
rename to cwltest/tests/test-data/required-unsupported.yml
diff --git a/tests/test-data/short-names.yml b/cwltest/tests/test-data/short-names.yml
similarity index 100%
rename from tests/test-data/short-names.yml
rename to cwltest/tests/test-data/short-names.yml
diff --git a/tests/test-data/with-and-without-short-names.yml b/cwltest/tests/test-data/with-and-without-short-names.yml
similarity index 100%
rename from tests/test-data/with-and-without-short-names.yml
rename to cwltest/tests/test-data/with-and-without-short-names.yml
diff --git a/tests/test_argparse.py b/cwltest/tests/test_argparse.py
similarity index 99%
rename from tests/test_argparse.py
rename to cwltest/tests/test_argparse.py
index 44c3c88..f0ef666 100644
--- a/tests/test_argparse.py
+++ b/cwltest/tests/test_argparse.py
@@ -15,4 +15,3 @@ def test_arg(self):
 
 if __name__ == '__main__':
     unittest.main()
-
diff --git a/tests/test_categories.py b/cwltest/tests/test_categories.py
similarity index 100%
rename from tests/test_categories.py
rename to cwltest/tests/test_categories.py
diff --git a/tests/test_compare.py b/cwltest/tests/test_compare.py
similarity index 100%
rename from tests/test_compare.py
rename to cwltest/tests/test_compare.py
diff --git a/tests/test_short_names.py b/cwltest/tests/test_short_names.py
similarity index 100%
rename from tests/test_short_names.py
rename to cwltest/tests/test_short_names.py
diff --git a/tests/util.py b/cwltest/tests/util.py
similarity index 100%
rename from tests/util.py
rename to cwltest/tests/util.py
diff --git a/release-test.sh b/release-test.sh
index f295ab9..e4a77ec 100755
--- a/release-test.sh
+++ b/release-test.sh
@@ -13,26 +13,29 @@ PYVER=${PYVER:=2.7}
 
 rm -Rf testenv${PYVER}_? || /bin/true
 
-export HEAD=`git rev-parse HEAD`
-virtualenv testenv${PYVER}_1 -p python${PYVER}
+export HEAD=${TRAVIS_PULL_REQUEST_SHA:-$(git rev-parse HEAD)}
+if [ "${RELEASE_SKIP}" != "head" ]
+then
+	virtualenv testenv${PYVER}_1 -p python${PYVER}
+	# First we test the head
+	source testenv${PYVER}_1/bin/activate
+	rm testenv${PYVER}_1/lib/python-wheels/setuptools* \
+		&& pip install --force-reinstall -U pip==${pipver} \
+	        && pip install setuptools==${setupver} wheel
+	make install-dependencies
+	make test
+	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
+	mkdir testenv${PYVER}_1/not-${module}
+	# if there is a subdir named '${module}' py.test will execute tests
+	# there instead of the installed module's tests
+	pushd testenv${PYVER}_1/not-${module}; ../bin/${run_tests}; popd
+fi
+
 virtualenv testenv${PYVER}_2 -p python${PYVER}
 virtualenv testenv${PYVER}_3 -p python${PYVER}
 virtualenv testenv${PYVER}_4 -p python${PYVER}
 virtualenv testenv${PYVER}_5 -p python${PYVER}
 
-# First we test the head
-source testenv${PYVER}_1/bin/activate
-rm testenv${PYVER}_1/lib/python-wheels/setuptools* \
-	&& pip install --force-reinstall -U pip==${pipver} \
-        && pip install setuptools==${setupver} wheel
-make install-dependencies
-make test
-pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
-mkdir testenv${PYVER}_1/not-${module}
-# if there is a subdir named '${module}' py.test will execute tests
-# there instead of the installed module's tests
-pushd testenv${PYVER}_1/not-${module}; ../bin/${run_tests}; popd
-
 
 # Secondly we test via pip
 
diff --git a/setup.cfg b/setup.cfg
index 4d2c12b..ee13171 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -7,7 +7,7 @@ ignore = E124,E128,E129,E201,E202,E225,E226,E231,E265,E271,E302,E303,F401,E402,E
 universal = 0
 
 [tool:pytest]
-testpaths = tests
+testpaths = cwltest/tests
 
 [aliases]
 test=pytest
diff --git a/setup.py b/setup.py
index 7c681a5..2c67360 100755
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,8 @@
       url="https://github.com/common-workflow-language/cwltest",
       download_url="https://github.com/common-workflow-language/cwltest",
       license='Apache 2.0',
-      packages=["cwltest", "tests"],
+      packages=["cwltest", "cwltest.tests"],
+      package_data={'cwltest.tests': 'test-data/*'},
       include_package_data=True,
       install_requires=install_requires,
       test_suite='tests',
@@ -42,7 +43,7 @@
       tests_require=['pytest'],
       entry_points={
           'console_scripts': ["cwltest=cwltest:main",
-                              "mock-cwl-runner=tests.mock_cwl_runner:main"]
+                              "mock-cwl-runner=cwltest.tests.mock_cwl_runner:main"]
       },
       zip_safe=True,
       cmdclass={'egg_info': tagger},