Directory for functional tests (#6)

* Directory for functional tests

* workflow template

* Updating disk_size variable to match default value in EXECUTOR_PLUGIN_DEFAULTS

* Adding functional tests

* Uncommenting test step for checks

* Updating test step

* Updating test step

* Delete workflow_ec2_test.py

* Moving unit tests into different dir

* Update tests.yml

* Fixes

* Adding more tests

* Fixes

* Fixes

* Checking aws credentials

* Adding terraform setup

* Update tests.yml

* Update tests.yml

* Fixes

* Update tests.yml

* Update tests.yml

* Update tests.yml

* Update variables.tf

* Update variables.tf

* Commenting run step

* Update README.md

* Update CHANGELOG.md

* Updated CHANGELOG.md

* Removed pytest invocations

* Added var for shared_credentials

* Update variables.tf

* Redacted unit tests dir and terraform action

* Redacting `setup_teardown_test`

* Removing terraform requirement

* Added back pytest to tests workflow

* added temp identity_test for tests to pass

Co-authored-by: Alejandro Esquivel <ae@alejandro.ltd>

.github/workflows/tests.yml

@@ -93,9 +93,9 @@ jobs:
         run: |
           VERSION="$(cat ./VERSION)"
           pip install ./dist/covalent-ec2-plugin-${VERSION}.tar.gz
-
-#      - name: Run tests
-#        run: PYTHONPATH=$PWD/tests pytest -vv tests/
+          
+      - name: Run tests
+        run: PYTHONPATH=$PWD/tests pytest -vv tests/ --ignore=./tests/functional_tests
 
       - name: Get latest release
         uses: actions-ecosystem/action-get-latest-tag@v1

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Added
+
+- Added unit and functional tests
+
 ### Updated
 
 - Updated terraform provisioner script to write conda location to `/etc/environment`

README.md

@@ -2,7 +2,7 @@
 
 <div align="center">
 
-<img src="https://raw.githubusercontent.com/AgnostiqHQ/covalent-ec2-plugin/main/assets/ec2_readme_banner.jpg" width=150%>
+<img src="https://github.com/AgnostiqHQ/covalent-ec2-plugin/blob/main/assets/ec2_readme_banner.jpg" width=150%>
 
 </div>
 

infra/variables.tf

@@ -74,7 +74,7 @@ variable "instance_type" {
 }
 
 variable "disk_size" {
-  default     = 32
+  default     = 8
   description = "Server disk size"
 }
 

tests/create_executor.py

@@ -0,0 +1,35 @@
+# # Copyright 2021 Agnostiq Inc.
+# #
+# # This file is part of Covalent.
+# # 
+# # Licensed under the GNU Affero General Public License 3.0 (the "License"). 
+# # A copy of the License may be obtained with this software package or at
+# #
+# #      https://www.gnu.org/licenses/agpl-3.0.en.html
+# #
+# # Use of this file is prohibited except in compliance with the License. Any 
+# # modifications or derivative works of this file must retain this copyright 
+# # notice, and modified files must contain a notice indicating that they have 
+# # been altered from the originals.
+# #
+# # Covalent is distributed in the hope that it will be useful, but WITHOUT
+# # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# # FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+# #
+# # Relief from the License may be granted by purchasing a commercial license.
+
+# # Ignore results folders
+
+import os
+import covalent as ct
+
+executor = ct.executor.EC2Executor(
+    username=os.getenv("SSH_EXECUTOR_USERNAME", "ubuntu"),
+    profile=os.getenv("AWS_PROFILE", "default"),
+    credentials_file=os.getenv("AWS_SHARED_CREDENTIALS_FILE"),
+    key_name=os.getenv("KEY_PAIR_NAME"),
+    ssh_key_file=os.getenv("SSH_EXECUTOR_SSH_KEY_FILE"),
+    vpc="",
+    subnet="",
+    cache_dir="/tmp"
+)

tests/functional_tests/basic_workflow_test.py

@@ -0,0 +1,26 @@
+import sys
+import covalent as ct
+from tests.create_executor import executor as ec2_exec
+
+
+@ct.electron(executor=ec2_exec)
+def join_words(a, b):
+    return "-".join([a, b])
+
+
+@ct.electron(executor=ec2_exec)
+def excitement(a):
+    return f"{a}!"
+
+
+@ct.lattice
+def simple_workflow(a, b):
+    phrase = join_words(a, b)
+    return excitement(phrase)
+
+
+dispatch_id = ct.dispatch(simple_workflow)("Hello", "Covalent")
+status = str(ct.get_result(dispatch_id=dispatch_id, wait=True).status)
+
+if status == str(ct.status.FAILED):
+    sys.exit(1)

tests/functional_tests/svm_workflow_test.py

@@ -0,0 +1,65 @@
+import sys
+import covalent as ct
+from tests.create_executor import executor as ec2_exec
+from numpy.random import permutation
+from sklearn import svm, datasets
+
+sklearn_install = "/home/ubuntu/miniconda3/envs/covalent-dev/bin/python -m pip install sklearn"
+numpy_install = "/home/ubuntu/miniconda3/envs/covalent-dev/bin/python -m pip install numpy==1.22.4"
+
+@ct.electron(
+    executor=ec2_exec, 
+    deps_bash=ct.DepsBash(commands=[f"{sklearn_install}"])
+    )
+def load_data():
+    iris = datasets.load_iris()
+    perm = permutation(iris.target.size)
+    iris.data = iris.data[perm]
+    iris.target = iris.target[perm]
+    return iris.data, iris.target
+
+@ct.electron(
+    executor=ec2_exec, 
+    deps_bash=ct.DepsBash(commands=[f"{sklearn_install}"])
+    )
+def train_svm(data, C, gamma):
+    X, y = data
+    clf = svm.SVC(C=C, gamma=gamma)
+    clf.fit(X[90:], y[90:])
+    return clf
+
+@ct.electron(
+    executor=ec2_exec,
+    deps_bash=ct.DepsBash(commands=[f"{numpy_install}", f"{sklearn_install}"])
+    )
+def score_svm(data, clf):
+    X_test, y_test = data
+    return clf.score(
+    	X_test[:90],
+	y_test[:90]
+    )
+
+@ct.lattice
+def run_experiment(C=1.0, gamma=0.7):
+    data = load_data()
+    clf = train_svm(
+    	data=data,
+	C=C,
+	gamma=gamma
+    )
+    score = score_svm(
+    	data=data,
+	clf=clf
+    )
+    return score
+
+dispatchable_func = ct.dispatch(run_experiment)
+
+dispatch_id = dispatchable_func(
+    	C=1.0,
+    	gamma=0.7
+    )
+status = str(ct.get_result(dispatch_id=dispatch_id, wait=True).status)
+
+if status == str(ct.status.FAILED):
+    sys.exit(1)

tests/identity_test.py

@@ -0,0 +1,6 @@
+
+import pytest
+
+# TODO: remove once there are unit tests in repo
+def test_identiy():
+    assert 1 == 1