mesh-1684: align with spine-core-aws-common and bring in ruff

.github/workflows/pull-request.yml

@@ -66,9 +66,8 @@ jobs:
       - name: install dependencies
         run: make install-ci
 
-# todo: disabled till diffs can be evaluated
-#      - name: black
-#        run: make black-check
+      - name: black
+        run: make black-check
 
       - name: start localstack
         run: make up
@@ -188,15 +187,11 @@ jobs:
       - name: install dependencies
         run: make install-ci
 
-# todo: not yet being used
-#      - name: black
-#        run: make black-check
-#
-#      - name: isort
-#        run: make isort-check
-#
-#      - name: flake8
-#        run: make flake8
+      - name: black
+        run: make black-check
+
+      - name: ruff
+        run: make ruff-ci
 
 # todo: typing needs attention
 #      - name: mypy

.tool-versions

@@ -1,5 +1,5 @@
 docker-compose-v1 2.17.3
-python 3.9.16
+python 3.9.12
 poetry 1.5.1
 terraform 1.4.5
 tfsec 1.28.1

Makefile

@@ -76,20 +76,21 @@ shellcheck:
 	@# Only swallow checking errors (rc=1), not fatal problems (rc=2)
 	docker run --rm -i -v ${PWD}:/mnt:ro koalaman/shellcheck -f gcc -e SC1090,SC1091 `find . \( -path "*/.venv/*" -prune -o -path "*/build/*" -prune -o -path "*/.tox/*" -prune -o -path "*/java_client/*" -prune  \) -o -type f -name '*.sh' -print` || test $$? -eq 1
 
+ruff: black
+	poetry run ruff --fix --show-fixes .
 
-flake8:
-	poetry run flake8
+ruff-check:
+	poetry run ruff .
 
-lint: flake8 mypy shellcheck
+ruff-ci:
+	poetry run ruff --format=github .
+
+lint: ruff mypy shellcheck
 
 black-check:
 	poetry run black . --check
 
-isort-check:
-	poetry run isort . -c
-
 black:
-	poetry run isort .
 	poetry run black .
 
 coverage-cleanup:
@@ -139,14 +140,14 @@ check-secrets:
 check-secrets-all:
 	./nhsd-git-secrets/pre-commit.sh
 
-#delete-hooks:
-#	rm .git/hooks/pre-commit 2>/dev/null || true
-#	rm .git/hooks/commit-msg 2>/dev/null || true
-#
-#.git/hooks/pre-commit:
-#	cp scripts/hooks/pre-commit.sh .git/hooks/pre-commit
-#
-#.git/hooks/commit-msg:
-#	cp scripts/hooks/commit-msg.sh .git/hooks/commit-msg
-#
-#refresh-hooks: delete-hooks .git/hooks/pre-commit .git/hooks/commit-msg
+delete-hooks:
+	rm .git/hooks/pre-commit 2>/dev/null || true
+	rm .git/hooks/commit-msg 2>/dev/null || true
+
+.git/hooks/pre-commit:
+	cp scripts/hooks/pre-commit.sh .git/hooks/pre-commit
+
+.git/hooks/commit-msg:
+	cp scripts/hooks/commit-msg.sh .git/hooks/commit-msg
+
+refresh-hooks: delete-hooks .git/hooks/pre-commit .git/hooks/commit-msg

mesh_client_aws_serverless/__init__.py

@@ -1,7 +1,6 @@
 """
 Module for common application functionality for MESH functions
 """
-from mesh_client_aws_serverless.mesh_mailbox import MeshMailbox, MeshMessage
-
+from mesh_aws_client.mesh_mailbox import MeshMailbox, MeshMessage
 
 __all__ = ["MeshMailbox", "MeshMessage"]

mesh_client_aws_serverless/cloudlogbase.cfg

@@ -1,3 +1,55 @@
+[LAMBDAINIT001]
+Log Level = CRITICAL
+Log Text = Error during initialisation with message={message}
+
+[LAMBDA0001]
+Log Level = INFO
+Log Text = Lambda cold start invoked for lambdaFunction="{function_name}" lambdaFunctionVersion="{function_version}" lambdaExecutionEnv="{aws_execution_env}" lambdaMemory="{function_memory_size}" in awsRegion="{aws_region}"
+
+[LAMBDA0002]
+Log Level = INFO
+Log Text = Lambda invoked aws_request_id={aws_request_id}
+
+[LAMBDA0003]
+Log Level = INFO
+Log Text = Lambda completed duration={duration} aws_request_id={aws_request_id}
+
+[LAMBDA9999]
+Log Level = ERROR
+Log Text = Unhandled exception caught with error="{error}"
+
+[UTI9992]
+Log Level = CRITICAL
+Log Text = Crash dump occurred. See spinevfmcrashdump index for details with originalLogReference={originalLogReference}
+
+[UTI9993]
+Log Level = TRACE
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9994]
+Log Level = DEBUG
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9995]
+Log Level = INFO
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9996]
+Log Level = WARN
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9997]
+Log Level = ERROR
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9998]
+Log Level = CRITICAL
+Log Text = Python logger '{logger}' reports {level} '{message}'
+
+[UTI9999]
+Log Level = INFO
+Log Text = No log information defined for logReference={logReference} so default text returned
+
 [MESH0001]
 Log Level = INFO
 Log Text = Loading settings for mailbox='{mailbox}' from parameter store
@@ -26,6 +78,10 @@ Log Text = List of message_count='{message_count}' in mailbox='{mailbox}' http_s
 Log Level = INFO
 Log Text = Acknowledging message_id='{message_id}' http_status='{http_status}'
 
+[MESHMBOX0007]
+Log Level = INFO
+Log Text = Cleanup of mesh mailbox='{mailbox}' has begun
+
 [MESHSEND0001]
 Log Level = INFO
 Log Text = Send file triggered for file='{file}' from bucket='{bucket}'
@@ -42,6 +98,10 @@ Log Text = Singleton check for mailbox='{mailbox}' failed with error='{error}'
 Log Level = INFO
 Log Text = Will send file='{file}' filesize="{file_size}" from bucket='{bucket}' to destMailbox='{dest_mailbox}' from srcMailbox='{src_mailbox}' with workflowId='{workflow_id}' in chunks='{chunks}' of chunk_size='{chunk_size}'
 
+[MESHSEND0004a]
+Log Level = INFO
+Log Text = For path='{path}' have the mapping destMailbox='{dest_mailbox}' from srcMailbox='{src_mailbox}' with workflowId='{workflow_id}'
+
 [MESHSEND0005]
 Log Level = INFO
 Log Text = Send chunk started for file='{file}' from bucket='{bucket}' chunk='{chunk_num}' of max_chunk='{max_chunk}'
@@ -52,9 +112,17 @@ Log Text = Received data from s3 file='{file}' from bucket='{bucket}' read bytes
 
 [MESHSEND0007]
 Log Level = INFO
-Log Text = Sent chunk of file='{file}' message_id='{message_id}' via MESH response_code='{http_status}'
+Log Text = Sent chunk='{chunk_num}' of max_chunk='{max_chunk}' for file='{file}' message_id='{message_id}' via MESH response_code='{http_status}'
+
+[MESHSEND0008]
+Log Level = INFO
+Log Text = chunk='{chunk_num}' is the final chunk out of max_chunk='{max_chunk}' for file='{file}' from bucket='{bucket}' and has been sent
 
-[MESHPOLL0003]
+[MESHPOLL0001]
+Log Level = INFOR
+Log Text = mailbox='{mailbox}' has polled message_count='{message_count}' many messages
+
+[MESHPOLL0002]
 Log Level = ERROR
 Log Text = msg='{error}' raised when polling mailbox='{mailbox}'
 
@@ -66,6 +134,14 @@ Log Text = Downloading messageId='{message_id}'
 Log Level = INFO
 Log Text = Downloaded length='{length}' bytes for messageId='{message_id}'
 
+[MESHFETCH0001b]
+Log Level = INFO
+Log Text = Getting chunk chunk_num='{chunk_num}' of max_chunk='{max_chunk}' for messageId='{message_id}'
+
+[MESHFETCH0001c]
+Log Level = INFO
+Log Text = Will store chunk_num='{chunk_num}' of messageId='{message_id}' in s3_folder='{s3_folder}' with s3_key='{s3_key}' on s3_bucket='{s3_bucket}'
+
 [MESHFETCH0002]
 Log Level = INFO
 Log Text = Persisted aws_part_id='{aws_part_id}' size='{aws_part_size}' for aws_upload_id='{aws_upload_id}' to S3 - Number of chunks='{number_of_chunks}'
@@ -80,7 +156,7 @@ Log Text = Found overflow file with size='{aws_part_size}' for aws_upload_id='{a
 
 [MESHFETCH0002c]
 Log Level = INFO
-Log Text = No overflow file found, passing ClientError for aws_upload_id='{aws_upload_id}' number_of_chunks='{number_of_chunks}'
+Log Text = No overflow file found, passing ClientError for aws_upload_id='{aws_upload_id}' number_of_chunks='{number_of_chunks}' client_error='{client_error}'
 
 [MESHFETCH0003]
 Log Level = INFO
@@ -94,7 +170,11 @@ Log Text = Preparing to upload part to S3 with size='{buffer_len}'
 Log Level = INFO
 Log Text = Last chunk downloaded, finished multipart upload and message acknowledged for message_id='{message_id}'
 
-[MESHFETCH0005]
+[MESHFETCH0005a]
+Log Level = INFO
+Log Text = Started multipart upload for key='{key}' to bucket='{bucket}'
+
+[MESHFETCH0005b]
 Log Level = CRITICAL
 Log Text = Failed to create multipart upload for key='{key}' to bucket='{bucket}' due to error='{error}'
 
@@ -104,7 +184,7 @@ Log Text = Failed to save file from MESH to S3 when upload part with content_len
 
 [MESHFETCH0007]
 Log Level = CRITICAL
-Log Text = Failed to save file from MESH id='{mesh_msg_id}' to S3 when completing the multipart upload for key='{key}' to bucket='{bucket}' with aws_upload_id='{aws_upload_id}' due to error='{error}' - NUMBER OF CHUNKS='{number_of_chunks}'
+Log Text = Failed to save file from MESH id='{mesh_msg_id}' to S3 when completing the multipart upload for key='{key}' to bucket='{bucket}' with aws_upload_id='{aws_upload_id}' due to error='{error}' max_chunks='{number_of_chunks}'
 
 [MESHFETCH0008]
 Log Level = INFO
@@ -116,7 +196,11 @@ Log Text = Attempting to create multipart upload for key='{key}' to bucket='{buc
 
 [MESHFETCH0010]
 Log Level = INFO
-Log Text = Report file received message_id='{message_id}'
+Log Text = Report file received message_id='{message_id}', writing HTTP Headers of Report to s3_bucket='{s3_bucket}' with s3_key='{s3_key}'
+
+[MESHFETCH0010a]
+Log Level = INFO
+Log Text = Writing HTTP Headers of Report to s3_bucket='{s3_bucket}' with s3_key='{s3_key}'
 
 [MESHFETCH0011]
 Log Level = INFO
@@ -128,4 +212,4 @@ Log Text = Message with message_id='{message_id}' acknowledged and successfully
 
 [MESHFETCH0013]
 Log Level = INFO
-Log Text = File with multiple chunks received message_id='{message_id}'
+Log Text = File with multiple chunks received message_id='{message_id}'

mesh_client_aws_serverless/mesh_check_send_parameters_application.py

@@ -11,10 +11,7 @@
 from spine_aws_common import LambdaApplication
 from spine_aws_common.utilities import human_readable_bytes
 
-from mesh_client_aws_serverless.mesh_common import (
-    MeshCommon,
-    SingletonCheckFailure,
-)
+from .mesh_common import MeshCommon, SingletonCheckFailure
 
 
 def calculate_chunks(file_size, chunk_size):
@@ -33,8 +30,7 @@ def __init__(self, additional_log_config=None, load_ssm_params=False):
         self.environment = self.system_config.get("Environment", "default")
         self.chunk_size = None
         self.send_message_step_function_name = self.system_config.get(
-            "SEND_MESSAGE_STEP_FUNCTION_NAME",
-            f"{self.environment}-send-message",
+            "SEND_MESSAGE_STEP_FUNCTION_NAME", f"{self.environment}-send-message"
         )
 
     def _get_internal_id(self):
@@ -56,30 +52,23 @@ def start(self):
         bucket = self.event.detail["requestParameters"]["bucketName"]
         key = self.event.detail["requestParameters"]["key"]
 
-        self.log_object.write_log(
-            "MESHSEND0001", None, {"bucket": bucket, "file": key}
-        )
+        self.log_object.write_log("MESHSEND0001", None, {"bucket": bucket, "file": key})
 
-        # TODO nicer failure, log error and parameters missing in
-        # SSM Parameter Store
-        (src_mailbox, dest_mailbox, workflow_id) = self._get_mapping(
-            bucket, key
-        )
+        # TODO nicer failure, log error and parameters missing in SSM Parameter Store
+        (src_mailbox, dest_mailbox, workflow_id) = self._get_mapping(bucket, key)
 
-        self.log_object.write_log(
-            "MESHSEND0002", None, {"mailbox": src_mailbox}
-        )
+        self.log_object.write_log("MESHSEND0002", None, {"mailbox": src_mailbox})
         try:
             MeshCommon.singleton_check(
                 src_mailbox, self.send_message_step_function_name
             )
-        except SingletonCheckFailure as exception:
+        except SingletonCheckFailure as e:
             self.response = MeshCommon.return_failure(
                 self.log_object,
                 HTTPStatus.TOO_MANY_REQUESTS.value,
                 "MESHSEND0003",
                 src_mailbox,
-                message=exception.msg,
+                message=e.msg,
             )
             return
 
@@ -145,6 +134,16 @@ def _get_mapping(self, bucket, key):
         src_mailbox = mailbox_mapping["src_mailbox"]
         dest_mailbox = mailbox_mapping["dest_mailbox"]
         workflow_id = mailbox_mapping["workflow_id"]
+        self.log_object.write_log(
+            "MESHSEND0004a",
+            None,
+            {
+                "src_mailbox": src_mailbox,
+                "dest_mailbox": dest_mailbox,
+                "workflow_id": workflow_id,
+                "path": path,
+            },
+        )
         return (src_mailbox, dest_mailbox, workflow_id)
 
     @staticmethod