Refactor & improve readability (#127)

-Extract logger into its own module
- Refactor main startup

ado_express/packages/__init__.py

@@ -1,3 +1,4 @@
-from .utils import *
-from .common import *
-from .authentication import *
+from .ado_express import *
+from .authentication import *
+from .shared import *
+from .toolbox import *

ado_express/packages/ado_express/__init__.py

@@ -0,0 +1 @@
+from .ado_express import *

ado_express/packages/ado_express/ado_express.py

@@ -0,0 +1,212 @@
+import concurrent.futures
+import logging
+from itertools import repeat
+
+from ado_express.packages.authentication import MSAuthentication
+from ado_express.packages.shared import Constants, EnvironmentVariables
+from ado_express.packages.shared.enums import DeploymentStatusLabel, ExplicitReleaseTypes
+from ado_express.packages.shared.models import DeploymentDetails, DeploymentStatus
+from ado_express.packages.toolbox import (ExcelManager, Logger,
+                                          ReleaseEnvironmentFinder,
+                                          ReleaseFinder,
+                                          UpdateProgressRetriever,
+                                          UpdateRelease, WorkItemManager,
+                                          run_helpers)
+
+
+class ADOExpress:
+
+    def __init__(self, environment_variables: EnvironmentVariables):
+        self.constants = Constants()
+        self.excel_manager = ExcelManager()
+
+        self.environment_variables = environment_variables
+        self.load_dependencies()
+
+        Logger(run_helpers.is_running_as_executable()).log_the_start_of_application(self.search_only)
+
+    def load_dependencies(self):
+        self.ms_authentication = MSAuthentication(self.environment_variables)
+        self.release_finder = ReleaseFinder(self.ms_authentication, self.environment_variables)
+        self.search_only = self.environment_variables.SEARCH_ONLY
+        self.via_env = self.environment_variables.VIA_ENV
+        self.via_latest = self.environment_variables.VIA_ENV_LATEST_RELEASE
+        self.queries = self.environment_variables.QUERIES
+
+    def prepare_result_excel_file(self):
+        new_df = self.excel_manager.create_dataframe(self.constants.DEPLOYMENT_PLAN_HEADERS)
+        self.excel_manager.save_or_concat_file(new_df, self.constants.SEARCH_RESULTS_DEPLOYMENT_PLAN_FILE_PATH, True) 
+
+    def updated_deployment_details_based_on_explicit_inclusion_and_exclusion(self, deployment_details):
+        new_deployment_details = []
+        explicit_deployment_values = self.environment_variables.EXPLICIT_RELEASE_VALUES
+
+        if explicit_deployment_values is None: return deployment_details
+
+        releases_to_deploy = explicit_deployment_values.get(ExplicitReleaseTypes.INCLUDE)
+        releases_not_to_deploy = explicit_deployment_values.get(ExplicitReleaseTypes.EXCLUDE)
+
+        if releases_to_deploy: [new_deployment_details.append(deployment_detail) if deployment_detail.release_name in releases_to_deploy else 99999 for deployment_detail in deployment_details]
+        elif releases_not_to_deploy: [new_deployment_details.append(deployment_detail) if deployment_detail.release_name not in releases_not_to_deploy else 99999 for deployment_detail in deployment_details]
+
+        if new_deployment_details == []: logging.error('Found no releases based on the explicit release values provided.')
+
+        return new_deployment_details
+
+    def get_crucial_release_definitions(self, deployment_details):
+        crucial_release_definitions = []
+        # First checks command line args, if not found, then checks the deployment plan file
+        if self.environment_variables.CRUCIAL_RELEASE_DEFINITIONS is not None and self.environment_variables.CRUCIAL_RELEASE_DEFINITIONS != []:
+            crucial_release_definitions = self.environment_variables.CRUCIAL_RELEASE_DEFINITIONS
+        else:
+            for deployment_detail in deployment_details:
+                if deployment_detail.is_crucial:
+                    crucial_release_definitions.append(deployment_detail.release_name)
+
+        return crucial_release_definitions
+
+    def get_deployment_details_from_query(self):
+        work_item_manager = WorkItemManager(self.ms_authentication)
+        found_releases = dict()
+
+        for query in self.queries:
+            build_ids = work_item_manager.get_query_build_ids(query)
+            search_result_releases = self.release_finder.get_releases_via_builds(build_ids)
+
+            for release_definition in search_result_releases:
+                if release_definition not in found_releases: found_releases[release_definition] = search_result_releases[release_definition]
+                elif found_releases[release_definition] < search_result_releases[release_definition]: found_releases[release_definition] = search_result_releases[release_definition]
+
+        rollback_dict = dict()
+        deployment_details = []
+
+        if not found_releases: return deployment_details
+
+        # Get rollback
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            rollbacks = executor.map(self.release_finder.get_release, {k for k, v in found_releases.items()}, repeat(self.via_env), repeat(True), repeat(self.via_latest))
+
+            for rollback in rollbacks:
+                if all(rollback.values()): rollback_dict |= rollback # If found rollback, add it to rollback_dict
+                else: found_releases.pop(next(iter(rollback))) # Else remove release key & value from found_releases
+
+        for release_location, target_release in found_releases.items():
+            project = release_location.split('/')[0] 
+            release_name = release_location.split('/')[1]
+            rollback_release = rollback_dict[release_location]
+            target_release_number = target_release.split('-')[1]
+            rollback_release_number = rollback_release.split('-')[1]
+
+            if run_helpers.needs_deployment(target_release_number, rollback_release_number):
+                deployment_detail = DeploymentDetails(project, release_name, target_release_number, rollback_release_number)
+                deployment_details.append(deployment_detail)
+
+                logging.info(f'Release found from query: Project:{project}, Release Definition:{release_name}, Target:{target_release_number}, Rollback:{rollback_release_number}')
+
+        return deployment_details
+
+    def get_deployment_detail_from_latest_release(self, deployment_detail: DeploymentDetails):
+        try:
+            target_release = self.release_finder.get_release(deployment_detail, find_via_env=self.via_env, rollback=False, via_latest=self.via_latest)
+            rollback_release = self.release_finder.get_release(deployment_detail, find_via_env=self.via_env, rollback=True, via_latest=self.via_latest)
+            target_release_number = target_release.name.split('-')[1]
+            rollback_release_number = rollback_release.name.split('-')[1]
+
+            if run_helpers.needs_deployment(target_release_number, rollback_release_number):
+                deployment_detail = DeploymentDetails(deployment_detail.release_project_name, deployment_detail.release_name, target_release_number, rollback_release_number, deployment_detail.is_crucial)
+
+                logging.info(f'Latest release found: Project:{deployment_detail.release_project_name}, Release Definition:{deployment_detail.release_name}, Target:{target_release_number}, Rollback:{rollback_release_number}')
+                return deployment_detail
+            else: return logging.info(f'No Deployable releases found: Project:{deployment_detail.release_project_name}, Release Definition:{deployment_detail.release_name}, Latest release found: Target:{target_release_number}, Rollback:{rollback_release_number}')
+
+        except:
+            logging.error(f'Latest release not found: Project:{deployment_detail.release_project_name}, Release Definition:{deployment_detail.release_name}\n - Possible cause: The release does not have either the source or target stage you are looking for')
+            return None
+
+    def search_and_log_details_only(self, deployment_detail: DeploymentDetails):
+        return self.release_finder.get_releases(deployment_detail, find_via_env=self.via_env)
+
+    def deploy_to_target_or_rollback(self, deployment_detail: DeploymentDetails, rollback: bool=False):
+        try:
+            if deployment_detail is not None: # The ThreadPoolExecutor may return None for some releases
+                update_manager = UpdateRelease(self.constants, self.ms_authentication, self.environment_variables, self.release_finder)
+
+                release_to_update = self.release_finder.get_release(deployment_detail, self.via_env, rollback, self.via_latest)
+
+                if rollback: 
+                    logging.info(f'Attempting to rollback: {deployment_detail.release_name}')
+                    update_manager.roll_back_release(deployment_detail, release_to_update)
+
+                    return True
+                else:
+                    update_manager = UpdateRelease(self.constants, self.ms_authentication, self.environment_variables, self.release_finder)
+                    attempt_was_successful, update_error = update_manager.update_release(deployment_detail, release_to_update)
+
+                    if not attempt_was_successful:
+                        logging.error(f'There was an error with deployment for: {deployment_detail.release_name}.\n:{update_error}')
+
+                return attempt_was_successful
+        except Exception as e:
+            logging.error(f'There was an error with deployment for: {deployment_detail.release_name}. Please check their status and continue manually.\nException:{e}')
+            return False
+
+    def get_deployment_status(self, deployment_detail: DeploymentDetails, rollback: bool=False):
+        try:
+            if deployment_detail is not None:
+                try:
+                    updating_release = self.release_finder.get_release(deployment_detail, self.via_env, rollback, self.via_latest)
+                except IndexError:
+                    errorMessage = f"Error: Cannot find the release for {deployment_detail.release_name}"
+                    logging.error(errorMessage)
+
+                    deployment_status = DeploymentStatus(errorMessage, 0, DeploymentStatusLabel.failed)
+                    return deployment_status
+
+                try:
+                    release_environment_finder = ReleaseEnvironmentFinder(self.ms_authentication, self.environment_variables)
+                    updating_release_environment = release_environment_finder.get_release_environment(deployment_detail, updating_release.id)
+                except IndexError:
+                    errorMessage = f"Error: Cannot find the release environment for {deployment_detail.release_name} and release ID {updating_release.id}"
+                    logging.error(errorMessage)
+
+                    deployment_status = DeploymentStatus(errorMessage, 0, DeploymentStatusLabel.failed)
+                    return deployment_status
+
+                release_progress = UpdateProgressRetriever(self.ms_authentication, self.environment_variables)
+                current_deployment_status = release_progress.monitor_release_progress(deployment_detail.release_project_name, updating_release, updating_release_environment.id)
+
+                return current_deployment_status
+
+        except Exception as e:
+            logging.error(f'There was an error with retrieving live deployment status of {updating_release.release_definition}.\nException:{e}')
+
+
+    def release_deployment_completed(self, deployment_detail, rollback=False): 
+        update_manager = UpdateRelease(self.constants, self.ms_authentication, self.environment_variables, self.release_finder)
+
+        release_to_update = self.release_finder.get_release(deployment_detail, self.via_env, rollback, self.via_latest)
+
+        deployment_is_complete, successfully_completed = update_manager.is_deployment_complete(deployment_detail, release_to_update)
+
+        return deployment_is_complete, successfully_completed
+
+    def run_release_deployments(self, deployment_details, is_deploying_crucial_releases, rollback=False, had_crucial_releases=False):
+        releases = []
+
+        if is_deploying_crucial_releases: logging.info('Deploying the crucial releases first')
+
+        with concurrent.futures.ThreadPoolExecutor() as executor: # Then, deploy the rest of the releases
+            if not is_deploying_crucial_releases and had_crucial_releases: 
+                logging.info('Deploying the rest of the releases')
+            elif not is_deploying_crucial_releases and had_crucial_releases:
+                logging.info('Deploying releases')
+
+            releases = executor.map(self.deploy_to_target_or_rollback, deployment_details, repeat(rollback))
+
+        return releases
+
+    def get_crucial_deployment_from_deployment_details(self, deployment_details, crucial_release_definitions):
+        return [x for x in deployment_details if x.release_name in crucial_release_definitions]
+
+    def remove_crucial_deployments_from_deployment_details(self, deployment_details, crucial_release_definitions):
+        return [x for x in deployment_details if x.release_name not in crucial_release_definitions]

ado_express/packages/authentication/ms_authentication/ms_authentication.py

@@ -1,7 +1,7 @@
 from azure.devops.connection import Connection
 from msrest.authentication import BasicAuthentication
 
-from ado_express.packages.common import EnvironmentVariables
+from ado_express.packages.shared import EnvironmentVariables
 
 class MSAuthentication:
 

ado_express/packages/shared/enums/__init__.py

@@ -0,0 +1,5 @@
+from .deployment_status_label import *
+from .environment_statuses import *
+from .explicit_release_types import *
+from .meta_enum import *
+from .relation_types import *

ado_express/packages/common/environment_variables.py → ado_express/packages/shared/environment_variables.py

@@ -2,10 +2,11 @@
 import os
 import re
 import sys
+
 import validators
 from dotenv import load_dotenv
 
-from ado_express.packages.common.enums.explicit_release_types import ExplicitReleaseTypes
+from ado_express.packages.shared.enums import ExplicitReleaseTypes
 
 none_types = ["none", "null", "nill", " ", ""]
 

ado_express/packages/toolbox/__init__.py

@@ -0,0 +1,5 @@
+from .asset_managers import *
+from .release_manager import *
+from .excel_manager import *
+from .logger import *
+from .run_helpers import *

ado_express/packages/toolbox/asset_managers/__init__.py

@@ -0,0 +1,4 @@
+from .deployment_plan import *
+from .release_environment_finder import *
+from .release_finder import *
+from .work_item_manager import *

ado_express/packages/utils/asset_retrievers/deployment_plan/deployment_plan.py → ado_express/packages/toolbox/asset_managers/deployment_plan/deployment_plan.py

@@ -1,9 +1,9 @@
 import numpy as np
 import pandas as pd
 
-from ado_express.packages.common.constants import Constants
-from ado_express.packages.common.environment_variables import EnvironmentVariables
-from ado_express.packages.common.models import DeploymentDetails
+from ado_express.packages.shared.constants import Constants
+from ado_express.packages.shared.environment_variables import EnvironmentVariables
+from ado_express.packages.shared.models import DeploymentDetails
 
 class DeploymentPlan():
 

ado_express/packages/toolbox/asset_managers/release_environment_finder/__init__.py

@@ -0,0 +1 @@
+from .release_environment_finder import *

ado_express/packages/utils/asset_retrievers/release_environment_finder/release_environment_finder.py → ado_express/packages/toolbox/asset_managers/release_environment_finder/release_environment_finder.py

@@ -1,7 +1,6 @@
 from ado_express.packages.authentication import MSAuthentication
-from ado_express.packages.common.environment_variables import \
-    EnvironmentVariables
-from ado_express.packages.common.models import ReleaseEnvironment
+from ado_express.packages.shared.environment_variables import EnvironmentVariables
+from ado_express.packages.shared.models import ReleaseEnvironment
 
 
 class ReleaseEnvironmentFinder:

ado_express/packages/utils/asset_retrievers/release_finder/release_finder.py → ado_express/packages/toolbox/asset_managers/release_finder/release_finder.py

@@ -3,12 +3,11 @@
 from itertools import repeat
 
 from ado_express.packages.authentication import MSAuthentication
-from ado_express.packages.common.constants import Constants
-from ado_express.packages.common.enums import ReleaseEnvironmentStatuses
-from ado_express.packages.common.environment_variables import \
+from ado_express.packages.shared.enums import ReleaseEnvironmentStatuses
+from ado_express.packages.shared.environment_variables import \
     EnvironmentVariables
-from ado_express.packages.common.models import DeploymentDetails
-from ado_express.packages.common.models.release_details import ReleaseDetails
+from ado_express.packages.shared.models import DeploymentDetails
+from ado_express.packages.shared.models.release_details import ReleaseDetails
 
 
 class ReleaseFinder:

ado_express/packages/utils/asset_retrievers/work_item_manager/work_item_manager.py → ado_express/packages/toolbox/asset_managers/work_item_manager/work_item_manager.py

@@ -2,7 +2,7 @@
 
 from ado_express.packages.authentication.ms_authentication.ms_authentication import \
     MSAuthentication
-from ado_express.packages.common.enums import RelationTypes
+from ado_express.packages.shared.enums import RelationTypes
 
 
 class WorkItemManager:

ado_express/packages/utils/excel_manager/excel_manager.py → ado_express/packages/toolbox/excel_manager/excel_manager.py

@@ -3,7 +3,7 @@
 
 from typing import List
 
-from ado_express.packages.common.models import DeploymentDetails
+from ado_express.packages.shared.models import DeploymentDetails
 
 class ExcelManager:
 

ado_express/packages/toolbox/logger/__init__.py

@@ -0,0 +1 @@
+from .logger import *

ado_express/packages/toolbox/logger/logger.py

@@ -0,0 +1,46 @@
+import logging
+import sys
+from datetime import datetime
+
+from pytz import timezone
+
+from ado_express.packages.shared.constants import Constants
+
+
+class Logger:
+
+    def __init__(self, is_running_as_executable):
+        if is_running_as_executable: self.log_to_console_only()
+        else: self.log_to_file_and_console()
+
+    def log_to_console_only():
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(levelname)s:%(asctime)s \t%(pathname)s:line:%(lineno)d \t%(message)s')
+
+    def log_to_file_and_console():
+        file_handler = logging.FileHandler(Constants.LOG_FILE_PATH, encoding='utf-8')
+        file_handler.setLevel(logging.INFO)
+
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(logging.INFO)
+
+        formatter = logging.Formatter('%(levelname)s:%(asctime)s \t%(pathname)s:line:%(lineno)d \t%(message)s')
+
+        file_handler.setFormatter(formatter)
+        console_handler.setFormatter(formatter)
+
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+
+        logger.addHandler(file_handler)
+        logger.addHandler(console_handler)
+
+        logging.info('Starting application')
+
+    def log_the_start_of_application(self, is_search_only: bool):
+        if is_search_only:
+            time_format = '%Y-%m-%d %H:%M:%S'
+            datetime_now = datetime.now(timezone('US/Eastern'))
+            logging.info('Starting the search...')
+            logging.info(f"Search Date & Time:{datetime_now.strftime(time_format)}\nResults:\n")
+        else:
+            logging.info('Starting the update...')

ado_express/packages/toolbox/release_manager/__init__.py

@@ -0,0 +1,2 @@
+from .update_progress_retriever import *
+from .update_release import *

ado_express/packages/toolbox/release_manager/update_progress_retriever/__init__.py

@@ -0,0 +1 @@
+from .update_progress_retriever import *