float_status.py

#!/usr/bin/env python3

import json
import sys
import time
import subprocess

from float_login import FloatLogin
from float_utils import logger


class FloatStatus:
    _STATUS_MAP = {
        'Submitted': 'running',
        'Initializing': 'running',
        'Starting': 'running',
        'Executing': 'running',
        'Capturing': 'running',
        'Floating': 'running',
        'Suspended': 'running',
        'Suspending': 'running',
        'Resuming': 'running',
        'Completed': 'success',
        'Cancelled': 'failed',
        'Cancelling': 'failed',
        'FailToComplete': 'failed',
        'FailToExecute': 'failed',
        'CheckpointFailed': 'failed',
        'Timedout': 'failed',
        'NoAvailableHost': 'failed',
        'Unknown': 'failed',
        'WaitingForLicense': 'failed'
    }

    def __init__(self):
        self._cmd = ['float', 'show', '--format', 'json']

    def job_status(self, jobid):
        cmd = self._cmd
        cmd.extend(['--job', jobid])

        try:
            FloatLogin().login()
            output = subprocess.check_output(cmd)
            output = json.loads(output.decode())
            float_status = output['status']
        except subprocess.CalledProcessError:
            msg = f"Failed to get show response for job: {jobid}"
            logger.exception(msg)
            raise
        except (UnicodeError, json.JSONDecodeError):
            msg = f"Failed to decode show response for job: {jobid}"
            logger.exception(msg)
            raise
        except KeyError:
            msg = f"Failed to obtain status for job: {jobid}"
            logger.exception(msg)
            raise

        status = self._STATUS_MAP[float_status]

        # There are too many status checks to log for normal use
        logger.debug(f"Submitted float show for job: {jobid}")
        logger.debug(f"With command: {cmd}")
        logger.debug(f"Obtained float status: {float_status}")
        logger.debug(f"OpCenter response:\n{output}")

        return status


if __name__ == '__main__':
    jobid = sys.argv[1]

    float_status = FloatStatus()
    status = None

    retry_int = 5
    num_retries = 4  # num_attempts - 1
    for attempt in range(num_retries + 1):
        try:
            status = float_status.job_status(jobid)
        except subprocess.CalledProcessError:
            if attempt < num_retries:
                logger.info(
                    f"Retrying status check for job {jobid}"
                    f" in {retry_int} seconds"
                )
                time.sleep(retry_int)
            continue
        break

    if status is None:
        logger.info(f"Failed to obtain status: marking job {jobid} as failed")
        status = 'failed'

    print(status)