bpt_model.py

# Adafruit Arduino Board Package Tool (bpt) Data Model
# Classes that support bpt's commands.  These classes represent the Arduino
# packages, board index, and configuration of the tool.
# Author: Tony DiCola
#
# Copyright (c) 2016 Adafruit Industries
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import configparser
import hashlib
import json
import logging
import os
import os.path
import posixpath
import re
import shutil
import tarfile
import tempfile

from git import Repo
from pkg_resources import parse_version


logger = logging.getLogger(__name__)


class BoardPackage(object):
    """Board package instance state (name, version, etc.)."""

    def __init__(self, parent, template, name, version, origin, archive_prefix=None):
        """Initialize board package instance with specified state:
          - parent = name of parent package that contains this instance
          - template = JSON string to use as template when writing this instance
                       back to the board index
          - name = name of the board package instance
          - version = version of the board package instance
          - origin = string that describes the source of this board package
          - archive_prefix = prefix to use when constructing the board package
                             archive filename
        """
        self._parent = parent
        self._name = name
        self._version = version
        self._origin = origin
        self._archive_prefix = archive_prefix
        self._template = template

    def get_parent(self):
        """Return the parent package name in the board index that owns this board
        package.
        """
        return self._parent

    def get_template(self):
        """Return a string template that contains a JSON object which will be
        inserted into the board index.
        """
        return self._template

    def get_version(self):
        """Return the version of this board package instance."""
        return self._version

    def get_name(self):
        """Return the name of this board package instance."""
        return self._name

    def get_origin(self):
        """Return a description of the origin for this package, i.e. the
        directory or Git repo that is the source for this package.
        """
        return self._origin

    def get_archive_name(self):
        """Return the name of the archive file for this board package.  This
        name is set as a '<prefix>-<version>.tar.bz2' where <prefix> defaults
        to the name of the package but can be overridden, and <version> is the
        version of the package.
        """
        prefix = self.get_name()
        if self._archive_prefix is not None:
            prefix = self._archive_prefix
        return '{0}-{1}.tar.bz2'.format(prefix, self.get_version())

    def close(self):
        """Close will be called at the termination of the program and gives a
        good place for subclasses to cleanup any temporary data.
        """
        pass

    def write_archive(self, target):
        """Create an archive that is compressed and in the expected format for
        a board package (.tar.bz2).  Will write the contents to the specified
        target file name.  Returns a tuple of (archive size in bytes, archive
        SHA256 hash).
        """
        raise NotImplementedError


class DirectoryBoardPackage(BoardPackage):
    """Board package that lives inside a directory on the machine."""

    def __init__(self, directory, origin=None, **kwargs):
        """Initialize board package instance pointing at the specified local
        directory.
        """
        self._directory = directory
        version = None
        # Check that the directory exists and has a platform.txt that is
        # readable (i.e. is an Arduino board package).
        platform_file = os.path.join(directory, 'platform.txt')
        with open(platform_file, 'r') as platform_txt:
            # Parse out the platform version.
            for line in platform_txt.readlines():
                # Look for 'version=<platform version>' line and grab the version.
                # Grab everything up to the end of line or start of comment (#).
                match = re.match('version=([^#]+)', line, flags=re.IGNORECASE)
                if match:
                    version = match.group(1).strip()
        # Check that a version was found inside the package.
        assert version is not None, 'Expected version for package: {0}'.format(name)
        # Set the origin location if it wasn't provided already.
        if origin is None:
            origin = 'directory: {0}'.format(directory)
        super(DirectoryBoardPackage, self).__init__(version=version, origin=origin,
            **kwargs)

    def write_archive(self, target):
        """Create an archive that is compressed and in the expected format for
        a board package (.tar.bz2).  Will write the contents to the specified
        target file name.  Returns a tuple of (archive size in bytes, archive
        SHA256 hash).
        """
        # Create .tar.bz2 archive of the package directory.
        # Put files inside a folder with same name as archive (minus extension)
        arcname = os.path.basename(target)[:-len('.tar.bz2')]
        with tarfile.open(target, 'w:bz2') as archive:
            archive.add(self._directory,
                arcname=arcname,
                filter=lambda x: None if x.name.startswith(arcname + '/.git') else x)  # Don't add .git folder!
        # Get the size of the archive.
        size = os.stat(target).st_size
        # Generate a SHA256 hash of the archive.
        with open(target, 'rb') as archive:
            sha256 = hashlib.sha256(archive.read()).hexdigest()
        return (size, sha256)


class GitBoardPackage(DirectoryBoardPackage):
    """Board package that lives in a remote Git repository."""

    def __init__(self, repo, repo_dir, **kwargs):
        """Initialize board package using the contents of the specified Git
        repository.  Repo should be a URL that can be cloned with Git and its
        contents will be cloned in a temporary directory.
        """
        # Create a temporary directory to clone the repository.
        self._local_dir = tempfile.mkdtemp()
        # Clone the repo and its submodules to the temp directory.
        logger.debug('GitBoardPackage cloning repo {0} to directory {1}'.format(repo, self._local_dir))
        cloned_repo = Repo.clone_from(repo, self._local_dir)
        cloned_repo.submodule_update(recursive=False)

        # Find path to repo dir inside cloned directory.
        target_dir = self._local_dir
        if repo_dir is not None:
            # Parse the directories from the board config repo_dir value.
            # Note that by convention this is in Unix-style path separators
            # (forward slashes, '/').
            repo_dir = posixpath.normpath(repo_dir)
            dirs = repo_dir.split(posixpath.sep)
            # Now use the current OS path join to point at the repo_dir.
            target_dir = os.path.join(self._local_dir, *dirs)
        # Finish initializing using the cloned repo directory.
        super(GitBoardPackage, self).__init__(target_dir, origin='git: {0}'.format(repo),
            **kwargs)

    def close(self):
        """Clean up temporary location that holds remote Git repository files."""
        if self._local_dir is not None:
            logger.debug('Deleting temporary directory {0}'.format(self._local_dir))
            shutil.rmtree(self._local_dir, ignore_errors=True)


class BoardIndex(object):
    """Board index that is the master list of packages published to Arduino
    clients.
    """

    def __init__(self, index_data):
        """Initialize board index with JSON decoded dict of board index data.
        """
        self._data = index_data
        # Load all the packages and add them to a dict indexed by package name
        # for quick lookup.
        self._packages = {}
        for package in self._data.get('packages', []):
            name = package.get('name')
            assert name is not None and name != '', 'Board index package must specify a name!'
            self._packages[name] = package

    def get_packages(self):
        """Retrieve a list of all the packages."""
        return self._packages.values()

    def get_platforms(self, package, name=None):
        """Retrieve a list of all platforms for the specified package name.
        Can optionally filter by platforms of the specified name (i.e. to get
        all the different versions of that platform).
        """
        platforms = self._packages[package].get('platforms', [])
        if name is None:
            return platforms
        else:
            return filter(lambda x: x.get('name') == name, platforms)

    def add_platform(self, package, platform):
        """Add a platform to the specified package."""
        parent = self._packages[package]
        parent.get('platforms', []).append(platform)

    def write_json(self):
        """Serialize the board index data into JSON so it can be written to a
        file.  Will return the JSON string of the data.
        """
        return json.dumps(self._data, indent=2, separators=(',', ': '))

    def transform_urls(self, transforms):
        """Transform all the urls inside platforms using the specified list
        of string transformations.  Each transform entry should be a 2-tuple
        with target, the string to search for from the beginning, and value,
        the string to replace target with.  For example the tuple ('https://',
        'http://') would convert SSL to non-SSL.  Note that target searching is
        case insensitive!
        """
        # Walk all the packages and platforms inside them.
        for package in self._data.get('packages', []):
            for platform in package.get('platforms', []):
                # Look for any url attribute and apply transformations.
                if 'url' not in platform:
                    continue
                for transform in transforms:
                    target, value = transform
                    # Look for the first instance of target in the string (being
                    # careful to be case insensitive).
                    url = platform['url']
                    start = url.lower().find(target.lower())
                    if start != -1:
                        # Replace target with value.
                        platform['url'] = ''.join([url[:start], value, url[start+len(target):]])

class BoardConfig(object):
    """Represents a board configuration INI file.  This configuration can define
    a list of board package locations, either as directories or Git repositories.
    This is useful for predefining a set of board packages in a configuration
    file that saves having to use a lot of command line parameters to specify
    each pacakge location.
    """

    def __init__(self, board_config):
        """Initialize board package config using the specified board_config
        file path.  If board_config is None or empty then no board pacakges will
        be loaded.  Otherwise board_config should point at an INI file with
        a section for each board package and attributes inside:
          - package = name of parent package inside board index (required)
          - directory = path to a directory on the machine with the board package
          - repo = git repository URL that holds the board package
        """
        self._packages = []
        # Load the INI file and process all the sections.
        self._config = configparser.RawConfigParser()
        self._config.read([board_config])
        for section in self._config.sections():
            logger.debug('Processing config file {0} section {1}'.format(board_config, section))
            # Process required options.
            parent = self._config.get(section, 'index_parent')
            template = self._config.get(section, 'index_template')
            # Process optional options.
            archive_prefix = None
            if self._config.has_option(section, 'archive_prefix'):
                archive_prefix = self._config.get(section, 'archive_prefix')
            # Look for a directory or repo and process accordingly.
            if self._config.has_option(section, 'directory') and \
                self._config.has_option(section, 'repo'):
                # Both options are ambiguous, fail.
                raise RuntimeError('Board package config is ambiguous with both directory and repo!')
            if self._config.has_option(section, 'directory'):
                # Create a local directory-based package source.
                directory = self._config.get(section, 'directory')
                self._packages.append(DirectoryBoardPackage(directory, parent=parent,
                    template=template, name=section, archive_prefix=archive_prefix))
            elif self._config.has_option(section, 'repo'):
                # Create a Git-based package source.
                repo = self._config.get(section, 'repo')
                # Grab optional repo_dir path to platforms.txt.
                repo_dir = None
                if self._config.has_option(section, 'repo_dir'):
                    repo_dir = self._config.get(section, 'repo_dir')
                self._packages.append(GitBoardPackage(repo, repo_dir, parent=parent,
                    template=template, name=section, archive_prefix=archive_prefix))
            else:
                # No known way to read this repo, fail.
                raise RuntimeError('Board package config must specify either directory or repo!')

    def get_packages(self):
        """Return the packages parsed by this configuration file."""
        return self._packages

    def get_package(self, package):
        """Return the specified package (by name), or None if it does not exist
        in the config.
        """
        packages = list(filter(lambda x: x.get_name() == package, self._packages))
        if len(packages) == 0:
            return None
        elif len(packages) == 1:
            return packages[0]
        else:
            raise RuntimeError('Found multiple packages with same name in config, ambiguous result!')