From fae81df11eabcfd91bcc4486390c7e4d5c4d133d Mon Sep 17 00:00:00 2001 From: Sebastien Mirolo Date: Thu, 12 Oct 2023 09:36:36 -0700 Subject: [PATCH] enables download of templates only, skip assets This requires the djaoapp project to use djaodjin-extended-templates>=0.4.5 --- deployutils/__init__.py | 2 +- .../management/commands/package_theme.py | 17 ++- deployutils/apps/django/themes.py | 103 +++++++++++------- deployutils/copy.py | 6 +- deployutils/djd.py | 16 ++- testsite/requirements.txt | 2 +- 6 files changed, 95 insertions(+), 51 deletions(-) diff --git a/deployutils/__init__.py b/deployutils/__init__.py index 61f0b45..b8b851f 100644 --- a/deployutils/__init__.py +++ b/deployutils/__init__.py @@ -22,4 +22,4 @@ # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = '0.10.6' +__version__ = '0.10.7-dev' diff --git a/deployutils/apps/django/management/commands/package_theme.py b/deployutils/apps/django/management/commands/package_theme.py index 04462e9..35656cd 100644 --- a/deployutils/apps/django/management/commands/package_theme.py +++ b/deployutils/apps/django/management/commands/package_theme.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, Djaodjin Inc. +# Copyright (c) 2023, Djaodjin Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -25,13 +25,15 @@ from __future__ import absolute_import from __future__ import unicode_literals -import sys +import logging from . import ResourceCommand from ... import settings from ...themes import (init_build_and_install_dirs, package_assets, package_theme, fill_package) +LOGGER = logging.getLogger(__name__.split('.',maxsplit=1)[0]) + class Command(ResourceCommand): """ @@ -84,6 +86,9 @@ class Command(ResourceCommand): def add_arguments(self, parser): super(Command, self).add_arguments(parser) + parser.add_argument('--verbose', action='store_true', dest='verbose', + default=False, + help='verbose mode') parser.add_argument('--app_name', action='store', dest='app_name', default=settings.APP_NAME, help='overrides the destination site name') @@ -103,6 +108,8 @@ def add_arguments(self, parser): ' (after excludes have been applied)') def handle(self, *args, **options): + if options['verbose']: + LOGGER.setLevel(logging.DEBUG) app_name = options['app_name'] build_dir, install_dir = init_build_and_install_dirs(app_name, build_dir=options['build_dir'], @@ -111,8 +118,10 @@ def handle(self, *args, **options): excludes=options['excludes'], includes=options['includes'], path_prefix=options['path_prefix']) - package_assets(app_name, build_dir=build_dir) + package_assets(app_name, build_dir=build_dir, + excludes=options['excludes'], + includes=options['includes']) zip_path = fill_package(app_name, build_dir=build_dir, install_dir=install_dir) - sys.stdout.write('package built: %s\n' % zip_path) + self.stdout.write('package built: %s\n' % zip_path) diff --git a/deployutils/apps/django/themes.py b/deployutils/apps/django/themes.py index 6ceaf2a..36f5f38 100644 --- a/deployutils/apps/django/themes.py +++ b/deployutils/apps/django/themes.py @@ -31,6 +31,7 @@ from django.template.base import (Parser, NodeList, TemplateSyntaxError) from django.template.backends.django import DjangoTemplates from django.template.context import Context +from django.utils._os import safe_join from django_assets.templatetags.assets import assets from jinja2.lexer import Lexer from webassets import Bundle @@ -234,47 +235,62 @@ def init_build_and_install_dirs(app_name, build_dir=None, install_dir=None): return build_dir, install_dir -def package_assets(app_name, build_dir):#pylint:disable=unused-argument +def package_assets(app_name, build_dir, + excludes=None, includes=None): + #pylint:disable=unused-argument resources_dest = os.path.join(build_dir, 'public') # Copy local resources (not under source control) to resources_dest. - excludes = ['--exclude', '*~', '--exclude', '.DS_Store', + exclude_args = ['--exclude', '*~', '--exclude', '.DS_Store', '--exclude', '.webassets-cache'] + if excludes: + for pat in excludes: + exclude_args += ['--exclude', pat] app_static_root = django_settings.STATIC_ROOT assert app_static_root is not None and app_static_root # When app_static_root ends with the static_url, we will want # to insert the app_name prefix. - static_root_parts = app_static_root.split(os.sep) + static_root_parts = app_static_root.split(os.path.sep) root_parts_idx = len(static_root_parts) root_idx = len(app_static_root) found = False orig_static_url = django_settings.STATIC_URL - orig_static_url_parts = orig_static_url.split('/') - if not orig_static_url_parts[0]: - orig_static_url_parts = orig_static_url_parts[1:] - if orig_static_url_parts[0] == app_name: - orig_static_url_parts = orig_static_url_parts[1:] - for url_part in reversed(orig_static_url_parts): - found = True # With ``break`` later on to default to False - # when zero iteration. - if url_part: - root_parts_idx = root_parts_idx - 1 - root_idx = root_idx - len(static_root_parts[root_parts_idx]) - 1 - if url_part != static_root_parts[root_parts_idx]: - found = False - break - if found: - app_static_root = os.path.join( - app_static_root[:root_idx], django_settings.STATIC_URL[1:-1]) + static_url_parts = orig_static_url.strip('/').split('/') + path_parts = app_static_root.strip('/').split('/') + root_idx = 0 + for path_part, url_part in zip(reversed(path_parts), + reversed(static_url_parts)): + if path_part != url_part: + break + root_idx += 1 + if root_idx: + app_static_root = os.path.sep + os.path.join(*path_parts[:-root_idx]) + if not app_static_root.endswith(os.path.sep): + app_static_root = app_static_root + os.path.sep # static_url is required per-Django to start and ends with a '/' # (i.e. '/static/'). # If we have a trailing '/', rsync will copy the content # of the directory instead of the directory itself. - cmdline = (['/usr/bin/rsync'] - + excludes + ['-az', '--safe-links', '--rsync-path', '/usr/bin/rsync'] - + [app_static_root, resources_dest]) - LOGGER.info(' '.join(cmdline)) - shell_command(cmdline) + cmdline_root = ['/usr/bin/rsync'] + exclude_args + [ + '-az', '--safe-links', '--rsync-path', '/usr/bin/rsync'] + if False and includes: + # XXX includes should add back excluded content to match + # the `package_theme` implementation. + for include in includes: + include_static_root = safe_join(app_static_root, include) + if os.path.exists(include_static_root): + include_parts = include.strip('/').split('/') + if len(include_parts) > 1: + include_resources_dest = safe_join( + resources_dest, os.path.join(*include_parts[:-1])) + if not os.path.exists(include_resources_dest): + os.makedirs(include_resources_dest) + cmdline = cmdline_root + [ + include_static_root, include_resources_dest] + shell_command(cmdline) + else: + cmdline = cmdline_root + [app_static_root, resources_dest] + shell_command(cmdline) def package_theme(app_name, build_dir, @@ -331,6 +347,22 @@ def fill_package_zip(zip_file, srcroot, prefix=''): fill_package_zip(zip_file, srcroot, prefix=pathname) +def _list_templates(srcroot, prefix=''): + """ + List all templates in srcroot + """ + results = [] + for pathname in os.listdir(os.path.join(srcroot, prefix)): + pathname = os.path.join(prefix, pathname) + source_name = os.path.join(srcroot, pathname) + if os.path.isfile(source_name): + results += [pathname] + elif os.path.isdir(source_name): + if not source_name.endswith('jinja2'): + results += _list_templates(srcroot, prefix=pathname) + return results + + def install_templates(srcroot, destroot, prefix='', excludes=None, includes=None, path_prefix=None): #pylint:disable=too-many-arguments,too-many-statements @@ -339,29 +371,29 @@ def install_templates(srcroot, destroot, prefix='', excludes=None, and its subdirectories. """ #pylint: disable=too-many-locals - if excludes is None: - excludes = [] + exclude_pats = [r'.*~', r'\.DS_Store'] + if excludes: + exclude_pats += excludes if includes is None: includes = [] if not os.path.exists(os.path.join(prefix, destroot)): os.makedirs(os.path.join(prefix, destroot)) - for pathname in os.listdir(os.path.join(srcroot, prefix)): - pathname = os.path.join(prefix, pathname) + for pathname in _list_templates(srcroot): + source_name = os.path.join(srcroot, pathname) + dest_name = os.path.join(destroot, pathname) excluded = False - for pat in excludes: - if re.match(pat, pathname): + for pat in exclude_pats: + if re.search(pat, pathname): excluded = True break if excluded: for pat in includes: - if re.match(pat, pathname): + if re.search(pat, pathname): excluded = False break if excluded: LOGGER.debug("skip %s", pathname) continue - source_name = os.path.join(srcroot, pathname) - dest_name = os.path.join(destroot, pathname) LOGGER.debug("%s %s %s", "install" if ( os.path.isfile(source_name) and not os.path.exists(dest_name)) else "pass", source_name, dest_name) @@ -464,6 +496,3 @@ def install_templates(srcroot, destroot, prefix='', excludes=None, except UnicodeDecodeError: LOGGER.warning("%s: Templates can only be constructed " "from unicode or UTF-8 strings.", source_name) - elif os.path.isdir(source_name): - install_templates(srcroot, destroot, prefix=pathname, - excludes=excludes, includes=includes, path_prefix=path_prefix) diff --git a/deployutils/copy.py b/deployutils/copy.py index 11970cd..2eb4638 100644 --- a/deployutils/copy.py +++ b/deployutils/copy.py @@ -76,12 +76,14 @@ def download(remote_location, remotes=None, prefix="", dry_run=False): '%s/./' % remote_location, dest_root], dry_run=dry_run) -def download_theme(args, base_url, api_key, prefix=None): +def download_theme(args, base_url, api_key, prefix=None, templates_only=False): """ Downloads a project theme. """ #pylint:disable=unused-argument api_themes_url = base_url + '/themes/download/' + if templates_only: + api_themes_url += '?templates_only=true' resp = requests.get(api_themes_url, auth=(api_key, "")) LOGGER.info("GET %s returns %d", api_themes_url, resp.status_code) fname = re.findall(r'filename="(.+)"', @@ -133,7 +135,7 @@ def upload(remote_location, remotes=None, ignores=None, shell_command(cmdline, dry_run=dry_run) -def upload_theme(args, base_url, api_key, prefix=None): +def upload_theme(args, base_url, api_key, prefix=None, templates_only=False): """ Uploads a new theme for a project. """ diff --git a/deployutils/djd.py b/deployutils/djd.py index 09f6e26..3ec6c3b 100644 --- a/deployutils/djd.py +++ b/deployutils/djd.py @@ -239,14 +239,18 @@ def pub_deploy(args, project="", account="", api_key=""): api_container_url, resp.status_code, resp.text) -def pub_download(args, project="", base_url="", api_key=""): +def pub_download(args, project="", base_url="", api_key="", + templates_only=False): """Download a theme package for a project. + --templates-only download templates only, + skip assets. """ project, base_url, api_key, updated = get_project_config( project=project, base_url=base_url, api_key=api_key) if updated: save_config() - download_theme(args, base_url, api_key, prefix=project) + download_theme(args, base_url, api_key, prefix=project, + templates_only=templates_only) def pub_init(args, project="", account="", base_url="", @@ -279,14 +283,15 @@ def pub_tunnel(args, project="", base_url="", api_key=""): ssh_reverse_tunnel(args, base_url, api_key, prefix=project) -def pub_upload(args, project="", base_url="", api_key=""): +def pub_upload(args, project="", base_url="", api_key="", templates_only=False): """Upload a theme package (or directory) for a project. """ project, base_url, api_key, updated = get_project_config( project=project, base_url=base_url, api_key=api_key) if updated: save_config() - upload_theme(args, base_url, api_key, prefix=project) + upload_theme(args, base_url, api_key, prefix=project, + templates_only=templates_only) def main(args): @@ -295,7 +300,6 @@ def main(args): """ global CONFIG_FILENAME try: - import __main__ parser = argparse.ArgumentParser( usage='%(prog)s [options] command\n\nVersion\n %(prog)s version ' + str(__version__), @@ -314,7 +318,7 @@ def main(args): '--config', action='store', default=os.path.join(os.getenv('HOME'), '.djd', 'credentials'), help='configuration file') - build_subcommands_parser(parser, __main__) + build_subcommands_parser(parser, sys.modules[__name__]) if len(args) <= 1: parser.print_help() diff --git a/testsite/requirements.txt b/testsite/requirements.txt index 15cb0e6..07d93df 100644 --- a/testsite/requirements.txt +++ b/testsite/requirements.txt @@ -1,4 +1,4 @@ -Django==3.2.19 +Django==3.2.20 djangorestframework==3.14.0 Jinja2==3.1.1 PyJWT==2.6.0