upgraded Python dependencies and adjusted to new ruamel.yaml API; fix…

… issue with pip config running as wrong user; fix issue with file upload previews triggering the browser to offer a download

CHANGELOG.md

@@ -1,11 +1,12 @@
 # Change Log
 
-## [1.4.89] - 2023-12-21
+## [1.4.89] - 2023-12-23
 
 ### Added
 - The `invite_user()` function.
 
 ### Changed
+- Upgraded dependencies.
 - The `/api/user/<user_id>` PATCH endpoint now allows changing whether
   the user is `active`.
 - Changed the background color of the signature canvas to grey when

Docker/initialize.sh

@@ -36,9 +36,9 @@ touch /var/run/docassemble/da_running
 
 export DEBIAN_FRONTEND=noninteractive
 if [ "${DAALLOWUPDATES:-true}" == "true" ]; then
-    echo "initialize: running apt-get clean" >&2
+    echo "initialize: Running apt-get clean" >&2
     apt-get clean &> /dev/null
-    echo "initialize: running apt-get update" >&2
+    echo "initialize: Running apt-get update" >&2
     apt-get -q -y update &> /dev/null
 fi
 
@@ -511,6 +511,14 @@ echo "initialize: Checking to see if this is the first time the server was initi
 
 DAINSTALLASROOT=true
 
+if [ "${DAROOTOWNED:-false}" == "true" ]; then
+    if [ "${DAALLOWUPDATES:-true}" == "true" ] || [ "${DAENABLEPLAYGROUND:-true}" == "true" ]; then
+        DAINSTALLASROOT=false
+    fi
+else
+    DAINSTALLASROOT=false
+fi
+
 if [ "${DASUPERVISORUSERNAME:-null}" != "null" ]; then
     export SUPERVISORCMD="supervisorctl --serverurl http://localhost:9001 --username ${DASUPERVISORUSERNAME} --password ${DASUPERVISORPASSWORD}"
 else
@@ -524,7 +532,6 @@ if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
             if [ "${DAALLOWUPDATES:-true}" == "true" ] \
                    || [ "${DAENABLEPLAYGROUND:-true}" == "true" ]; then
                 chown -R www-data:www-data /usr/share/docassemble/local3.10
-                DAINSTALLASROOT=false
             else
                 echo "initialize: Python virtual environment is read-only" >&2
             fi
@@ -546,7 +553,6 @@ if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
             chown -R www-data:www-data /usr/share/docassemble/local3.10
             chown -R www-data:www-data /usr/share/docassemble/config \
                   /usr/share/docassemble/webapp/docassemble.wsgi
-            DAINSTALLASROOT=false
         fi
         touch /etc/hasbeeninitialized
     else

Dockerfile

@@ -53,38 +53,38 @@ bash -c \
 && cp /usr/local/bin/unoconv /usr/bin/unoconv \
 && /usr/bin/python3 -m venv --copies /usr/share/docassemble/local3.10 \
 && source /usr/share/docassemble/local3.10/bin/activate \
-&& pip3 install --upgrade pip==23.2.1 \
-&& pip3 install --upgrade wheel==0.41.0 \
-&& pip3 install --upgrade mod_wsgi==4.9.4 \
+&& pip3 install --upgrade pip==23.3.2 \
+&& pip3 install --upgrade wheel==0.42.0 \
+&& pip3 install --upgrade mod_wsgi==5.0.0 \
 && pip3 install --upgrade \
-   acme==2.7.1 \
-   certbot==2.7.1 \
-   certbot-apache==2.7.1 \
-   certbot-nginx==2.7.1 \
-   certifi==2023.7.22 \
+   acme==2.8.0 \
+   certbot==2.8.0 \
+   certbot-apache==2.8.0 \
+   certbot-nginx==2.8.0 \
+   certifi==2023.11.17 \
    cffi==1.16.0 \
-   charset-normalizer==3.3.0 \
-   click==8.1.6 \
+   charset-normalizer==3.3.2 \
+   click==8.1.7 \
    ConfigArgParse==1.7 \
    configobj==5.0.8 \
-   cryptography==41.0.4 \
+   cryptography==41.0.7 \
    distro==1.8.0 \
-   idna==3.4 \
-   joblib==1.3.1 \
-   josepy==1.13.0 \
+   idna==3.6 \
+   joblib==1.3.2 \
+   josepy==1.14.0 \
    nltk==3.8.1 \
    parsedatetime==2.6 \
    pycparser==2.21 \
-   pyOpenSSL==23.2.0 \
+   pyOpenSSL==23.3.0 \
    pyparsing==3.1.1 \
    pyRFC3339==1.1 \
    python-augeas==1.1.0 \
    pytz==2023.3.post1 \
-   regex==2023.6.3 \
+   regex==2023.10.3 \
    requests==2.31.0 \
    six==1.16.0 \
-   tqdm==4.65.0 \
-   urllib3==2.0.6 \
+   tqdm==4.66.1 \
+   urllib3==2.0.7 \
 && pip3 install \
    /tmp/docassemble/docassemble \
    /tmp/docassemble/docassemble_base \

docassemble_base/docassemble/base/data/sources/base-words.yml

@@ -55,7 +55,6 @@
 "API Keys": Null
 "Apply": Null
 "App": Null
-"A QR code": Null
 ", are attached": Null
 "A reset password email has been sent to '%(email)s'. Open that email and follow the instructions to reset your password.": Null
 "are you": Null
@@ -788,6 +787,7 @@
 "PyPI package": Null
 "PyPI Password": Null
 "PyPI Username": Null
+"Python version %s": Null
 "Qatar": Null
 "question": Null
 "Question": Null
@@ -1194,7 +1194,6 @@
 "Verification code": Null
 "Verify": Null
 "Verify your phone": Null
-"Version ": Null
 "Version": Null
 "Version number does not conform to PEP 440": Null
 "Viet Nam": Null

docassemble_base/docassemble/base/parse.py

@@ -23,7 +23,7 @@
 import xml.etree.ElementTree as ET
 from html.parser import HTMLParser
 from itertools import groupby, chain
-from ruamel import yaml
+import ruamel.yaml
 from jinja2 import ChainableUndefined
 from jinja2.runtime import StrictUndefined, UndefinedError
 from jinja2.exceptions import TemplateError
@@ -58,6 +58,17 @@
 from docassemble.base.mako.exceptions import SyntaxException, CompileException
 from docassemble.base.astparser import myvisitnode
 
+prettyyaml = ruamel.yaml.YAML(typ=['safe', 'string'], pure=True)
+prettyyaml.indent(mapping=2, sequence=4, offset=2)
+prettyyaml.default_flow_style = False
+prettyyaml.default_style = '|'
+prettyyaml.allow_unicode = True
+altyaml = ruamel.yaml.YAML(typ=['safe', 'string'], pure=True)
+altyaml.indent(mapping=2, sequence=4, offset=2)
+altyaml.default_flow_style = False
+altyaml.default_style = '|'
+altyaml.allow_unicode = False
+safeyaml = ruamel.yaml.YAML(typ=['safe', 'string'], pure=True)
 equals_byte = bytes('=', 'utf-8')
 RangeType = type(range(1, 2))
 NoneType = type(None)
@@ -2031,9 +2042,9 @@ class Question:
     def idebug(self, data):
         if hasattr(self, 'from_source') and hasattr(self, 'package'):
             if isinstance(self.line_number, int):
-                return f"\nIn file {self.from_source.path} in the block on line {self.line_number} from package {self.package}:\n\n" + yaml.dump(data)
-            return "\nIn file " + str(self.from_source.path) + " from package " + str(self.package) + ":\n\n" + yaml.dump(data)
-        return yaml.dump(data)
+                return f"\nIn file {self.from_source.path} in the block on line {self.line_number} from package {self.package}:\n\n" + safeyaml.dump_to_string(data)
+            return "\nIn file " + str(self.from_source.path) + " from package " + str(self.package) + ":\n\n" + safeyaml.dump_to_string(data)
+        return safeyaml.dump_to_string(data)
 
     def __init__(self, orig_data, caller, **kwargs):
         if not isinstance(orig_data, dict):
@@ -6791,7 +6802,7 @@ def parse_fields(self, the_list, register_target, uses_field):
                     if not uses_value_label:
                         result_dict['label'] = TextObject(key, question=self)
                     self.embeds = True
-                    result_dict['key'] = Question(value, self.interview, register_target=register_target, source=self.from_source, package=self.package, source_code=codecs.decode(bytearray(yaml.safe_dump(value, default_flow_style=False, default_style='|', allow_unicode=True), encoding='utf-8'), 'utf-8'))
+                    result_dict['key'] = Question(value, self.interview, register_target=register_target, source=self.from_source, package=self.package, source_code=prettyyaml.dump_to_string(value))
                 elif isinstance(value, str):
                     if value in ('exit', 'logout', 'exit_logout', 'leave') and 'url' in the_dict:
                         self.embeds = True
@@ -7534,7 +7545,7 @@ def prepare_attachment(self, attachment, the_user_dict):
                                     modified_metadata[key] = data + str('[END]')
                                 else:
                                     modified_metadata[key] = data
-                            the_markdown += '---\n' + codecs.decode(bytearray(yaml.safe_dump(modified_metadata, default_flow_style=False, default_style='|', allow_unicode=False), encoding='utf-8'), 'utf-8') + "...\n"
+                            the_markdown += '---\n' + altyaml.dump_to_string(modified_metadata) + "\n...\n"
                         docassemble.base.functions.set_context('pandoc')
                         the_markdown += the_content.text(the_user_dict)
                         # logmessage("Markdown is:\n" + repr(the_markdown) + "END")
@@ -8214,15 +8225,15 @@ def read_from(self, source):
             if source.testing:
                 try:
                     # logmessage("Package is " + str(source_package))
-                    document = yaml.safe_load(source_code)
+                    document = safeyaml.load(source_code)
                     if document is not None:
                         question = Question(document, self, source=source, package=source_package, source_code=source_code, line_number=line_number)
                         self.names_used.update(question.fields_used)
                 except Exception as errMess:
                     # logmessage(str(source_code))
                     try:
                         # Correct line numbers to be global to the YAML
-                        if isinstance(errMess, yaml.error.MarkedYAMLError):
+                        if isinstance(errMess, safeyaml.error.MarkedYAMLError):
                             if errMess.context_mark is not None:
                                 errMess.context_mark.line += (line_number - 1)
                             if errMess.problem_mark is not None:
@@ -8232,21 +8243,18 @@ def read_from(self, source):
                         try:
                             logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error was:\n\n{errMess}')
                         except:
-                            if isinstance(errMess, yaml.error.MarkedYAMLError):
-                                logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error type was:\n\n{errMess.problem}')
-                            else:
-                                logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error type was:\n\n' + errMess.__class__.__name__)
+                            logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error type was:\n\n' + errMess.__class__.__name__)
                     self.success = False
             else:
                 try:
-                    document = yaml.safe_load(source_code)
+                    document = safeyaml.load(source_code)
                 except Exception as errMess:
                     self.success = False
                     # logmessage("Error: " + str(source_code))
                     # str(source_code)
                     try:
                         # Correct line numbers to be global to the YAML
-                        if isinstance(errMess, yaml.error.MarkedYAMLError):
+                        if isinstance(errMess, safeyaml.error.MarkedYAMLError):
                             if errMess.context_mark is not None:
                                 errMess.context_mark.line += (line_number - 1)
                             if errMess.problem_mark is not None:

docassemble_base/docassemble/base/pdftk.py

@@ -367,7 +367,7 @@ def fill_template(template, data_strings=None, data_names=None, hidden=None, rea
         if len(image_todo) > 0:
             for item in image_todo:
                 xone, yone, xtwo, ytwo = fields[item['field']]['rect']
-                logmessage("Trying to save to page " + repr(item['pageno'] - 1))
+                # logmessage("Trying to save to page " + repr(item['pageno'] - 1))
                 with Pdf.open(item['overlay_file']) as overlay_file:
                     overlay_page = overlay_file.pages[0]
                     pdf.pages[item['pageno'] - 1].add_overlay(overlay_page, rect=pikepdf.Rectangle(xone, yone, xtwo, ytwo))