Merge pull request #989 from pierotofy/desktop

Desktop mode

Dockerfile

@@ -20,7 +20,6 @@ RUN apt-get -qq update && apt-get -qq install -y nodejs
 # Install Python3, GDAL, nginx, letsencrypt, psql
 RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-12 gettext
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
-RUN ln -s /usr/bin/pip3 /usr/bin/pip && pip install -U pip
 
 # Install pip reqs
 ADD requirements.txt /webodm/
@@ -40,8 +39,8 @@ WORKDIR /webodm
 RUN npm install --quiet -g webpack@4.16.5 && npm install --quiet -g webpack-cli@4.2.0 && npm install --quiet && webpack --mode production
 RUN echo "UTC" > /etc/timezone
 RUN python manage.py collectstatic --noinput
-RUN bash app/scripts/plugin_cleanup.sh && echo "from app.plugins import build_plugins;build_plugins()" | python manage.py shell
-RUN bash translate.sh build safe
+RUN python manage.py rebuildplugins
+RUN python manage.py translate build --safe
 
 # Cleanup
 RUN apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y

app/boot.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import kombu
 from django.contrib.auth.models import Permission
@@ -82,7 +83,7 @@ def boot():
             s.app_logo.save(os.path.basename(settings.APP_DEFAULT_LOGO), File(open(settings.APP_DEFAULT_LOGO, 'rb')))
 
             logger.info("Created settings")
-
+        
         init_plugins()
 
         if not settings.TESTING:

app/management/commands/addnode.py

@@ -0,0 +1,23 @@
+import os
+from django.core.management.base import BaseCommand
+from nodeodm.models import ProcessingNode
+
+class Command(BaseCommand):
+    requires_system_checks = []
+
+    def add_arguments(self, parser):
+        parser.add_argument("host", type=str)
+        parser.add_argument("port", type=int)
+        parser.add_argument("--label", type=str, required=False, default="", help="Node label")
+        parser.add_argument("--token", type=str, required=False, default="", help="Node token")
+
+        super(Command, self).add_arguments(parser)
+
+    def handle(self, **options):
+        ProcessingNode.objects.update_or_create(hostname=options.get('host'), 
+                defaults={
+                    'hostname': options.get('host'), 
+                    'port': options.get('port'), 
+                    'label': options.get('label', ''),
+                    'token': options.get('token', '')
+                })

app/management/commands/rebuildplugins.py

@@ -0,0 +1,27 @@
+import os
+import shutil
+import glob
+from django.core.management.base import BaseCommand
+from app.plugins import build_plugins
+
+def cleanup():
+    # Delete all node_modules and build directories within plugins' public/ folders
+    root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".."))
+    for d in glob.glob(os.path.join(root, "plugins", "**", "public", "node_modules")):
+        shutil.rmtree(d)
+        print("R\t" + d)
+    for d in glob.glob(os.path.join(root, "plugins", "**", "public", "build")):
+        shutil.rmtree(d)
+        print("R\t" + d)
+
+    print("Cleanup done!")
+
+class Command(BaseCommand):
+    requires_system_checks = []
+
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+
+    def handle(self, **options):
+        cleanup()
+        build_plugins()

app/management/commands/translate.py

@@ -0,0 +1,54 @@
+import os
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+
+from app.scripts.extract_odm_strings import extract_odm_strings
+from app.scripts.extract_plugin_manifest_strings import extract_plugin_manifest_strings
+from app.scripts.extract_potree_strings import extract_potree_strings
+
+root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".."))
+
+class Command(BaseCommand):
+    requires_system_checks = []
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", type=str, choices=['extract', 'build'])
+        parser.add_argument("--safe", action='store_true', required=False, help="Skip invalid languages")
+        super(Command, self).add_arguments(parser)
+
+    def handle(self, **options):
+        with open(os.path.join(root, "LOCALES")) as f:
+            locales = f.read().strip().split(" ")
+
+        if options.get('action') == 'extract':
+            print("Extracting .po files from Django/React")
+            locale_params = ["--locale=%s" % l for l in locales]
+
+            locale_dir = os.path.join(root, "locale")
+            if not os.path.exists(locale_dir):
+                os.mkdir(locale_dir)
+
+            extract_potree_strings(
+                os.path.join(root, "app", "static", "app", "js", "vendor", "potree", "build", "potree", "resources", "lang", "en", "translation.json"), 
+                os.path.join(root, "app", "static", "app", "js", "translations", "potree_autogenerated.js")
+            )
+            extract_odm_strings(
+                "https://raw.githubusercontent.com/OpenDroneMap/ODM/master/opendm/config.py",
+                os.path.join(root, "app", "static", "app", "js", "translations", "odm_autogenerated.js")
+            )
+            extract_plugin_manifest_strings(
+                os.path.join(root, "plugins"), 
+                os.path.join(root, "app", "translations", "plugin_manifest_autogenerated.py")
+            )
+
+            call_command('makemessages', '--keep-pot', *locale_params, '--ignore=build', '--ignore=app/templates/app/registration/*')
+            call_command('makemessages_djangojs', '--keep-pot', *locale_params, '-d=djangojs', '--extension=jsx', '--extension=js', '--ignore=build', '--ignore=app/static/app/js/vendor', '--ignore=app/static/app/bundles', '--ignore=node_modules', '--language=Python')
+
+        elif options.get('action') == 'build':
+            if options.get('safe'):
+                for l in locales:
+                    print("Building %s .po files into .mo" % l)
+                    call_command('compilemessages', '--locale=%s' % l)
+            else:
+                print("Building .po files into .mo")
+                call_command('compilemessages')

app/plugins/grass_engine.py

@@ -26,8 +26,6 @@ def __init__(self):
 
         if self.grass_binary is None:
             logger.warning("Could not find a GRASS 7 executable. GRASS scripts will not work.")
-        else:
-            logger.info("Initializing GRASS engine using {}".format(self.grass_binary))
 
     def create_context(self, serialized_context = {}):
         if self.grass_binary is None: raise GrassEngineException("GRASS engine is unavailable")

app/plugins/plugin_base.py

@@ -41,7 +41,7 @@ def check_requirements(self):
                 if not os.path.exists(self.get_python_packages_path()):
                     os.makedirs(self.get_python_packages_path(), exist_ok=True)
 
-                p = subprocess.Popen(['pip', 'install', '-U', '-r', 'requirements.txt',
+                p = subprocess.Popen(['python', '-m', 'pip', 'install', '-U', '-r', 'requirements.txt',
                                   '--target', self.get_python_packages_path()],
                                      cwd=self.get_path())
                 p.wait()

app/plugins/pyutils.py

@@ -27,7 +27,7 @@ def requirements_installed(requirements_file, python_path):
     """
     env = os.environ.copy()
     env["PYTHONPATH"] = env.get("PYTHONPATH", "") + ":" + python_path
-    reqs = subprocess.check_output(['pip', 'freeze'], env=env)
+    reqs = subprocess.check_output(['python', '-m', 'pip', 'freeze'], env=env)
     installed_packages = [r.decode().split('==')[0] for r in reqs.split()]
     deps = parse_requirements(requirements_file)
 

app/scripts/extract_odm_strings.py

@@ -3,52 +3,6 @@
 import argparse, os, urllib.request, ast, sys
 from io import StringIO
 
-parser = argparse.ArgumentParser(description='Extract ODM strings.')
-parser.add_argument('input', type=str,
-                    help='URL to ODM\'s config.py')
-parser.add_argument('output', type=str,
-                    help='Where to write resulting translation file')
-args = parser.parse_args()
-
-url = args.input
-outfile = args.output
-
-strings = []
-print("Fetching %s ..." % url)
-res = urllib.request.urlopen(url)
-config_file = res.read().decode('utf-8')
-# config_file = open("test.py").read()
-
-options = {}
-class ArgumentParserStub(argparse.ArgumentParser):
-    def add_argument(self, *args, **kwargs):
-        argparse.ArgumentParser.add_argument(self, *args, **kwargs)
-        options[args[0]] = {}
-        for name, value in kwargs.items():
-            options[args[0]][str(name)] = str(value)
-
-    def add_mutually_exclusive_group(self):
-        return ArgumentParserStub()
-
-# Voodoo! :)
-# ( parse AST, extract "def config()" function, set module to only
-# contain that function, execute module in current scope, 
-# run config function)
-root = ast.parse(config_file)
-new_body = []
-for stmt in root.body:
-    # Assignments
-    if hasattr(stmt, 'targets'):
-        new_body.append(stmt)
-
-    # Functions
-    elif hasattr(stmt, 'name'):
-        new_body.append(stmt)
-
-root.body = new_body
-exec(compile(root, filename="<ast>", mode="exec"))
-
-
 # Misc variables needed to get config to run
 __version__ = '?'
 class context:    
@@ -63,22 +17,72 @@ class log:
     def ODM_ERROR(s):
         pass
 
-config(["--project-path", "/bogus", "name"], parser=ArgumentParserStub())
-for opt in options:
-    h = options[opt].get('help')
-    if h:
-        h = h.replace("\n", "")
-        strings.append(h)
-
-strings = list(set(strings))
-print("Found %s ODM strings" % len(strings))
-if len(strings) > 0:
-    with open(outfile, "w") as f:
-        f.write("// Auto-generated with extract_odm_strings.py, do not edit!\n\n")
-
-        for s in strings:
-            f.write("_(\"%s\");\n" % s.replace("\"", "\\\""))
-
-    print("Wrote %s" % outfile)
-else:
-    print("No strings found")
+def extract_odm_strings(url, outfile):
+    strings = []
+    print("Fetching %s ..." % url)
+    res = urllib.request.urlopen(url)
+    config_file = res.read().decode('utf-8')
+    # config_file = open("test.py").read()
+
+    options = {}
+    class ArgumentParserStub(argparse.ArgumentParser):
+        def add_argument(self, *args, **kwargs):
+            argparse.ArgumentParser.add_argument(self, *args, **kwargs)
+            options[args[0]] = {}
+            for name, value in kwargs.items():
+                options[args[0]][str(name)] = str(value)
+
+        def add_mutually_exclusive_group(self):
+            return ArgumentParserStub()
+
+    # Voodoo! :)
+    # ( parse AST, extract "def config()" function, set module to only
+    # contain that function, execute module in current scope, 
+    # run config function)
+    root = ast.parse(config_file)
+    new_body = []
+    for stmt in root.body:
+        # Assignments
+        if hasattr(stmt, 'targets'):
+            new_body.append(stmt)
+
+        # Functions
+        elif hasattr(stmt, 'name'):
+            new_body.append(stmt)
+
+    root.body = new_body
+    exec(compile(root, filename="<ast>", mode="exec"), globals())
+
+
+
+    config(["--project-path", "/bogus", "name"], parser=ArgumentParserStub())
+    for opt in options:
+        h = options[opt].get('help')
+        if h:
+            h = h.replace("\n", "")
+            strings.append(h)
+
+    strings = list(set(strings))
+    print("Found %s ODM strings" % len(strings))
+    if len(strings) > 0:
+        with open(outfile, "w") as f:
+            f.write("// Auto-generated with extract_odm_strings.py, do not edit!\n\n")
+
+            for s in strings:
+                f.write("_(\"%s\");\n" % s.replace("\"", "\\\""))
+
+        print("Wrote %s" % outfile)
+    else:
+        print("No strings found")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Extract ODM strings.')
+    parser.add_argument('input', type=str,
+                        help='URL to ODM\'s config.py')
+    parser.add_argument('output', type=str,
+                        help='Where to write resulting translation file')
+    args = parser.parse_args()
+
+    url = args.input
+    outfile = args.output

app/scripts/extract_plugin_manifest_strings.py

@@ -1,33 +1,37 @@
 #!/usr/bin/python3
 
-import argparse, json, os, glob
+import json, os, glob
 
-parser = argparse.ArgumentParser(description='Extract plugin manifest strings.')
-parser.add_argument('input', type=str,
-                    help='Path to plugins directory')
-parser.add_argument('output', type=str,
-                    help='Where to write resulting translation file')
-args = parser.parse_args()
+def extract_plugin_manifest_strings(plugins_dir, output):
+    strings = []
+    manifests = glob.glob(os.path.join(plugins_dir, "*", "manifest.json"), recursive=True)
+    print("Found %s manifests" % len(manifests))
 
-strings = []
-manifests = glob.glob(os.path.join(args.input, "*", "manifest.json"), recursive=True)
-print("Found %s manifests" % len(manifests))
+    for m in manifests:
+        with open(m) as f:
+            j = json.loads(f.read())
+            if j.get("description"):
+                strings.append(j.get("description")) 
 
-for m in manifests:
-    with open(m) as f:
-        j = json.loads(f.read())
-        if j.get("description"):
-            strings.append(j.get("description")) 
+    print("Found %s manifest strings" % len(strings))
+    if len(strings) > 0:
+        with open(output, "w") as f:
+            f.write("// Auto-generated with extract_plugin_manifest_strings.py, do not edit!\n\n")
+            f.write("from django.utils.translation import gettext as _\n")
+
+            for s in strings:
+                f.write("_(\"%s\")\n" % s.replace("\"", "\\\""))
 
-print("Found %s manifest strings" % len(strings))
-if len(strings) > 0:
-    with open(args.output, "w") as f:
-        f.write("// Auto-generated with extract_plugin_manifest_strings.py, do not edit!\n\n")
-        f.write("from django.utils.translation import gettext as _\n")
-
-        for s in strings:
-            f.write("_(\"%s\")\n" % s.replace("\"", "\\\""))
+        print("Wrote %s" % output)
+    else:
+        print("No strings found")
 
-    print("Wrote %s" % args.output)
-else:
-    print("No strings found")
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='Extract plugin manifest strings.')
+    parser.add_argument('input', type=str,
+                        help='Path to plugins directory')
+    parser.add_argument('output', type=str,
+                        help='Where to write resulting translation file')
+    args = parser.parse_args()
+    extract_plugin_manifest_strings(args.input, args.output)