Big update (#1021)

* fix typo

* fix connection error because of api version

* RDM002

* rdm002

* update mqtt sensor

* fix sensor type

* add remove apiuser

* deprecated utcnow

The method "utcnow" in class "datetime" is deprecated
  Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .now(datetime.UTC)

* Update __init__.py

* Update configHandler.py

* Update views.py

* add config setting

* Update configHandler.py

* Update daylightSensor.py

* Update stateFetch.py

* Update stateFetch.py

* Update stateFetch.py

* Update stateFetch.py

* Update daylightSensor.py

* Update discover.py

* Update scheduler.py

* replace localhost with IP

* update views.py, githubinstall.sh, updateManager.py

* download debug

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update logger.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update configHandler.py

* Update bug-report.yml

* Update bug-report.yml

* log

* Update bug-report.yml

* Update logger.py

* remove sleep, add WebUI version, fix logfile loglevel

* update installer, logger, bug-report

* delete old webui

* more remove old ui

* Update bug-report.yml

* Update Dockerfile

* move diyhue.log

.build/Dockerfile

@@ -34,7 +34,6 @@ COPY ./BridgeEmulator/configManager/ /opt/hue-emulator/configManager/
 COPY ./BridgeEmulator/logManager/ /opt/hue-emulator/logManager/
 COPY ./BridgeEmulator/HueEmulator3.py /opt/hue-emulator/
 COPY ./BridgeEmulator/githubInstall.sh /opt/hue-emulator/
-COPY ./BridgeEmulator/githubUIInstall.sh /opt/hue-emulator/
 
 ## Install the web interface
 RUN curl -sL https://github.com/diyhue/diyHueUI/releases/latest/download/DiyHueUI-release.zip -o diyHueUI.zip \

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -51,7 +51,10 @@ body:
       description: "Please paste the full error log here"
       value: |
         ```
-        Please paste the full error log here
+        This can be downloaded from the webUI.
+        DiyHue > Bridge > Brdige Control > Download debug > Log tar
+        This is also included in Full debug
+        If diyHue does not start upload from normal log
         ```
     validations:
       required: true
@@ -66,7 +69,28 @@ body:
     id: os
     attributes:
       label: "Please enter your operating system details here"
-      description: "You can provide the output from: uname -mrs"
+      description: "You can provide the output from the web ui DiyHue>Bridge>System debug information and change Hardware to match you hardware"
+      value: |
+        If diyHue does not star use "uname -mrs"
+        Hue-Emulator Version:
+        Architecture:
+        OS:
+        Os version:
+        Os release:
+        Hardware:
+    validations:
+      required: true
+  - type: textarea
+    id: debug
+    attributes:
+      label: "Upload debug tar"
+      description: "Please upload config_debug.tar"
+      value: |
+        ```
+        This can be downloaded from the webUI.
+        DiyHue > Bridge > Brdige Control > Download debug > Full debug
+        Do not upload from Force config dump, Force config dump contains sensitive information
+        ``
     validations:
       required: true
   - type: markdown

BridgeEmulator/configManager/configHandler.py

@@ -2,12 +2,14 @@
 from configManager.argumentHandler import parse_arguments
 from datetime import datetime
 import os
+import subprocess
 import json
 import logManager
 import yaml
 import uuid
 import weakref
 from HueObjects import Light, Group, EntertainmentConfiguration, Scene, ApiUser, Rule, ResourceLink, Schedule, Sensor, BehaviorInstance
+from time import sleep
 try:
     from time import tzset
 except ImportError:
@@ -235,7 +237,6 @@ def load_config(self):
             raise SystemExit("CRITICAL! Config file was not loaded")
         bridgeConfig = self.yaml_config
 
-
     def save_config(self, backup=False, resource="all"):
         path = self.configDir + '/'
         if backup:
@@ -267,16 +268,57 @@ def save_config(self, backup=False, resource="all"):
             _write_yaml(filePath, dumpDict)
             logging.debug("Dump config file " + filePath)
 
-
     def reset_config(self):
-        backup = self.save_config(True)
+        backup = self.save_config(backup=True)
         try:
-            os.remove(self.configDir + "/*.yaml")
+            os.popen('rm -r ' + self.configDir + '/*.yaml')
         except:
             logging.exception("Something went wrong when deleting the config")
         self.load_config()
         return backup
 
+    def restore_backup(self):
+        try:
+            os.popen('rm -r ' + self.configDir + '/*.yaml')
+        except:
+            logging.exception("Something went wrong when deleting the config")
+        subprocess.run('cp -r ' + self.configDir + '/backup/*.yaml ' + self.configDir + '/', shell=True, capture_output=True, text=True)
+        load = self.load_config()
+        return load
+
+    def download_config(self):
+        self.save_config()
+        subprocess.run('tar --exclude=' + "'config_debug.yaml'" + ' -cvf ' + self.configDir + '/config.tar ' + self.configDir + '/*.yaml', shell=True, capture_output=True, text=True)
+        return self.configDir + "/config.tar"
+
+    def download_log(self):
+        subprocess.run('tar -cvf ' + self.configDir + '/diyhue_log.tar ' +
+                 self.configDir.replace('/config', '') + '/*.log* ',
+                 shell=True, capture_output=True, text=True)
+        return self.configDir + "/diyhue_log.tar"
+
+    def download_debug(self):
+        _write_yaml(self.configDir + "/config_debug.yaml", self.yaml_config["config"])
+        debug = _open_yaml(self.configDir + "/config_debug.yaml")
+        debug["whitelist"] = "privately"
+        debug["Hue Essentials key"] = "privately"
+        debug["users"] = "privately"
+        info = {}
+        info["OS"] = os.uname().sysname
+        info["Architecture"] = os.uname().machine
+        info["os_version"] = os.uname().version
+        info["os_release"] = os.uname().release
+        info["Hue-Emulator Version"] = subprocess.run("stat -c %y HueEmulator3.py", shell=True, capture_output=True, text=True).stdout.replace("\n", "")
+        info["WebUI Version"] = subprocess.run("stat -c %y flaskUI/templates/index.html", shell=True, capture_output=True, text=True).stdout.replace("\n", "")
+        _write_yaml(self.configDir + "/config_debug.yaml", debug)
+        _write_yaml(self.configDir + "/system_info.yaml", info)
+        subprocess.run('tar --exclude=' + "'config.yaml'" + ' -cvf ' + self.configDir + '/config_debug.tar ' +
+                 self.configDir + '/*.yaml ' +
+                 self.configDir.replace('/config', '') + '/*.log* ',
+                 shell=True, capture_output=True, text=True)
+        os.popen('rm -r ' + self.configDir + '/config_debug.yaml')
+        return self.configDir + "/config_debug.tar"
+
     def write_args(self, args):
         self.yaml_config = configInit.write_args(args, self.yaml_config)
 

BridgeEmulator/flaskUI/README.md

@@ -0,0 +1,11 @@
+# DIYHUE UI Repository
+The Web UI used by [DiyHue Emulator](https://github.com/diyhue/diyHue).
+
+
+## How to get started with UI development
+ - Download latest LTS [Node JS](https://nodejs.org/en/download/) version
+ - `git clone https://github.com/diyhue/diyHueUI.git`
+ - `cd diyhueui`
+ - `npm install` to install all dependencies
+ - Replace the IP address in this line in `app.js` with your DiyHue Emulator IP `const HOST_IP = "http://192.168.1.10";`
+ - `npm start` to start a development instance

BridgeEmulator/flaskUI/core/views.py

@@ -1,4 +1,4 @@
-from flask import render_template, request, Blueprint, redirect, url_for, make_response
+from flask import render_template, request, Blueprint, redirect, url_for, make_response, send_file
 from werkzeug.security import generate_password_hash,check_password_hash
 from flaskUI.core.forms import LoginForm
 import flask_login
@@ -13,6 +13,7 @@
 import os
 import sys
 import logManager
+import subprocess
 logging = logManager.logger.get_logger(__name__)
 bridgeConfig = configManager.bridgeConfig.yaml_config
 core = Blueprint('core',__name__)
@@ -39,7 +40,6 @@ def get_lights():
         result[light] = object.save()
     return result
 
-
 @core.route('/sensors')
 #@flask_login.login_required
 def get_sensors():
@@ -48,7 +48,6 @@ def get_sensors():
         result[sensor] = object.save()
     return result
 
-
 @core.route('/light-types', methods=['GET', 'POST'])
 #@flask_login.login_required
 def get_light_types():
@@ -82,23 +81,69 @@ def pairTradfri():
 
 @core.route('/save')
 def save_config():
-    configManager.bridgeConfig.save_config()
-    return "config saved"
+    if request.args.get('backup', type = str) == "True":
+        configManager.bridgeConfig.save_config(backup=True)
+        return "backup config\n"
+    else:
+        configManager.bridgeConfig.save_config()
+        return "config saved\n"
+
+@core.route('/reset_config')
+@flask_login.login_required
+def reset_config():
+    configManager.bridgeConfig.reset_config()
+    return "config reset\n"
+
+@core.route('/restore_config')
+@flask_login.login_required
+def restore_config():
+    configManager.bridgeConfig.restore_backup()
+    return "restore config\n"
+
+@core.route('/download_config')
+@flask_login.login_required
+def download_config():
+    path = configManager.bridgeConfig.download_config()
+    return send_file(path, as_attachment=True)
+
+@core.route('/download_log')
+#@flask_login.login_required
+def download_log():
+    path = configManager.bridgeConfig.download_log()
+    return send_file(path, as_attachment=True)
+
+@core.route('/download_debug')
+#@flask_login.login_required
+def download_debug():
+    path = configManager.bridgeConfig.download_debug()
+    return send_file(path, as_attachment=True)
 
 @core.route('/restart')
 def restart():
     logging.info("restart " + str(sys.executable) + " with args : " + str(sys.argv))
     os.execl(sys.executable, sys.executable, *sys.argv)
     return "restart python with args"
 
+@core.route('/info')
+#@flask_login.login_required
+def info():
+    response = {}
+    response["sysname"] = os.uname().sysname
+    response["machine"] = os.uname().machine
+    response["os_version"] = os.uname().version
+    response["os_release"] = os.uname().release
+    response["diyhue"] = subprocess.run("stat -c %y HueEmulator3.py", shell=True, capture_output=True, text=True).stdout.replace("\n", "")
+    response["webui"] = subprocess.run("stat -c %y flaskUI/templates/index.html", shell=True, capture_output=True, text=True).stdout.replace("\n", "")
+    return response
+
 @core.route('/login', methods=['GET', 'POST'])
 def login():
     form = LoginForm()
     if request.method == 'GET':
         return render_template('login.html', form=form)
     email = form.email.data
     if email not in bridgeConfig["config"]["users"]:
-        return 'User don\'t exist'
+        return 'User don\'t exist\n'
     if check_password_hash(bridgeConfig["config"]["users"][email]['password'],form.password.data):
         user = User()
         user.id = email
@@ -107,8 +152,7 @@ def login():
 
     logging.info("Hashed pass: " + generate_password_hash(form.password.data))
 
-    return 'Bad login'
-
+    return 'Bad login\n'
 
 @core.route('/description.xml')
 def description_xml():
@@ -118,8 +162,6 @@ def description_xml():
     resp.headers['Content-type'] = 'text/xml'
     return resp
 
-
-
 @core.route('/logout')
 @flask_login.login_required
 def logout():

BridgeEmulator/flaskUI/espDevices.py

@@ -6,7 +6,7 @@
 from flask import request
 from functions.rules import rulesProcessor
 from sensors.discover import addHueMotionSensor, addHueSwitch, addHueRotarySwitch
-from datetime import datetime
+from datetime import datetime, timezone
 from threading import Thread
 from time import sleep
 
@@ -106,7 +106,7 @@ def get(self):
                             else:
                                 result = {"fail": "unknown device"}
                             obj.dxState["lastupdated"] = current_time
-                            obj.state["lastupdated"] = datetime.utcnow().strftime(
+                            obj.state["lastupdated"] = datetime.now(timezone.utc).strftime(
                                 "%Y-%m-%dT%H:%M:%S")
                             rulesProcessor(obj, current_time)
                             result = {"success": "command applied"}

BridgeEmulator/flaskUI/restful.py

@@ -9,7 +9,7 @@
 from subprocess import Popen
 from threading import Thread
 from time import sleep
-from datetime import datetime
+from datetime import datetime, timezone
 from lights.discover import scanForLights, manualAddLight
 from functions.core import capabilities, staticConfig, nextFreeId
 from flask_restful import Resource
@@ -50,7 +50,7 @@ def authorize(username, resource='', resourceId='', resourceParam=''):
         logging.debug(str(resourceId) + " has no attribute " + str(resourceParam))
         return [{"error": {"type": 3, "address": "/" + resource + "/" + resourceId + "/" + resourceParam, "description": "resource, " + resource + "/" + resourceId + "/" + resourceParam + ", not available"}}]
     if request.remote_addr != "127.0.0.1":
-        bridgeConfig["apiUsers"][username].last_use_date = datetime.utcnow().strftime(
+        bridgeConfig["apiUsers"][username].last_use_date = datetime.now(timezone.utc).strftime(
             "%Y-%m-%dT%H:%M:%S")
     return ["success"]
 
@@ -60,7 +60,7 @@ def buildConfig():
     config = bridgeConfig["config"]
     result.update({"Hue Essentials key": config["Hue Essentials key"], "Remote API enabled": config["Remote API enabled"], "apiversion": config["apiversion"], "bridgeid": config["bridgeid"],
                    "ipaddress": config["ipaddress"], "netmask": config["netmask"], "gateway": config["gateway"], "mac": config["mac"], "name": config["name"], "swversion": config["swversion"], "swupdate2": config["swupdate2"], "timezone": config["timezone"]})
-    result["UTC"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
+    result["UTC"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
     result["localtime"] = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
     result["whitelist"] = {}
     for key, user in bridgeConfig["apiUsers"].items():
@@ -232,7 +232,7 @@ def post(self, username, resource):
             bridgeConfig[resource][new_object_id] = HueObjects.Schedule(postDict)
         newObject = bridgeConfig[resource][new_object_id]
         if v2Resource != "none":
-            streamMessage = {"creationtime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
+            streamMessage = {"creationtime": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
                              "data": [],
                              "id_v1": "/" + resource + "/" + new_object_id,
                              "id": str(uuid.uuid4()),
@@ -343,7 +343,7 @@ def put(self, username, resource, resourceid):
             if "state" in putDict:
                 for state in putDict["state"].keys():
                     bridgeConfig["sensors"][resourceid].dxState[state] = currentTime
-                bridgeConfig["sensors"][resourceid].state["lastupdated"] = datetime.utcnow(
+                bridgeConfig["sensors"][resourceid].state["lastupdated"] = datetime.now(timezone.utc
                 ).strftime("%Y-%m-%dT%H:%M:%S")
                 bridgeConfig["sensors"][resourceid].dxState["lastupdated"] = currentTime
         elif resource == "groups":
@@ -442,7 +442,7 @@ def put(self, username, resource, resourceid, param):
             bridgeConfig[resource][resourceid].state.update(putDict)
             for state in putDict.keys():
                 bridgeConfig["sensors"][resourceid].dxState[state] = currentTime
-            bridgeConfig["sensors"][resourceid].state["lastupdated"] = datetime.utcnow(
+            bridgeConfig["sensors"][resourceid].state["lastupdated"] = datetime.now(timezone.utc
             ).strftime("%Y-%m-%dT%H:%M:%S")
             bridgeConfig["sensors"][resourceid].dxState["lastupdated"] = currentTime
             rulesProcessor(bridgeConfig[resource][resourceid], currentTime)