optimal performance (#11)

* optimal performance

* change gzip to stream

* add coverage duration limit

* change interval variable name

* delete useless reference

* Add response headers

lyrebird_api_coverage/app_ui.py

@@ -2,7 +2,7 @@
 
 import lyrebird
 import os
-from flask import request, jsonify, Response
+from flask import request, jsonify, Response, stream_with_context
 from lyrebird_api_coverage.client.context import app_context
 
 from lyrebird_api_coverage.client import context
@@ -13,6 +13,7 @@
 from lyrebird_api_coverage.handlers.import_file_handler import ImportHandler
 from lyrebird_api_coverage.handlers.result_handler import ResultHandler, PLUGINS_DUMP_DIR
 from lyrebird import context
+import time
 
 
 class AppUI(lyrebird.PluginView):
@@ -22,61 +23,15 @@ def index(self):
         """
         return self.render_template('index.html')
 
-    def get_coverage_data(self):
-        # 获取app_context里面缓存的测试数据
-        # 如果内存为空，则视为首次进入该页面
-        data = app_context.merge_list
-        if not data:
-            # 获取base_data_config文件信息
-            base_dict = BaseDataHandler().get_base_source()
-            # 如果import的文件异常
-            if isinstance(base_dict, Response):
-                resp = base_dict
-            else:
-                mergeAlgorithm.first_result_handler(base_dict)
-                mergeAlgorithm.coverage_arithmetic(base_dict)
-                resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage})
-        # 若不为空，则视为有测试缓存
-        else:
-            resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage})
-        return resp
+    def generate(self, data):
+        rtext = json.dumps(data)
+        yield rtext
 
-    # 获取init base数据 以及 测试缓存数据 API
     def get_test_data(self):
-        # 获取app_context里面缓存的测试数据
-        # 如果内存为空，则视为首次进入该页面
-        data = app_context.merge_list
-        if not data:
-            # 获取base_data_config文件信息
-            base_dict = BaseDataHandler().get_base_source()
-            # 如果import的文件异常
-            if isinstance(base_dict, Response):
-                resp = base_dict
-            else:
-                mergeAlgorithm.first_result_handler(base_dict)
-                resp = jsonify({'test_data': app_context.merge_list})
-        # 若不为空，则视为有测试缓存
-        else:
-            resp = jsonify({'test_data': app_context.merge_list})
-        return resp
+        return Response(stream_with_context(self.generate({'test_data': app_context.merge_list})), content_type='application/json')
 
     def get_coverage(self):
-        # 获取app_context里面缓存的测试数据
-        # 如果内存为空，则视为首次进入该页面
-        data = app_context.coverage
-        if not data:
-            # 获取base_data_config文件信息
-            base_dict = BaseDataHandler().get_base_source()
-            # 如果import的文件异常
-            if isinstance(base_dict, Response):
-                resp = base_dict
-            else:
-                mergeAlgorithm.coverage_arithmetic(base_dict)
-                resp = jsonify(app_context.coverage)
-        # 若不为空，则视为有测试缓存
-        else:
-            resp = jsonify(app_context.coverage)
-        return resp
+        return Response(stream_with_context(self.generate(app_context.coverage)), content_type='application/json')
 
     def save_result(self):
         # 传入文件名
@@ -103,7 +58,7 @@ def import_base(self):
     def get_filter_conf(self):
         msg = FilterHandler().get_filer_conf()
         # 如果返回的string包含报错信息，则是报错
-        if "筛除配置文件有误" in msg:
+        if isinstance(msg, str):
             return context.make_fail_response(msg)
         else:
             return jsonify(msg)
@@ -129,6 +84,14 @@ def get_base_info(self):
                         'version_code': app_context.version_code})
 
     def on_create(self):
+
+        # 获取base_data_config文件信息
+        base_dict = BaseDataHandler().get_base_source()
+        # 如果import的文件异常
+        if not isinstance(base_dict, Response):
+            mergeAlgorithm.first_result_handler(base_dict)
+            mergeAlgorithm.coverage_arithmetic(base_dict)
+
         # 设置模板目录（可选，设置模板文件目录。默认值templates）
         self.set_template_root('lyrebird_api_coverage')
         # 设置静态文件目录（可选，设置静态文件目录。默认值static）
@@ -143,9 +106,6 @@ def on_create(self):
         self.add_url_rule('/getTest', view_func=self.get_test_data)
         # 获取内存里保存的测试覆盖率信息
         self.add_url_rule('/getCoverage', view_func=self.get_coverage)
-
-        self.add_url_rule('/getCoverageData', view_func=self.get_coverage_data)
-
         # 保存测试数据在本地
         self.add_url_rule('/saveResult', view_func=self.save_result, methods=['POST'])
         # 续传测试结果

lyrebird_api_coverage/client/context.py

@@ -31,6 +31,12 @@ def __init__(self):
         self.version_code = None
         # user_info
         self.user_info = {}
+        # 记录请求最后的时间，避免频繁emit io消息
+        self.endtime = 0
+        # 记录上次coverage变化的时间，避免频繁emit io消息
+        self.covtime = 0
+        # 时间间隔，每隔指定时间触发1次socket io消息，防止刷新频繁
+        self.SOCKET_PUSH_INTERVAL = 1
 
 
 # 单例模式

lyrebird_api_coverage/client/merge_algorithm.py

@@ -1,8 +1,7 @@
 import lyrebird
-
 from lyrebird_api_coverage.client import format_url
-
 from lyrebird_api_coverage.client.context import app_context
+import time
 
 
 class MergeAlgorithm:
@@ -103,10 +102,13 @@ def coverage_handler(self):
             coverage = round(test_len / app_context.coverage['len'] * 100, 2)
         # 为了传给Overbridge的socket信息format数据格式
         app_context.coverage['total'] = coverage
+        # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线
         if not history_coverage == coverage:
-            # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线
-            lyrebird.emit('update', app_context.coverage, namespace='/charts')
-            lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage')
+            handler_time = time.time()
+            # 限制频繁emit io msg，在两次之间大于指定时间间隔才会emit
+            if handler_time - app_context.covtime > app_context.SOCKET_PUSH_INTERVAL:
+                lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage')
+                app_context.covtime = handler_time
             by_priority = [p.get('value') for p in app_context.coverage['priorities']]
             lyrebird.publish('coverage', 
                 dict(

lyrebird_api_coverage/client/report.py

@@ -8,12 +8,13 @@
 
 class ReportHandler:
     def check_url_info(self, url, device_ip):
-        specific_data = list(filter(lambda x: x.get('url') == url, app_context.merge_list))[0]
-        desc = specific_data.get('desc')
-        if specific_data.get('status') == 1:
+        specific_list = list(filter(lambda x: x.get('url') == url, app_context.merge_list))
+        if specific_list and specific_list[0].get('status') == 1:
+            desc = specific_list[0].get('desc')
             count_flag = 1
-            priority = specific_data.get('priority')
+            priority = specific_list[0].get('priority')
         else:
+            desc = 'N/A'
             count_flag = -1
             priority = -1
         info_dict = {'url': url, 'desc': desc, 'priority': priority, 'count_flag': count_flag,

lyrebird_api_coverage/proxy/proxy_handler.py

@@ -10,6 +10,7 @@
 from lyrebird_api_coverage.client.report import report_worker
 from lyrebird_api_coverage.client.url_compare import compare_query
 from lyrebird_api_coverage.handlers.base_source_handler import BaseDataHandler
+import time
 
 
 """
@@ -23,6 +24,7 @@ class MyDataHandler:
     # self.user_list = []
 
     def handle(self, handler_context: HandlerContext):
+        req_starttime = time.time()
         # UI自动化等需要mock的手段orgin url 需要调用这个方法
         org_url = handler_context.get_origin_url()
         if not org_url:
@@ -34,36 +36,24 @@ def handle(self, handler_context: HandlerContext):
         path_id = handler_context.id
         device_ip = handler_context.client_address
 
-        # 当请求过来的时候，base还没有init，需要进行init处理
-        if not app_context.base_list:
-            base_dict = BaseDataHandler().get_base_source()
-            if isinstance(base_dict, Response):
-                get_logger().error('API-Coverage base file is None.')
-            else:
-                mergeAlgorithm.first_result_handler(base_dict)
-
         if path in app_context.base_list:
             # merge到 context merge list中
             mergeAlgorithm.merge_handler_new(path, path_id)
             # 在base里的需要去计算下覆盖率
             mergeAlgorithm.coverage_handler()
-            # path传给覆盖详情表格
-            lyrebird.emit('test_data message', path, namespace='/api_coverage')
-            # 如果设备信息抓取到,进行上报
-            # if device_ip in app_context.info:
+            # 进行上报
             report_worker(path, device_ip)
+            # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率
+            self.emit(req_starttime, path)
 
         # 如果path配置了对应的参数
-        if path in list(app_context.path_param_dic.keys()):
+        elif path in app_context.path_param_dic:
             ulr_list = app_context.path_param_dic[path]
             flag = 0
             for item in ulr_list:
                 if compare_query(item['url'], handler_context.request.url):
                     mergeAlgorithm.merge_handler_new(item['url_base'], path_id)
                     mergeAlgorithm.coverage_handler()
-                    lyrebird.emit('test_data message', item['url_base'], namespace='/api_coverage')
-                    # 如果设备信息抓取到,进行上报
-                    # if device_ip in app_context.info:
                     report_worker(item['url_base'], device_ip)
                     flag = 1
             # 如果参数组合不存在，提取关注的字段
@@ -82,21 +72,21 @@ def handle(self, handler_context: HandlerContext):
 
                 mergeAlgorithm.merge_handler_new(url_pgroup, path_id)
                 mergeAlgorithm.coverage_handler()
-                lyrebird.emit('test_data message', url_pgroup, namespace='/api_coverage')
-                # 如果设备信息抓取到,进行上报
-                # if device_ip in app_context.info:
                 report_worker(url_pgroup, device_ip)
+
+            # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率
+            self.emit(req_starttime, path)
+
 
-        # 如果不在base里，需要判断这些API是否被筛掉
+        # 如果不在base里，不需要merge到数据中
         else:
-            # 如果不在筛除列表内，才进行merge等一系列算法
-            if not Filter().filter_all(path):
-                # merge到 context merge list中
-                mergeAlgorithm.merge_handler_new(path, path_id)
-                # 传给api_coverage前端的socket信息
-                lyrebird.emit('test_data message', path, namespace='/api_coverage')
-                # 如果设备信息抓取到,进行上报
-                # if device_ip in app_context.info:
-                report_worker(path, device_ip)
-            else:
-                pass
+            # mergeAlgorithm.merge_handler_new(path, path_id)
+            # 进行上报
+            report_worker(path, device_ip)
+
+
+    def emit(self, starttime, path):
+        duration = starttime - app_context.endtime
+        if duration > app_context.SOCKET_PUSH_INTERVAL:
+            app_context.endtime = starttime
+            lyrebird.emit('test_data message', path, namespace='/api_coverage')

lyrebird_api_coverage/static/component/main.vue

@@ -16,7 +16,6 @@
 <script>
 //websocket namespace /api_coverage
 let apicoverageIO = io("/api_coverage");
-let timer = null;
 
 module.exports = {
   mounted: function() {
@@ -28,15 +27,7 @@ module.exports = {
       loadCoverageData();
     });
     apicoverageIO.on("test_data message", function(msg) {
-      console.log(msg);
       loadDetailData();
-      if (!timer) {
-        timer = setTimeout(() => {
-          loadDetailData();
-          clearTimeout(timer);
-          timer = null;
-        }, 1000);
-      }
     });
   },
   data: function() {
@@ -47,9 +38,6 @@ module.exports = {
       targetContext: null
     };
   },
-  updated: function() {
-    console.log("init");
-  },
   methods: {
     loadCoverageData: function() {
       this.$http.get("/ui/plugin/api_coverage/getCoverage").then(

lyrebird_api_coverage/templates/index.html

@@ -2,8 +2,4 @@
     <apicoverage></apicoverage>
 </div>
 
-
-<link href="https://cdn.bootcss.com/iview/2.14.0/styles/iview.css" rel="stylesheet">
-<script src="https://cdn.bootcss.com/iview/2.14.0/iview.min.js"></script>
-
 <script src="/ui/plugin/api_coverage/static/js/main.js"></script>

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name='lyrebird-api-coverage',
-    version='0.2.6',
+    version='0.2.7',
     packages=['lyrebird_api_coverage'],
     url='https://github.com/meituan/lyrebird-api-coverage',
     author='HBQA',