From ee90fd48a4dd20d460f15c460c6c5862a958a89c Mon Sep 17 00:00:00 2001 From: Qi Gao Date: Tue, 11 Dec 2018 17:41:41 +0800 Subject: [PATCH] optimal performance (#11) * optimal performance * change gzip to stream * add coverage duration limit * change interval variable name * delete useless reference * Add response headers --- lyrebird_api_coverage/app_ui.py | 72 +++++-------------- lyrebird_api_coverage/client/context.py | 6 ++ .../client/merge_algorithm.py | 12 ++-- lyrebird_api_coverage/client/report.py | 9 +-- lyrebird_api_coverage/proxy/proxy_handler.py | 52 ++++++-------- .../static/component/main.vue | 12 ---- lyrebird_api_coverage/templates/index.html | 4 -- setup.py | 2 +- 8 files changed, 56 insertions(+), 113 deletions(-) diff --git a/lyrebird_api_coverage/app_ui.py b/lyrebird_api_coverage/app_ui.py index 96bbc6d2..efbf42f5 100644 --- a/lyrebird_api_coverage/app_ui.py +++ b/lyrebird_api_coverage/app_ui.py @@ -2,7 +2,7 @@ import lyrebird import os -from flask import request, jsonify, Response +from flask import request, jsonify, Response, stream_with_context from lyrebird_api_coverage.client.context import app_context from lyrebird_api_coverage.client import context @@ -13,6 +13,7 @@ from lyrebird_api_coverage.handlers.import_file_handler import ImportHandler from lyrebird_api_coverage.handlers.result_handler import ResultHandler, PLUGINS_DUMP_DIR from lyrebird import context +import time class AppUI(lyrebird.PluginView): @@ -22,61 +23,15 @@ def index(self): """ return self.render_template('index.html') - def get_coverage_data(self): - # 获取app_context里面缓存的测试数据 - # 如果内存为空，则视为首次进入该页面 - data = app_context.merge_list - if not data: - # 获取base_data_config文件信息 - base_dict = BaseDataHandler().get_base_source() - # 如果import的文件异常 - if isinstance(base_dict, Response): - resp = base_dict - else: - mergeAlgorithm.first_result_handler(base_dict) - mergeAlgorithm.coverage_arithmetic(base_dict) - resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage}) - # 若不为空，则视为有测试缓存 - else: - resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage}) - return resp + def generate(self, data): + rtext = json.dumps(data) + yield rtext - # 获取init base数据 以及 测试缓存数据 API def get_test_data(self): - # 获取app_context里面缓存的测试数据 - # 如果内存为空，则视为首次进入该页面 - data = app_context.merge_list - if not data: - # 获取base_data_config文件信息 - base_dict = BaseDataHandler().get_base_source() - # 如果import的文件异常 - if isinstance(base_dict, Response): - resp = base_dict - else: - mergeAlgorithm.first_result_handler(base_dict) - resp = jsonify({'test_data': app_context.merge_list}) - # 若不为空，则视为有测试缓存 - else: - resp = jsonify({'test_data': app_context.merge_list}) - return resp + return Response(stream_with_context(self.generate({'test_data': app_context.merge_list})), content_type='application/json') def get_coverage(self): - # 获取app_context里面缓存的测试数据 - # 如果内存为空，则视为首次进入该页面 - data = app_context.coverage - if not data: - # 获取base_data_config文件信息 - base_dict = BaseDataHandler().get_base_source() - # 如果import的文件异常 - if isinstance(base_dict, Response): - resp = base_dict - else: - mergeAlgorithm.coverage_arithmetic(base_dict) - resp = jsonify(app_context.coverage) - # 若不为空，则视为有测试缓存 - else: - resp = jsonify(app_context.coverage) - return resp + return Response(stream_with_context(self.generate(app_context.coverage)), content_type='application/json') def save_result(self): # 传入文件名 @@ -103,7 +58,7 @@ def import_base(self): def get_filter_conf(self): msg = FilterHandler().get_filer_conf() # 如果返回的string包含报错信息，则是报错 - if "筛除配置文件有误" in msg: + if isinstance(msg, str): return context.make_fail_response(msg) else: return jsonify(msg) @@ -129,6 +84,14 @@ def get_base_info(self): 'version_code': app_context.version_code}) def on_create(self): + + # 获取base_data_config文件信息 + base_dict = BaseDataHandler().get_base_source() + # 如果import的文件异常 + if not isinstance(base_dict, Response): + mergeAlgorithm.first_result_handler(base_dict) + mergeAlgorithm.coverage_arithmetic(base_dict) + # 设置模板目录（可选，设置模板文件目录。默认值templates） self.set_template_root('lyrebird_api_coverage') # 设置静态文件目录（可选，设置静态文件目录。默认值static） @@ -143,9 +106,6 @@ def on_create(self): self.add_url_rule('/getTest', view_func=self.get_test_data) # 获取内存里保存的测试覆盖率信息 self.add_url_rule('/getCoverage', view_func=self.get_coverage) - - self.add_url_rule('/getCoverageData', view_func=self.get_coverage_data) - # 保存测试数据在本地 self.add_url_rule('/saveResult', view_func=self.save_result, methods=['POST']) # 续传测试结果 diff --git a/lyrebird_api_coverage/client/context.py b/lyrebird_api_coverage/client/context.py index 67e677c7..fca1b489 100644 --- a/lyrebird_api_coverage/client/context.py +++ b/lyrebird_api_coverage/client/context.py @@ -31,6 +31,12 @@ def __init__(self): self.version_code = None # user_info self.user_info = {} + # 记录请求最后的时间，避免频繁emit io消息 + self.endtime = 0 + # 记录上次coverage变化的时间，避免频繁emit io消息 + self.covtime = 0 + # 时间间隔，每隔指定时间触发1次socket io消息，防止刷新频繁 + self.SOCKET_PUSH_INTERVAL = 1 # 单例模式 diff --git a/lyrebird_api_coverage/client/merge_algorithm.py b/lyrebird_api_coverage/client/merge_algorithm.py index c63a3640..09a3b128 100644 --- a/lyrebird_api_coverage/client/merge_algorithm.py +++ b/lyrebird_api_coverage/client/merge_algorithm.py @@ -1,8 +1,7 @@ import lyrebird - from lyrebird_api_coverage.client import format_url - from lyrebird_api_coverage.client.context import app_context +import time class MergeAlgorithm: @@ -103,10 +102,13 @@ def coverage_handler(self): coverage = round(test_len / app_context.coverage['len'] * 100, 2) # 为了传给Overbridge的socket信息format数据格式 app_context.coverage['total'] = coverage + # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线 if not history_coverage == coverage: - # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线 - lyrebird.emit('update', app_context.coverage, namespace='/charts') - lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage') + handler_time = time.time() + # 限制频繁emit io msg，在两次之间大于指定时间间隔才会emit + if handler_time - app_context.covtime > app_context.SOCKET_PUSH_INTERVAL: + lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage') + app_context.covtime = handler_time by_priority = [p.get('value') for p in app_context.coverage['priorities']] lyrebird.publish('coverage', dict( diff --git a/lyrebird_api_coverage/client/report.py b/lyrebird_api_coverage/client/report.py index e2d5bb9a..7a738588 100644 --- a/lyrebird_api_coverage/client/report.py +++ b/lyrebird_api_coverage/client/report.py @@ -8,12 +8,13 @@ class ReportHandler: def check_url_info(self, url, device_ip): - specific_data = list(filter(lambda x: x.get('url') == url, app_context.merge_list))[0] - desc = specific_data.get('desc') - if specific_data.get('status') == 1: + specific_list = list(filter(lambda x: x.get('url') == url, app_context.merge_list)) + if specific_list and specific_list[0].get('status') == 1: + desc = specific_list[0].get('desc') count_flag = 1 - priority = specific_data.get('priority') + priority = specific_list[0].get('priority') else: + desc = 'N/A' count_flag = -1 priority = -1 info_dict = {'url': url, 'desc': desc, 'priority': priority, 'count_flag': count_flag, diff --git a/lyrebird_api_coverage/proxy/proxy_handler.py b/lyrebird_api_coverage/proxy/proxy_handler.py index 5a802af3..7555a35d 100644 --- a/lyrebird_api_coverage/proxy/proxy_handler.py +++ b/lyrebird_api_coverage/proxy/proxy_handler.py @@ -10,6 +10,7 @@ from lyrebird_api_coverage.client.report import report_worker from lyrebird_api_coverage.client.url_compare import compare_query from lyrebird_api_coverage.handlers.base_source_handler import BaseDataHandler +import time """ @@ -23,6 +24,7 @@ class MyDataHandler: # self.user_list = [] def handle(self, handler_context: HandlerContext): + req_starttime = time.time() # UI自动化等需要mock的手段orgin url 需要调用这个方法 org_url = handler_context.get_origin_url() if not org_url: @@ -34,36 +36,24 @@ def handle(self, handler_context: HandlerContext): path_id = handler_context.id device_ip = handler_context.client_address - # 当请求过来的时候，base还没有init，需要进行init处理 - if not app_context.base_list: - base_dict = BaseDataHandler().get_base_source() - if isinstance(base_dict, Response): - get_logger().error('API-Coverage base file is None.') - else: - mergeAlgorithm.first_result_handler(base_dict) - if path in app_context.base_list: # merge到 context merge list中 mergeAlgorithm.merge_handler_new(path, path_id) # 在base里的需要去计算下覆盖率 mergeAlgorithm.coverage_handler() - # path传给覆盖详情表格 - lyrebird.emit('test_data message', path, namespace='/api_coverage') - # 如果设备信息抓取到,进行上报 - # if device_ip in app_context.info: + # 进行上报 report_worker(path, device_ip) + # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率 + self.emit(req_starttime, path) # 如果path配置了对应的参数 - if path in list(app_context.path_param_dic.keys()): + elif path in app_context.path_param_dic: ulr_list = app_context.path_param_dic[path] flag = 0 for item in ulr_list: if compare_query(item['url'], handler_context.request.url): mergeAlgorithm.merge_handler_new(item['url_base'], path_id) mergeAlgorithm.coverage_handler() - lyrebird.emit('test_data message', item['url_base'], namespace='/api_coverage') - # 如果设备信息抓取到,进行上报 - # if device_ip in app_context.info: report_worker(item['url_base'], device_ip) flag = 1 # 如果参数组合不存在，提取关注的字段 @@ -82,21 +72,21 @@ def handle(self, handler_context: HandlerContext): mergeAlgorithm.merge_handler_new(url_pgroup, path_id) mergeAlgorithm.coverage_handler() - lyrebird.emit('test_data message', url_pgroup, namespace='/api_coverage') - # 如果设备信息抓取到,进行上报 - # if device_ip in app_context.info: report_worker(url_pgroup, device_ip) + + # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率 + self.emit(req_starttime, path) + - # 如果不在base里，需要判断这些API是否被筛掉 + # 如果不在base里，不需要merge到数据中 else: - # 如果不在筛除列表内，才进行merge等一系列算法 - if not Filter().filter_all(path): - # merge到 context merge list中 - mergeAlgorithm.merge_handler_new(path, path_id) - # 传给api_coverage前端的socket信息 - lyrebird.emit('test_data message', path, namespace='/api_coverage') - # 如果设备信息抓取到,进行上报 - # if device_ip in app_context.info: - report_worker(path, device_ip) - else: - pass + # mergeAlgorithm.merge_handler_new(path, path_id) + # 进行上报 + report_worker(path, device_ip) + + + def emit(self, starttime, path): + duration = starttime - app_context.endtime + if duration > app_context.SOCKET_PUSH_INTERVAL: + app_context.endtime = starttime + lyrebird.emit('test_data message', path, namespace='/api_coverage') diff --git a/lyrebird_api_coverage/static/component/main.vue b/lyrebird_api_coverage/static/component/main.vue index e83f87b4..2a9e3aae 100644 --- a/lyrebird_api_coverage/static/component/main.vue +++ b/lyrebird_api_coverage/static/component/main.vue @@ -16,7 +16,6 @@ - \ No newline at end of file diff --git a/setup.py b/setup.py index 307580d4..6ee6f1e2 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='lyrebird-api-coverage', - version='0.2.6', + version='0.2.7', packages=['lyrebird_api_coverage'], url='https://github.com/meituan/lyrebird-api-coverage', author='HBQA',