diff --git a/initcommit.txt b/initcommit.txt deleted file mode 100644 index 421376d..0000000 --- a/initcommit.txt +++ /dev/null @@ -1 +0,0 @@ -dummy diff --git a/requirements.in b/requirements.in index 9b71c94..6fd8b5a 100644 --- a/requirements.in +++ b/requirements.in @@ -13,4 +13,4 @@ pytest requests SQLAlchemy wtforms -ensembl-py@git+https://github.com/Ensembl/ensembl-py.git@1.0.0dev2#egg=ensembl-py +ensembl-py@git+https://github.com/Ensembl/ensembl-py.git@1.2.2 diff --git a/requirements.txt b/requirements.txt index 6a380f7..71f4519 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements.in @@ -28,11 +28,11 @@ elasticsearch==6.8.2 # via -r requirements.in email-validator==2.0.0.post1 # via -r requirements.in -ensembl-hive @ git+https://github.com/Ensembl/ensembl-hive.git@main +ensembl-hive @ git+https://github.com/Ensembl/ensembl-hive.git # via ensembl-py ensembl-prodinf-core==2.0.5 # via -r requirements.in -ensembl-py @ git+https://github.com/Ensembl/ensembl-py.git@1.0.0dev2 +ensembl-py @ git+https://github.com/Ensembl/ensembl-py.git@1.2.2 # via -r requirements.in exceptiongroup==1.1.1 # via pytest @@ -57,6 +57,10 @@ idna==3.4 # via # email-validator # requests +importlib-metadata==7.0.1 + # via flask +importlib-resources==6.1.1 + # via jsonschema iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 @@ -84,6 +88,8 @@ packaging==23.1 # via # deprecation # pytest +pkgutil-resolve-name==1.3.10 + # via jsonschema pluggy==1.0.0 # via pytest pyrsistent==0.19.3 @@ -137,6 +143,10 @@ wtforms==3.0.1 # -r requirements.in # bootstrap-flask # flask-wtf +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/ensembl/production/datacheck/app/main.py b/src/ensembl/production/datacheck/app/main.py index ba2eb35..554abbf 100644 --- a/src/ensembl/production/datacheck/app/main.py +++ b/src/ensembl/production/datacheck/app/main.py @@ -69,7 +69,7 @@ es_ssl = app.config['ES_SSL'] app_es_data_source = app.config['APP_ES_DATA_SOURCE'] -if app.env == 'development': +if app.debug: # ENV dev (assumed run from builtin server, so update script_name at wsgi level) app.wsgi_app = DispatcherMiddleware( Response('Not Found', status=404), @@ -141,6 +141,17 @@ def servers_list(): return jsonify(get_servers_list()) +@app.route('/config', methods=['GET']) +def config(): + """ Disply all configs vars""" + config = {key: value if 'pass' not in key.lower() else 'XXXXXX' for key, value in DatacheckConfig.__dict__.items() if + key[:1] != '_'} + return render_template( + 'config.html', + config=config + ) + + @app.route('/servers/dict', methods=['GET']) def servers_dict(): return jsonify(get_servers_dict()) diff --git a/src/ensembl/production/datacheck/templates/config.html b/src/ensembl/production/datacheck/templates/config.html new file mode 100644 index 0000000..5ecd6f9 --- /dev/null +++ b/src/ensembl/production/datacheck/templates/config.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} +{% block styles %} + {{ super() }} +{% endblock %} +{% block content %} + {{ super() }} + +
+
+
+
+ Datacheck Configuration set +
+
+ + + {% for key, value in config.items() %} + + + + + {% endfor %} + +
{{ key }}{{ value }}
+
+
+
+
+{% endblock %} diff --git a/src/ensembl/production/datacheck/utils.py b/src/ensembl/production/datacheck/utils.py index e2b29b7..95cf6e4 100644 --- a/src/ensembl/production/datacheck/utils.py +++ b/src/ensembl/production/datacheck/utils.py @@ -70,7 +70,7 @@ def get_datacheck_results(jsonfile_path: str, ] }) if len(res['hits']['hits']) == 0: - raise ElasticsearchException(f"""No Hits Found for given params jsonfile_path {jsonfile_path} """) + raise ElasticsearchException(f"""No Hits Found for given params jsonfile_path {jsonfile_path}""") return {"status": True, "message": "", "result": res['hits']['hits'][0]['_source']['content']} diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 34f8d8f..8d34c70 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -22,88 +22,86 @@ from ensembl.production.core.es import ElasticsearchConnectionManager from ensembl.production.datacheck.app.main import app - -dc_success_result_es_doc = { - "job_id": "1", - "input_details": { - "datacheck_groups": [], - "server_uri": None, - "config_file": None, - "datacheck_names": [ - "SpeciesCommonName" - ], - "dbname": [ - "mus_musculus_core_107_39" - ], - "datacheck_types": [], - "registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm", - "timestamp": "Fri Jul 1 15:12:42 2022", - "db_type": "core", - "target_url": None, - "server_url": None, - "tag": "Vertebrate datachecks, pre-handover to web ()" - }, - "division": "EnsemblVertebrates", - "content": {}, - "file": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json" - } - - -dc_failed_result_es_doc = { - "job_id": "2", - "input_details": { - "datacheck_groups": [], - "server_uri": None, - "config_file": None, - "datacheck_names": [ - "SpeciesCommonName" - ], - "dbname": [ - "mus_musculus_core_107_39" - ], - "datacheck_types": [], - "registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm", - "timestamp": "Fri Jul 1 15:12:42 2022", - "db_type": "core", - "target_url": None, - "server_url": None, - "tag": "Vertebrate datachecks, pre-handover to web ()" - }, - "division": "EnsemblVertebrates", - "content": { - "camarhynchus_parvulus, core, camarhynchus_parvulus_core_108_11, EnsemblVertebrates" : { - "CompareMetaKeys" : { - "ok" : 0, - "tests" : { - "not ok 4 - Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11" : [ - " Failed test 'Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11'", - " at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.", - " Structures begin differing at:", - " $got->{ENSCPVG00005007000} = Does not exist", - " $expected->{ENSCPVG00005007000} = HASH(0x568a2c0)", - "Looks like you failed 1 test of 4." +dc_success_result_es_doc = { + "job_id": "1", + "input_details": { + "datacheck_groups": [], + "server_uri": None, + "config_file": None, + "datacheck_names": [ + "SpeciesCommonName" + ], + "dbname": [ + "mus_musculus_core_107_39" + ], + "datacheck_types": [], + "registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm", + "timestamp": "Fri Jul 1 15:12:42 2022", + "db_type": "core", + "target_url": None, + "server_url": None, + "tag": "Vertebrate datachecks, pre-handover to web ()" + }, + "division": "EnsemblVertebrates", + "content": {}, + "file": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json" +} + +dc_failed_result_es_doc = { + "job_id": "2", + "input_details": { + "datacheck_groups": [], + "server_uri": None, + "config_file": None, + "datacheck_names": [ + "SpeciesCommonName" + ], + "dbname": [ + "mus_musculus_core_107_39" + ], + "datacheck_types": [], + "registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm", + "timestamp": "Fri Jul 1 15:12:42 2022", + "db_type": "core", + "target_url": None, + "server_url": None, + "tag": "Vertebrate datachecks, pre-handover to web ()" + }, + "division": "EnsemblVertebrates", + "content": { + "camarhynchus_parvulus, core, camarhynchus_parvulus_core_108_11, EnsemblVertebrates": { + "CompareMetaKeys": { + "ok": 0, + "tests": { + "not ok 4 - Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11": [ + " Failed test 'Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11'", + " at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.", + " Structures begin differing at:", + " $got->{ENSCPVG00005007000} = Does not exist", + " $expected->{ENSCPVG00005007000} = HASH(0x568a2c0)", + "Looks like you failed 1 test of 4." ] } } }, - "corvus_moneduloides, core, corvus_moneduloides_core_108_1, EnsemblVertebrates" : { - "CompareMetaKeys" : { - "ok" : 0, - "tests" : { - "not ok 4 - Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1" : [ - " Failed test 'Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1'", - " at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.", - " Structures begin differing at:", - " $got->{ENSCMUG00005009949} = Does not exist", - " $expected->{ENSCMUG00005009949} = HASH(0x78dd4f0)", - "Looks like you failed 1 test of 4." + "corvus_moneduloides, core, corvus_moneduloides_core_108_1, EnsemblVertebrates": { + "CompareMetaKeys": { + "ok": 0, + "tests": { + "not ok 4 - Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1": [ + " Failed test 'Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1'", + " at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.", + " Structures begin differing at:", + " $got->{ENSCMUG00005009949} = Does not exist", + " $expected->{ENSCMUG00005009949} = HASH(0x78dd4f0)", + "Looks like you failed 1 test of 4." ] } } } - }, - "file": "/homes/user/test_es_output/user_sL3mnrNTRrr1/results_by_species.json" - } + }, + "file": "/homes/user/test_es_output/user_sL3mnrNTRrr1/results_by_species.json" +} def wait_for(url: str, retries: int = 2, backoff: float = 0.2): @@ -115,71 +113,58 @@ def wait_for(url: str, retries: int = 2, backoff: float = 0.2): manager = urllib3.PoolManager(retries=retry) manager.request("GET", url) -@pytest.fixture(scope="session") + +@pytest.fixture(scope="session", autouse=True) def elastic_search(): es_host = 'http://localhost:9200/' wait_for(es_host) with ElasticsearchConnectionManager("localhost", "9200", "", "", False) as es: logging.info("EsInfo", es.client.info()) + def search(body: dict) -> None: es.client.indices.flush() es.client.indices.refresh() return es.client.search(index="datacheck_results", body=body) try: - #set mock es data + # set mock es data es.client.index(index="datacheck_results", body=dc_success_result_es_doc, doc_type="report") es.client.index(index="datacheck_results", body=dc_failed_result_es_doc, doc_type="report") logging.info("Test Indexes created") yield search except: + logging.exception("Test Indexes documents creation failed") raise RuntimeWarning("Unable to create indexes!") finally: if es.client.indices.exists("datacheck_results"): es.client.indices.delete("datacheck_results") + @pytest.fixture() def es_query(): return { - "query": { - "bool": { - "must": [ - { - "match": { - "division.keyword": { - "query": "EnsemblVertebrates", - "operator": "and" - } - } - }, - { - "match": { - "file.keyword": { - "query": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json", - "operator": "and" - } - } - } - ] - } - }, - "size": 1, - "sort": [ - { - "report_time": { - "unmapped_type": "keyword", - "order": "desc" - } - } - ] + "query": { + "term": { + "file.keyword": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json" } + }, + "size": 1, + "sort": [ + { + "report_time": { + "unmapped_type": "keyword", + "order": "desc" + } + } + ] + } + - @pytest.fixture def appclient(): - app.config['ENS_VERSION'] = '108' + app.config['ENS_VERSION'] = '110' app.config['TESTING'] = True - app.config['ES_HOST'] ='localhost' + app.config['ES_HOST'] = 'localhost' app.config['ES_PORT'] = '9200' app.config['ES_USER'] = '' app.config['ES_PASSWORD'] = '' @@ -189,6 +174,3 @@ def appclient(): with app.test_client() as appclient: yield appclient - - - diff --git a/src/tests/test_dc_results_from_es.py b/src/tests/test_dc_results_from_es.py index a51928a..c14b1b3 100644 --- a/src/tests/test_dc_results_from_es.py +++ b/src/tests/test_dc_results_from_es.py @@ -24,6 +24,7 @@ def test_get_dc_results_success(appclient, elastic_search, es_query): response = appclient.get( '/jobs/details?jsonfile=/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json') data = json.loads(response.data) + print(data) assert response.status_code == 200 assert data == {}