Skip to content

Commit

Permalink
Merge pull request #36 from Ensembl/release/2.0.3
Browse files Browse the repository at this point in the history 
Added page to view configs
  • Loading branch information
@marcoooo
marcoooo authored Feb 23, 2024
2 parents f26dbb2 + 5d03929 commit 98505ce
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 120 deletions.
1 change: 0 additions & 1 deletion initcommit.txt
View file

This file was deleted.

2 changes: 1 addition & 1 deletion requirements.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@ pytest
requests
SQLAlchemy
wtforms
ensembl-py@git+https://github.com/Ensembl/ensembl-py.git@1.0.0dev2#egg=ensembl-py
ensembl-py@git+https://github.com/Ensembl/ensembl-py.git@1.2.2
16 changes: 13 additions & 3 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# This file is autogenerated by pip-compile with Python 3.10
# This file is autogenerated by pip-compile with Python 3.8
# by the following command:
#
# pip-compile requirements.in
Expand Down Expand Up @@ -28,11 +28,11 @@ elasticsearch==6.8.2
# via -r requirements.in
email-validator==2.0.0.post1
# via -r requirements.in
ensembl-hive @ git+https://github.com/Ensembl/ensembl-hive.git@main
ensembl-hive @ git+https://github.com/Ensembl/ensembl-hive.git
# via ensembl-py
ensembl-prodinf-core==2.0.5
# via -r requirements.in
ensembl-py @ git+https://github.com/Ensembl/ensembl-py.git@1.0.0dev2
ensembl-py @ git+https://github.com/Ensembl/ensembl-py.git@1.2.2
# via -r requirements.in
exceptiongroup==1.1.1
# via pytest
Expand All @@ -57,6 +57,10 @@ idna==3.4
# via
# email-validator
# requests
importlib-metadata==7.0.1
# via flask
importlib-resources==6.1.1
# via jsonschema
iniconfig==2.0.0
# via pytest
itsdangerous==2.1.2
Expand Down Expand Up @@ -84,6 +88,8 @@ packaging==23.1
# via
# deprecation
# pytest
pkgutil-resolve-name==1.3.10
# via jsonschema
pluggy==1.0.0
# via pytest
pyrsistent==0.19.3
Expand Down Expand Up @@ -137,6 +143,10 @@ wtforms==3.0.1
# -r requirements.in
# bootstrap-flask
# flask-wtf
zipp==3.17.0
# via
# importlib-metadata
# importlib-resources

# The following packages are considered to be unsafe in a requirements file:
# setuptools
13 changes: 12 additions & 1 deletion src/ensembl/production/datacheck/app/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
es_ssl = app.config['ES_SSL']
app_es_data_source = app.config['APP_ES_DATA_SOURCE']

if app.env == 'development':
if app.debug:
# ENV dev (assumed run from builtin server, so update script_name at wsgi level)
app.wsgi_app = DispatcherMiddleware(
Response('Not Found', status=404),
Expand Down Expand Up @@ -141,6 +141,17 @@ def servers_list():
return jsonify(get_servers_list())


@app.route('/config', methods=['GET'])
def config():
""" Disply all configs vars"""
config = {key: value if 'pass' not in key.lower() else 'XXXXXX' for key, value in DatacheckConfig.__dict__.items() if
key[:1] != '_'}
return render_template(
'config.html',
config=config
)


@app.route('/servers/dict', methods=['GET'])
def servers_dict():
return jsonify(get_servers_dict())
Expand Down
29 changes: 29 additions & 0 deletions src/ensembl/production/datacheck/templates/config.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{% extends "base.html" %}
{% block styles %}
{{ super() }}
{% endblock %}
{% block content %}
{{ super() }}

<div class="row table-responsive">
<div class="col-12">
<div class="card shadow p-3 mb-5 bg-white rounded">
<div class="card-header">
Datacheck Configuration set
</div>
<div class="card-body">
<table class="bootstrap-table table-striped table result">
<tbody>
{% for key, value in config.items() %}
<tr>
<td class="bg-secondary">{{ key }}</td>
<td>{{ value }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endblock %}
2 changes: 1 addition & 1 deletion src/ensembl/production/datacheck/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def get_datacheck_results(jsonfile_path: str,
]
})
if len(res['hits']['hits']) == 0:
raise ElasticsearchException(f"""No Hits Found for given params jsonfile_path {jsonfile_path} """)
raise ElasticsearchException(f"""No Hits Found for given params jsonfile_path {jsonfile_path}""")

return {"status": True, "message": "", "result": res['hits']['hits'][0]['_source']['content']}

Expand Down
208 changes: 95 additions & 113 deletions src/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,88 +22,86 @@
from ensembl.production.core.es import ElasticsearchConnectionManager
from ensembl.production.datacheck.app.main import app


dc_success_result_es_doc = {
"job_id": "1",
"input_details": {
"datacheck_groups": [],
"server_uri": None,
"config_file": None,
"datacheck_names": [
"SpeciesCommonName"
],
"dbname": [
"mus_musculus_core_107_39"
],
"datacheck_types": [],
"registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm",
"timestamp": "Fri Jul 1 15:12:42 2022",
"db_type": "core",
"target_url": None,
"server_url": None,
"tag": "Vertebrate datachecks, pre-handover to web ()"
},
"division": "EnsemblVertebrates",
"content": {},
"file": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json"
}


dc_failed_result_es_doc = {
"job_id": "2",
"input_details": {
"datacheck_groups": [],
"server_uri": None,
"config_file": None,
"datacheck_names": [
"SpeciesCommonName"
],
"dbname": [
"mus_musculus_core_107_39"
],
"datacheck_types": [],
"registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm",
"timestamp": "Fri Jul 1 15:12:42 2022",
"db_type": "core",
"target_url": None,
"server_url": None,
"tag": "Vertebrate datachecks, pre-handover to web ()"
},
"division": "EnsemblVertebrates",
"content": {
"camarhynchus_parvulus, core, camarhynchus_parvulus_core_108_11, EnsemblVertebrates" : {
"CompareMetaKeys" : {
"ok" : 0,
"tests" : {
"not ok 4 - Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11" : [
" Failed test 'Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11'",
" at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.",
" Structures begin differing at:",
" $got->{ENSCPVG00005007000} = Does not exist",
" $expected->{ENSCPVG00005007000} = HASH(0x568a2c0)",
"Looks like you failed 1 test of 4."
dc_success_result_es_doc = {
"job_id": "1",
"input_details": {
"datacheck_groups": [],
"server_uri": None,
"config_file": None,
"datacheck_names": [
"SpeciesCommonName"
],
"dbname": [
"mus_musculus_core_107_39"
],
"datacheck_types": [],
"registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm",
"timestamp": "Fri Jul 1 15:12:42 2022",
"db_type": "core",
"target_url": None,
"server_url": None,
"tag": "Vertebrate datachecks, pre-handover to web ()"
},
"division": "EnsemblVertebrates",
"content": {},
"file": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json"
}

dc_failed_result_es_doc = {
"job_id": "2",
"input_details": {
"datacheck_groups": [],
"server_uri": None,
"config_file": None,
"datacheck_names": [
"SpeciesCommonName"
],
"dbname": [
"mus_musculus_core_107_39"
],
"datacheck_types": [],
"registry_file": "/hps/software/users/ensembl/ensw/registries/production/st1b.pm",
"timestamp": "Fri Jul 1 15:12:42 2022",
"db_type": "core",
"target_url": None,
"server_url": None,
"tag": "Vertebrate datachecks, pre-handover to web ()"
},
"division": "EnsemblVertebrates",
"content": {
"camarhynchus_parvulus, core, camarhynchus_parvulus_core_108_11, EnsemblVertebrates": {
"CompareMetaKeys": {
"ok": 0,
"tests": {
"not ok 4 - Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11": [
" Failed test 'Gene IDs, positions, and biotype groups are the same between camarhynchus_parvulus_core_108_11 and camarhynchus_parvulus_core_107_11'",
" at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.",
" Structures begin differing at:",
" $got->{ENSCPVG00005007000} = Does not exist",
" $expected->{ENSCPVG00005007000} = HASH(0x568a2c0)",
"Looks like you failed 1 test of 4."
]
}
}
},
"corvus_moneduloides, core, corvus_moneduloides_core_108_1, EnsemblVertebrates" : {
"CompareMetaKeys" : {
"ok" : 0,
"tests" : {
"not ok 4 - Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1" : [
" Failed test 'Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1'",
" at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.",
" Structures begin differing at:",
" $got->{ENSCMUG00005009949} = Does not exist",
" $expected->{ENSCMUG00005009949} = HASH(0x78dd4f0)",
"Looks like you failed 1 test of 4."
"corvus_moneduloides, core, corvus_moneduloides_core_108_1, EnsemblVertebrates": {
"CompareMetaKeys": {
"ok": 0,
"tests": {
"not ok 4 - Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1": [
" Failed test 'Gene IDs, positions, and biotype groups are the same between corvus_moneduloides_core_108_1 and corvus_moneduloides_core_107_1'",
" at /hps/software/users/ensembl/repositories/enseven/ensembl-datacheck/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm line 143.",
" Structures begin differing at:",
" $got->{ENSCMUG00005009949} = Does not exist",
" $expected->{ENSCMUG00005009949} = HASH(0x78dd4f0)",
"Looks like you failed 1 test of 4."
]
}
}
}
},
"file": "/homes/user/test_es_output/user_sL3mnrNTRrr1/results_by_species.json"
}
},
"file": "/homes/user/test_es_output/user_sL3mnrNTRrr1/results_by_species.json"
}


def wait_for(url: str, retries: int = 2, backoff: float = 0.2):
Expand All @@ -115,71 +113,58 @@ def wait_for(url: str, retries: int = 2, backoff: float = 0.2):
manager = urllib3.PoolManager(retries=retry)
manager.request("GET", url)

@pytest.fixture(scope="session")

@pytest.fixture(scope="session", autouse=True)
def elastic_search():
es_host = 'http://localhost:9200/'
wait_for(es_host)
with ElasticsearchConnectionManager("localhost", "9200", "", "", False) as es:
logging.info("EsInfo", es.client.info())

def search(body: dict) -> None:
es.client.indices.flush()
es.client.indices.refresh()
return es.client.search(index="datacheck_results", body=body)

try:
#set mock es data
# set mock es data
es.client.index(index="datacheck_results", body=dc_success_result_es_doc, doc_type="report")
es.client.index(index="datacheck_results", body=dc_failed_result_es_doc, doc_type="report")
logging.info("Test Indexes created")
yield search
except:
logging.exception("Test Indexes documents creation failed")
raise RuntimeWarning("Unable to create indexes!")
finally:
if es.client.indices.exists("datacheck_results"):
es.client.indices.delete("datacheck_results")


@pytest.fixture()
def es_query():
return {
"query": {
"bool": {
"must": [
{
"match": {
"division.keyword": {
"query": "EnsemblVertebrates",
"operator": "and"
}
}
},
{
"match": {
"file.keyword": {
"query": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json",
"operator": "and"
}
}
}
]
}
},
"size": 1,
"sort": [
{
"report_time": {
"unmapped_type": "keyword",
"order": "desc"
}
}
]
"query": {
"term": {
"file.keyword": "/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json"
}
},
"size": 1,
"sort": [
{
"report_time": {
"unmapped_type": "keyword",
"order": "desc"
}
}
]
}



@pytest.fixture
def appclient():
app.config['ENS_VERSION'] = '108'
app.config['ENS_VERSION'] = '110'
app.config['TESTING'] = True
app.config['ES_HOST'] ='localhost'
app.config['ES_HOST'] = 'localhost'
app.config['ES_PORT'] = '9200'
app.config['ES_USER'] = ''
app.config['ES_PASSWORD'] = ''
Expand All @@ -189,6 +174,3 @@ def appclient():

with app.test_client() as appclient:
yield appclient



1 change: 1 addition & 0 deletions src/tests/test_dc_results_from_es.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def test_get_dc_results_success(appclient, elastic_search, es_query):
response = appclient.get(
'/jobs/details?jsonfile=/homes/user/test_es_output/user_sL2nmrNTRkjE/results_by_species.json')
data = json.loads(response.data)
print(data)
assert response.status_code == 200
assert data == {}

Expand Down

0 comments on commit 98505ce

Please sign in to comment.