From 2a3005c7989100b0753139cfb0e5a7bad81547d8 Mon Sep 17 00:00:00 2001 From: Alexander Kiel Date: Sun, 29 Sep 2024 16:58:43 +0200 Subject: [PATCH] Enable Keep-Alive Backend Connections on Ingress Previously the HTTP connections to the backend were not kept alive. So the performance of many small reads suffered. See the difference in docs/performance/load-testing.md --- docs/performance/load-testing.md | 34 +++++++++++++----------- modules/ingress/default.conf.template | 8 +++++- modules/load-test/Makefile | 8 +++++- modules/load-test/http-req-duration.jq | 14 ++++++++++ modules/load-test/patient-everything.js | 30 +++++++-------------- modules/load-test/read-single-patient.js | 32 +++++++--------------- modules/load-test/report.sh | 5 ++++ 7 files changed, 69 insertions(+), 62 deletions(-) create mode 100644 modules/load-test/http-req-duration.jq create mode 100755 modules/load-test/report.sh diff --git a/docs/performance/load-testing.md b/docs/performance/load-testing.md index 38b22e53c..e2614eab6 100644 --- a/docs/performance/load-testing.md +++ b/docs/performance/load-testing.md @@ -17,26 +17,28 @@ The load testing tool [k6][1] is used to create load from another host in the sa ### Results -| Dataset | System | VUs | Req/s | med | q95 | q99 | -|---------|--------|----:|------:|----:|----:|----:| -| 1M | A5N46 | 1 | 1330 | 0.5 | 0.7 | 1.1 | -| 1M | A5N46 | 2 | 2793 | 0.5 | 0.7 | 0.9 | -| 1M | A5N46 | 4 | 5250 | 0.6 | 0.7 | 1.1 | -| 1M | A5N46 | 8 | 9749 | 0.6 | 0.8 | 1.3 | -| 1M | A5N46 | 16 | 17012 | 0.7 | 1.0 | 1.7 | -| 1M | A5N46 | 32 | 24035 | 1.0 | 1.9 | 3.4 | +| Dataset | System | VUs | Req/s | med | q95 | q99 | +|---------|--------|----:|------:|-----:|-----:|-----:| +| 1M | A5N46 | 1 | 1405 | 0.50 | 0.74 | 1.47 | +| 1M | A5N46 | 2 | 3907 | 0.45 | 0.57 | 0.67 | +| 1M | A5N46 | 4 | 7248 | 0.53 | 0.59 | 0.69 | +| 1M | A5N46 | 8 | 13381 | 0.55 | 0.67 | 0.88 | +| 1M | A5N46 | 16 | 23678 | 0.60 | 0.82 | 1.21 | +| 1M | A5N46 | 32 | 38314 | 0.73 | 1.13 | 1.90 | +| 1M | A5N46 | 48 | 45679 | 0.89 | 1.58 | 3.22 | +| 1M | A5N46 | 64 | 48868 | 1.07 | 2.20 | 4.12 | ## Patient Everything ### Results -| Dataset | System | VUs | Req/s | med | q95 | q99 | -|---------|--------|----:|------:|-----:|------:|------:| -| 1M | A5N46 | 1 | 40.50 | 15.7 | 28.3 | 54.7 | -| 1M | A5N46 | 2 | 73.07 | 16.9 | 31.7 | 41.7 | -| 1M | A5N46 | 4 | 126.6 | 19.9 | 39.6 | 61.3 | -| 1M | A5N46 | 8 | 192.4 | 26.7 | 56.2 | 87.3 | -| 1M | A5N46 | 16 | 240.9 | 46.1 | 101.6 | 127.5 | -| 1M | A5N46 | 32 | 272.2 | 91.8 | 166.6 | 200.9 | +| Dataset | System | VUs | Req/s | med | q95 | q99 | +|---------|--------|----:|------:|------:|------:|------:| +| 1M | A5N46 | 1 | 40.50 | 15.7 | 28.3 | 54.7 | +| 1M | A5N46 | 2 | 73.07 | 16.9 | 31.7 | 41.7 | +| 1M | A5N46 | 4 | 162.9 | 21.9 | 42.3 | 59.0 | +| 1M | A5N46 | 8 | 234.9 | 30.5 | 60.1 | 90.2 | +| 1M | A5N46 | 16 | 261.2 | 57.6 | 98.7 | 125.1 | +| 1M | A5N46 | 32 | 258.7 | 119.4 | 174.3 | 202.1 | [1]: diff --git a/modules/ingress/default.conf.template b/modules/ingress/default.conf.template index cf369967b..08a8100d9 100644 --- a/modules/ingress/default.conf.template +++ b/modules/ingress/default.conf.template @@ -1,7 +1,12 @@ +upstream backend { + server backend:8080; + keepalive 64; +} + map $http_authorization $upstream { default http://frontend:3000; - "~Bearer" http://backend:8080; + "~Bearer" http://backend; } server { @@ -38,6 +43,7 @@ server { proxy_pass $upstream; proxy_set_header x-forwarded-proto https; proxy_set_header x-forwarded-host ${BLAZE_HOST}; + proxy_set_header Connection ""; } } diff --git a/modules/load-test/Makefile b/modules/load-test/Makefile index 95055ff9e..22fe6b123 100644 --- a/modules/load-test/Makefile +++ b/modules/load-test/Makefile @@ -1 +1,7 @@ -.PHONY: fmt lint test-coverage cloc-prod cloc-test +run-read-single-patient: + TZ=UTC k6 run --out json=/tmp/test-results.json --no-summary read-single-patient.js + +report: + ./report.sh /tmp/test-results.json + +.PHONY: fmt lint test-coverage cloc-prod cloc-test run-read-single-patient diff --git a/modules/load-test/http-req-duration.jq b/modules/load-test/http-req-duration.jq new file mode 100644 index 000000000..31950df71 --- /dev/null +++ b/modules/load-test/http-req-duration.jq @@ -0,0 +1,14 @@ +[ inputs | select( + .type == "Point" and + .metric == "http_req_duration" and + .data.tags.vus_active == "16" and + .data.tags.name == "everything" and + .data.tags.status == "200" +).data ] as $data | [ $data[].value ] | sort as $durations | +{ + "req_s": (length / (($data[-1].time[:19] + "Z") | fromdate - (($data[0].time[:19] + "Z") | fromdate))), + "cnt": length, + "med": $durations[(length * 0.5) | round], + "q95": $durations[(length * 0.95) | round], + "q99": $durations[(length * 0.99) | round] +} diff --git a/modules/load-test/patient-everything.js b/modules/load-test/patient-everything.js index 8560a5be2..01cbad4b7 100644 --- a/modules/load-test/patient-everything.js +++ b/modules/load-test/patient-everything.js @@ -1,18 +1,13 @@ import http from 'k6/http'; -import { check, fail } from 'k6'; -import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; +import exec from 'k6/execution'; +import { fail } from 'k6'; +import { randomItem } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js'; export const options = { - thresholds: { - http_req_failed: [{ threshold: 'rate<0.01' }], // http errors should be less than 0.1% - http_req_duration: ['p(99)<100'] // 99% of requests should be below 100ms - }, - setupTimeout: '300s', insecureSkipTLSVerify: true, - - summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'], + discardResponseBodies: true, stages: [ { duration: '30s', target: 16 }, @@ -24,7 +19,7 @@ export const options = { const base = 'https://blaze.srv.local/fhir'; //const base = 'http://localhost:8080/fhir'; //const base = 'http://blaze-test-host:8080/fhir'; -const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3MjY4MTQ5MTMsImlhdCI6MTcyNjgxMTMxMywianRpIjoiOGE3NDMyMWYtZDgzNy00NmY2LTlmNDQtODM4OTM2MWU1Y2EzIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuNCIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuNCJ9.OahQzB5-dBZww3qAdR1_HV3wsuvAU0HnbQAvZl1Nt61E3TfBCrfDVBZxFEoXFBNL-9NJUwDd5IonbF7GyumgyNs9B7oplKaJ71X6q6sE5lLMz2jFWCax0-CPovCQlHtubkO8p9OR1CiGPIldGUhz_K1uQsDGSVfo1QAY0Z_QuFohxz-mqqxy2Dk9I-WcIS_ypjLsBBO2TqaajSt6nn8mt-ZNUpmDEM-TME7ac2Mum42AOtkIVE_sToZCBOQCQu9TTFrgiYWWVYyBxJIyv3NHrGRImdluorQV-9fAdM1RJLjWV5lPqqLiivljr000IkQsk88a6nbmj84rWWTjKBU33A'; +const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3Mjc2MjQ2MjUsImlhdCI6MTcyNzYyMTAyNSwianRpIjoiMzdlMTc1MjgtODc1ZC00Njc2LTljZjAtODYxMjE4NWQ0NTUxIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuMyIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuMyJ9.KGRpmMd_dmGilvZskup-o7bW-Cwn5gPgnKshUN6EhtyH47WQyofIgKYekx3LiLYgyvYO8QW0LTwdmWAkW55WLQxq84G-AiwyfYiItaaoJuIWOBrGZu59ATNualVxhoCHUEJy9W3Ndkajp9ZXDyonvOY2AxtSxlJG6RdBeIQ2b4_zjRAdKqziG3e6g9ufZIlil0uO3ZRZVR-zd19mRXXmG_zCceP_fHFbRvyth4w8lm0M_L5HBQRatvkUHFCaXL2HGO6VLUTEPywLKpXdX403cL7mu7CebGUXwBa9XybacOScBVJzwjPcwQq-3RRnrIaPltQ-zBboP8hLyGhyOLYygg'; const commonHeaders = { 'Accept': 'application/fhir+json', @@ -41,25 +36,18 @@ const everythingParams = { const searchTypeParams = { headers: commonHeaders, + responseType: "text", tags: { name: 'search-type' } }; export default function({ patientIds }) { - const id = randomItem(patientIds); + exec.vu.tags['vus_active'] = exec.instance.vusActive; - const res = http.get(`${base}/Patient/${id}/$everything?_count=1000`, everythingParams); - - check(res, { - 'response code was 200': (res) => res.status === 200 - }); - - const body = res.json(); + const id = randomItem(patientIds); - check(body, { - 'response body is an searchset Bundle': (body) => body.resourceType === 'Bundle' && body.type === 'searchset', - }); + http.get(`${base}/Patient/${id}/$everything?_count=1000`, everythingParams); } function readPatientIds(url) { diff --git a/modules/load-test/read-single-patient.js b/modules/load-test/read-single-patient.js index 8ec89b03c..76f135878 100644 --- a/modules/load-test/read-single-patient.js +++ b/modules/load-test/read-single-patient.js @@ -1,18 +1,13 @@ import http from 'k6/http'; -import { check, fail } from 'k6'; -import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; +import exec from 'k6/execution'; +import { fail } from 'k6'; +import { randomItem } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js'; export const options = { - thresholds: { - http_req_failed: [{ threshold: 'rate<0.01' }], // http errors should be less than 0.1% - http_req_duration: ['p(99)<100'] // 99% of requests should be below 100ms - }, - setupTimeout: '300s', insecureSkipTLSVerify: true, - - summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'], + discardResponseBodies: true, stages: [ { duration: '30s', target: 64 }, @@ -24,7 +19,7 @@ export const options = { const base = 'https://blaze.srv.local/fhir'; //const base = 'http://localhost:8080/fhir'; //const base = 'http://blaze-test-host:8080/fhir'; -const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3MjY3NzM4MzIsImlhdCI6MTcyNjc3MDIzMiwianRpIjoiOWVhODk2MzYtNGMzNy00NWMyLTkwMDQtZjA0MDkyYjA2YzhiIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuNCIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuNCJ9.gC4lWwCsW-tA_hYLyNjafb10wTmYYKKxmAzAFIeIWHjMmqD5mj2Vg8vkOK8Dy_7DfeHPtMP3_Yavqwd7b3ggdvyJwg3gdMVZniLDBwf0Jt5v_ldih6MXn19l2WfUsfucMFWCkZsAekApB7Vp2s6dKZcwvc3noA5uCBSn2sSaUt8xFoSBOsOgQRNVw2U4pT7eVzyjz5bXC_ZlTqyfQAtmU2k0pedhFZ0U4m3L-zhydSgz-wfEhaDyWjC1IgM5Q8kUj9gVoQtFS6zZ_959J6BOXdnq6ozMLlxZSXjGJ0EH1zxu0pWJMA-WU8C96VhsmxD6w8GynXB2b9SQeqOoxeXlAw'; +const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3Mjc1NTQxODgsImlhdCI6MTcyNzU1MDU4OCwianRpIjoiNzM4OTQyZmQtOGViZS00MzM1LWFkZDgtYzQyMjMxZDUzNjMyIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuMyIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuMyJ9.ND8MJD-SUNoQXvts4pYheQBoV2DZ57vVejUFkpkMSWoodu_kxVeOhHfLqv1y0jP6VPGa2-XAuxeJQj_5wzVsvfAWoWw793svoGCVDErGxL0g6jySoxDbaoJ3HIWwV0muOgf9PVoVlJg3iYEc8qGGsT6HYXhOb3BSFObaDHcDRUnerbECaDoAVcOhuR9fMy0pCFg2TBBkjW_ldR0lNpckxZyesl0dxSMt3VXlC_FUVmSWqh_u7KSmh1VhyvK1zw5GACd3qGppfrvdAHW9rlQ5VHIJneFPOYjqcCiK4Gvy0EVceU8POar2-crd3_e8kXt_QCeIOgP0N3jL994y6E0_pA'; const commonHeaders = { 'Accept': 'application/fhir+json', @@ -41,26 +36,17 @@ const readParams = { const searchTypeParams = { headers: commonHeaders, + responseType: "text", tags: { name: 'search-type' } }; export default function({ patientIds }) { - const id = randomItem(patientIds); - - const res = http.get(`${base}/Patient/${id}`, readParams); + exec.vu.tags['vus_active'] = exec.instance.vusActive; - check(res, { - 'response code was 200': (res) => res.status === 200 - }); - - const body = res.json(); - - check(body, { - 'response body is a Patient': (body) => body.resourceType === 'Patient', - 'Patient id': (body) => body.id === id - }); + const id = randomItem(patientIds); + http.get(`${base}/Patient/${id}`, readParams); } function readPatientIds(url) { diff --git a/modules/load-test/report.sh b/modules/load-test/report.sh new file mode 100755 index 000000000..8b613a84c --- /dev/null +++ b/modules/load-test/report.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +RESULTS_FILE="$1" + +jq -f http-req-duration.jq "$RESULTS_FILE"