Enable Keep-Alive Backend Connections on Ingress

Previously the HTTP connections to the backend were not kept alive. So the performance of many small reads suffered. See the difference in docs/performance/load-testing.md
samply · Sep 29, 2024 · 2a3005c · 2a3005c
1 parent 3bc5675
commit 2a3005c
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 62 deletions.
diff --git a/docs/performance/load-testing.md b/docs/performance/load-testing.md
@@ -17,26 +17,28 @@ The load testing tool [k6][1] is used to create load from another host in the sa
 
 ### Results
 
-| Dataset | System | VUs | Req/s | med | q95 | q99 |
-|---------|--------|----:|------:|----:|----:|----:|
-| 1M      | A5N46  |   1 |  1330 | 0.5 | 0.7 | 1.1 |
-| 1M      | A5N46  |   2 |  2793 | 0.5 | 0.7 | 0.9 |
-| 1M      | A5N46  |   4 |  5250 | 0.6 | 0.7 | 1.1 |
-| 1M      | A5N46  |   8 |  9749 | 0.6 | 0.8 | 1.3 |
-| 1M      | A5N46  |  16 | 17012 | 0.7 | 1.0 | 1.7 |
-| 1M      | A5N46  |  32 | 24035 | 1.0 | 1.9 | 3.4 |
+| Dataset | System | VUs | Req/s |  med |  q95 |  q99 |
+|---------|--------|----:|------:|-----:|-----:|-----:|
+| 1M      | A5N46  |   1 |  1405 | 0.50 | 0.74 | 1.47 |
+| 1M      | A5N46  |   2 |  3907 | 0.45 | 0.57 | 0.67 |
+| 1M      | A5N46  |   4 |  7248 | 0.53 | 0.59 | 0.69 |
+| 1M      | A5N46  |   8 | 13381 | 0.55 | 0.67 | 0.88 |
+| 1M      | A5N46  |  16 | 23678 | 0.60 | 0.82 | 1.21 |
+| 1M      | A5N46  |  32 | 38314 | 0.73 | 1.13 | 1.90 |
+| 1M      | A5N46  |  48 | 45679 | 0.89 | 1.58 | 3.22 |
+| 1M      | A5N46  |  64 | 48868 | 1.07 | 2.20 | 4.12 |
 
 ## Patient Everything
 
 ### Results
 
-| Dataset | System | VUs | Req/s |  med |   q95 |   q99 |
-|---------|--------|----:|------:|-----:|------:|------:|
-| 1M      | A5N46  |   1 | 40.50 | 15.7 |  28.3 |  54.7 |
-| 1M      | A5N46  |   2 | 73.07 | 16.9 |  31.7 |  41.7 |
-| 1M      | A5N46  |   4 | 126.6 | 19.9 |  39.6 |  61.3 |
-| 1M      | A5N46  |   8 | 192.4 | 26.7 |  56.2 |  87.3 |
-| 1M      | A5N46  |  16 | 240.9 | 46.1 | 101.6 | 127.5 |
-| 1M      | A5N46  |  32 | 272.2 | 91.8 | 166.6 | 200.9 |
+| Dataset | System | VUs | Req/s |   med |   q95 |   q99 |
+|---------|--------|----:|------:|------:|------:|------:|
+| 1M      | A5N46  |   1 | 40.50 |  15.7 |  28.3 |  54.7 |
+| 1M      | A5N46  |   2 | 73.07 |  16.9 |  31.7 |  41.7 |
+| 1M      | A5N46  |   4 | 162.9 |  21.9 |  42.3 |  59.0 |
+| 1M      | A5N46  |   8 | 234.9 |  30.5 |  60.1 |  90.2 |
+| 1M      | A5N46  |  16 | 261.2 |  57.6 |  98.7 | 125.1 |
+| 1M      | A5N46  |  32 | 258.7 | 119.4 | 174.3 | 202.1 |
 
 [1]: <https://k6.io>
diff --git a/modules/ingress/default.conf.template b/modules/ingress/default.conf.template
@@ -1,7 +1,12 @@
+upstream backend {
+    server backend:8080;
+    keepalive 64;
+}
+
 map $http_authorization $upstream {
     default http://frontend:3000;
 
-    "~Bearer" http://backend:8080;
+    "~Bearer" http://backend;
 }
 
 server {
@@ -38,6 +43,7 @@ server {
         proxy_pass $upstream;
         proxy_set_header x-forwarded-proto https;
         proxy_set_header x-forwarded-host ${BLAZE_HOST};
+        proxy_set_header Connection "";
     }
 }
 

diff --git a/modules/load-test/Makefile b/modules/load-test/Makefile
@@ -1 +1,7 @@
-.PHONY: fmt lint test-coverage cloc-prod cloc-test
+run-read-single-patient:
+	TZ=UTC k6 run --out json=/tmp/test-results.json --no-summary read-single-patient.js
+
+report:
+	./report.sh /tmp/test-results.json
+
+.PHONY: fmt lint test-coverage cloc-prod cloc-test run-read-single-patient
diff --git a/modules/load-test/http-req-duration.jq b/modules/load-test/http-req-duration.jq
@@ -0,0 +1,14 @@
+[ inputs | select(
+  .type == "Point" and
+  .metric == "http_req_duration" and
+  .data.tags.vus_active == "16" and
+  .data.tags.name == "everything" and
+  .data.tags.status == "200"
+).data ] as $data | [ $data[].value ] | sort as $durations |
+{
+  "req_s": (length / (($data[-1].time[:19] + "Z") | fromdate - (($data[0].time[:19] + "Z") | fromdate))),
+  "cnt": length,
+  "med": $durations[(length * 0.5) | round],
+  "q95": $durations[(length * 0.95) | round],
+  "q99": $durations[(length * 0.99) | round]
+}
diff --git a/modules/load-test/patient-everything.js b/modules/load-test/patient-everything.js
@@ -1,18 +1,13 @@
 import http from 'k6/http';
-import { check, fail } from 'k6';
-import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js';
+import exec from 'k6/execution';
+import { fail } from 'k6';
+import { randomItem } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js';
 
 export const options = {
 
-	thresholds: {
-		http_req_failed: [{ threshold: 'rate<0.01' }], // http errors should be less than 0.1%
-		http_req_duration: ['p(99)<100'] // 99% of requests should be below 100ms
-	},
-
 	setupTimeout: '300s',
 	insecureSkipTLSVerify: true,
-
-	summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'],
+	discardResponseBodies: true,
 
 	stages: [
 		{ duration: '30s', target: 16 },
@@ -24,7 +19,7 @@ export const options = {
 const base = 'https://blaze.srv.local/fhir';
 //const base = 'http://localhost:8080/fhir';
 //const base = 'http://blaze-test-host:8080/fhir';
-const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3MjY4MTQ5MTMsImlhdCI6MTcyNjgxMTMxMywianRpIjoiOGE3NDMyMWYtZDgzNy00NmY2LTlmNDQtODM4OTM2MWU1Y2EzIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuNCIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuNCJ9.OahQzB5-dBZww3qAdR1_HV3wsuvAU0HnbQAvZl1Nt61E3TfBCrfDVBZxFEoXFBNL-9NJUwDd5IonbF7GyumgyNs9B7oplKaJ71X6q6sE5lLMz2jFWCax0-CPovCQlHtubkO8p9OR1CiGPIldGUhz_K1uQsDGSVfo1QAY0Z_QuFohxz-mqqxy2Dk9I-WcIS_ypjLsBBO2TqaajSt6nn8mt-ZNUpmDEM-TME7ac2Mum42AOtkIVE_sToZCBOQCQu9TTFrgiYWWVYyBxJIyv3NHrGRImdluorQV-9fAdM1RJLjWV5lPqqLiivljr000IkQsk88a6nbmj84rWWTjKBU33A';
+const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3Mjc2MjQ2MjUsImlhdCI6MTcyNzYyMTAyNSwianRpIjoiMzdlMTc1MjgtODc1ZC00Njc2LTljZjAtODYxMjE4NWQ0NTUxIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuMyIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuMyJ9.KGRpmMd_dmGilvZskup-o7bW-Cwn5gPgnKshUN6EhtyH47WQyofIgKYekx3LiLYgyvYO8QW0LTwdmWAkW55WLQxq84G-AiwyfYiItaaoJuIWOBrGZu59ATNualVxhoCHUEJy9W3Ndkajp9ZXDyonvOY2AxtSxlJG6RdBeIQ2b4_zjRAdKqziG3e6g9ufZIlil0uO3ZRZVR-zd19mRXXmG_zCceP_fHFbRvyth4w8lm0M_L5HBQRatvkUHFCaXL2HGO6VLUTEPywLKpXdX403cL7mu7CebGUXwBa9XybacOScBVJzwjPcwQq-3RRnrIaPltQ-zBboP8hLyGhyOLYygg';
 
 const commonHeaders = {
 	'Accept': 'application/fhir+json',
@@ -41,25 +36,18 @@ const everythingParams = {
 
 const searchTypeParams = {
 	headers: commonHeaders,
+	responseType: "text",
 	tags: {
 		name: 'search-type'
 	}
 };
 
 export default function({ patientIds }) {
-	const id = randomItem(patientIds);
+	exec.vu.tags['vus_active'] = exec.instance.vusActive;
 
-	const res = http.get(`${base}/Patient/${id}/$everything?_count=1000`, everythingParams);
-
-	check(res, {
-		'response code was 200': (res) => res.status === 200
-	});
-
-	const body = res.json();
+	const id = randomItem(patientIds);
 
-	check(body, {
-		'response body is an searchset Bundle': (body) => body.resourceType === 'Bundle' && body.type === 'searchset',
-	});
+	http.get(`${base}/Patient/${id}/$everything?_count=1000`, everythingParams);
 }
 
 function readPatientIds(url) {

diff --git a/modules/load-test/read-single-patient.js b/modules/load-test/read-single-patient.js
@@ -1,18 +1,13 @@
 import http from 'k6/http';
-import { check, fail } from 'k6';
-import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js';
+import exec from 'k6/execution';
+import { fail } from 'k6';
+import { randomItem } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js';
 
 export const options = {
 
-	thresholds: {
-		http_req_failed: [{ threshold: 'rate<0.01' }], // http errors should be less than 0.1%
-		http_req_duration: ['p(99)<100'] // 99% of requests should be below 100ms
-	},
-
 	setupTimeout: '300s',
 	insecureSkipTLSVerify: true,
-
-	summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'count'],
+	discardResponseBodies: true,
 
 	stages: [
 		{ duration: '30s', target: 64 },
@@ -24,7 +19,7 @@ export const options = {
 const base = 'https://blaze.srv.local/fhir';
 //const base = 'http://localhost:8080/fhir';
 //const base = 'http://blaze-test-host:8080/fhir';
-const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3MjY3NzM4MzIsImlhdCI6MTcyNjc3MDIzMiwianRpIjoiOWVhODk2MzYtNGMzNy00NWMyLTkwMDQtZjA0MDkyYjA2YzhiIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuNCIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuNCJ9.gC4lWwCsW-tA_hYLyNjafb10wTmYYKKxmAzAFIeIWHjMmqD5mj2Vg8vkOK8Dy_7DfeHPtMP3_Yavqwd7b3ggdvyJwg3gdMVZniLDBwf0Jt5v_ldih6MXn19l2WfUsfucMFWCkZsAekApB7Vp2s6dKZcwvc3noA5uCBSn2sSaUt8xFoSBOsOgQRNVw2U4pT7eVzyjz5bXC_ZlTqyfQAtmU2k0pedhFZ0U4m3L-zhydSgz-wfEhaDyWjC1IgM5Q8kUj9gVoQtFS6zZ_959J6BOXdnq6ozMLlxZSXjGJ0EH1zxu0pWJMA-WU8C96VhsmxD6w8GynXB2b9SQeqOoxeXlAw';
+const accessToken = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ5VmMwcnVQZjdrMDgxN2JWMWF0ZFoycWpJUUFqYnR3RUpiZklvZ3k3aElzIn0.eyJleHAiOjE3Mjc1NTQxODgsImlhdCI6MTcyNzU1MDU4OCwianRpIjoiNzM4OTQyZmQtOGViZS00MzM1LWFkZDgtYzQyMjMxZDUzNjMyIiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5zcnYubG9jYWwvcmVhbG1zL2JsYXplIiwic3ViIjoiNzJhN2YzN2UtYjMzZi00OTA4LTlhZDktMzNiZTBkNGMxNjIwIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiYWNjb3VudCIsInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6ImVtYWlsIHByb2ZpbGUiLCJjbGllbnRJZCI6ImFjY291bnQiLCJjbGllbnRIb3N0IjoiMTcyLjE4LjAuMyIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwicHJlZmVycmVkX3VzZXJuYW1lIjoic2VydmljZS1hY2NvdW50LWFjY291bnQiLCJjbGllbnRBZGRyZXNzIjoiMTcyLjE4LjAuMyJ9.ND8MJD-SUNoQXvts4pYheQBoV2DZ57vVejUFkpkMSWoodu_kxVeOhHfLqv1y0jP6VPGa2-XAuxeJQj_5wzVsvfAWoWw793svoGCVDErGxL0g6jySoxDbaoJ3HIWwV0muOgf9PVoVlJg3iYEc8qGGsT6HYXhOb3BSFObaDHcDRUnerbECaDoAVcOhuR9fMy0pCFg2TBBkjW_ldR0lNpckxZyesl0dxSMt3VXlC_FUVmSWqh_u7KSmh1VhyvK1zw5GACd3qGppfrvdAHW9rlQ5VHIJneFPOYjqcCiK4Gvy0EVceU8POar2-crd3_e8kXt_QCeIOgP0N3jL994y6E0_pA';
 
 const commonHeaders = {
 	'Accept': 'application/fhir+json',
@@ -41,26 +36,17 @@ const readParams = {
 
 const searchTypeParams = {
 	headers: commonHeaders,
+	responseType: "text",
 	tags: {
 		name: 'search-type'
 	}
 };
 
 export default function({ patientIds }) {
-	const id = randomItem(patientIds);
-
-	const res = http.get(`${base}/Patient/${id}`, readParams);
+	exec.vu.tags['vus_active'] = exec.instance.vusActive;
 
-	check(res, {
-		'response code was 200': (res) => res.status === 200
-	});
-
-	const body = res.json();
-
-	check(body, {
-		'response body is a Patient': (body) => body.resourceType === 'Patient',
-		'Patient id': (body) => body.id === id
-	});
+	const id = randomItem(patientIds);
+	http.get(`${base}/Patient/${id}`, readParams);
 }
 
 function readPatientIds(url) {

diff --git a/modules/load-test/report.sh b/modules/load-test/report.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+RESULTS_FILE="$1"
+
+jq -f http-req-duration.jq "$RESULTS_FILE"