diff --git a/src/shadowbox/server/manager_metrics.spec.ts b/src/shadowbox/server/manager_metrics.spec.ts index a66097b1a..33931e51b 100644 --- a/src/shadowbox/server/manager_metrics.spec.ts +++ b/src/shadowbox/server/manager_metrics.spec.ts @@ -41,6 +41,19 @@ describe('PrometheusManagerMetrics', () => { const managerMetrics = new PrometheusManagerMetrics( new QueryMapPrometheusClient( { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { + resultType: 'vector', + result: [ + { + metric: { + location: 'US', + asn: '49490', + asorg: 'Test AS Org', + }, + value: [1739284734, '1234'], + }, + ], + }, 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[0s])) by (location, asn, asorg)': { resultType: 'vector', @@ -93,6 +106,22 @@ describe('PrometheusManagerMetrics', () => { }, }, { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { + resultType: 'matrix', + result: [ + { + metric: { + location: 'US', + asn: '49490', + asorg: 'Test AS Org', + }, + values: [ + [1738959398, '5678'], + [1739284734, '1234'], + ], + }, + ], + }, 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[300s])) by (access_key)': { resultType: 'matrix', result: [ @@ -102,7 +131,7 @@ describe('PrometheusManagerMetrics', () => { }, values: [ [1738959398, '1000'], - [1738959398, '2000'], + [1739284734, '2000'], ], }, ], @@ -116,7 +145,7 @@ describe('PrometheusManagerMetrics', () => { }, values: [ [1738959398, '1000'], - [1738959398, '0'], + [1739284734, '0'], ], }, ], @@ -128,19 +157,38 @@ describe('PrometheusManagerMetrics', () => { const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0}); expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{ - "server": [ - { - "location": "US", - "asn": 49490, - "asOrg": "Test AS Org", - "dataTransferred": { + "server": { + "tunnelTime": { + "seconds": 1000 + }, + "dataTransferred": { + "total": { "bytes": 1000 }, - "tunnelTime": { - "seconds": 1000 + "current": { + "bytes": 1234 + }, + "peak": { + "data": { + "bytes": 5678 + }, + "timestamp": 1738959398 } - } - ], + }, + "locations": [ + { + "location": "US", + "asn": 49490, + "asOrg": "Test AS Org", + "dataTransferred": { + "bytes": 1000 + }, + "tunnelTime": { + "seconds": 1000 + } + } + ] + }, "accessKeys": [ { "accessKeyId": 0, @@ -152,9 +200,9 @@ describe('PrometheusManagerMetrics', () => { }, "connection": { "lastConnected": 1738959398, - "lastTrafficSeen": 1738959398, - "peakDevices": { - "count": 4, + "lastTrafficSeen": 1739284734, + "peakDeviceCount": { + "data": 4, "timestamp": 1738959398 } } @@ -168,6 +216,19 @@ describe('PrometheusManagerMetrics', () => { const managerMetrics = new PrometheusManagerMetrics( new QueryMapPrometheusClient( { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { + resultType: 'vector', + result: [ + { + metric: { + location: 'US', + asn: '49490', + asorg: 'Test AS Org', + }, + value: [1739284734, '1234'], + }, + ], + }, 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[0s])) by (location, asn, asorg)': { resultType: 'vector', @@ -218,6 +279,22 @@ describe('PrometheusManagerMetrics', () => { }, }, { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { + resultType: 'matrix', + result: [ + { + metric: { + location: 'US', + asn: '49490', + asorg: 'Test AS Org', + }, + values: [ + [1738959398, '5678'], + [1739284734, '1234'], + ], + }, + ], + }, 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[300s])) by (access_key)': { resultType: 'matrix', result: [ @@ -253,30 +330,49 @@ describe('PrometheusManagerMetrics', () => { const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0}); expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{ - "server": [ - { - "location": "CA", - "asn": null, - "asOrg": null, - "dataTransferred": { - "bytes": 0 + "server": { + "tunnelTime": { + "seconds": 1000 + }, + "dataTransferred": { + "total": { + "bytes": 1000 }, - "tunnelTime": { - "seconds": 1000 + "current": { + "bytes": 1234 + }, + "peak": { + "data": { + "bytes": 5678 + }, + "timestamp": 1738959398 } }, - { - "location": "US", - "asn": 49490, - "asOrg": "Test AS Org", - "dataTransferred": { - "bytes": 1000 + "locations": [ + { + "location": "CA", + "asn": null, + "asOrg": null, + "dataTransferred": { + "bytes": 0 + }, + "tunnelTime": { + "seconds": 1000 + } }, - "tunnelTime": { - "seconds": 0 + { + "location": "US", + "asn": 49490, + "asOrg": "Test AS Org", + "dataTransferred": { + "bytes": 1000 + }, + "tunnelTime": { + "seconds": 0 + } } - } - ], + ] + }, "accessKeys": [ { "accessKeyId": 1, @@ -289,8 +385,8 @@ describe('PrometheusManagerMetrics', () => { "connection": { "lastConnected": null, "lastTrafficSeen": null, - "peakDevices": { - "count": 0, + "peakDeviceCount": { + "data": 0, "timestamp": null } } @@ -306,8 +402,8 @@ describe('PrometheusManagerMetrics', () => { "connection": { "lastConnected": 1738959398, "lastTrafficSeen": 1738959398, - "peakDevices": { - "count": 4, + "peakDeviceCount": { + "data": 4, "timestamp": 1738959398 } } diff --git a/src/shadowbox/server/manager_metrics.ts b/src/shadowbox/server/manager_metrics.ts index 1afeab175..1f372fdde 100644 --- a/src/shadowbox/server/manager_metrics.ts +++ b/src/shadowbox/server/manager_metrics.ts @@ -29,18 +29,30 @@ interface Data { bytes: number; } -interface PeakDevices { - count: number; +interface TimedData { + data: T; timestamp: number | null; } interface ConnectionStats { lastConnected: number | null; lastTrafficSeen: number | null; - peakDevices: PeakDevices; + peakDeviceCount: TimedData; +} + +interface BandwidthStats { + total: Data; + current: Data; + peak: TimedData; } interface ServerMetricsServerEntry { + tunnelTime: Duration; + dataTransferred: BandwidthStats; + locations: ServerMetricsLocationEntry[]; +} + +interface ServerMetricsLocationEntry { location: string; asn: number | null; asOrg: string | null; @@ -56,7 +68,7 @@ interface ServerMetricsAccessKeyEntry { } interface ServerMetrics { - server: ServerMetricsServerEntry[]; + server: ServerMetricsServerEntry; accessKeys: ServerMetricsAccessKeyEntry[]; } @@ -88,7 +100,7 @@ export class PrometheusManagerMetrics implements ManagerMetrics { } async getServerMetrics(timeframe: Duration): Promise { - const now = new Date().getTime(); + const now = new Date().getTime() / 1000; // We need to calculate consistent start and end times for Prometheus range // queries. Rounding the end time *up* to the nearest multiple of the step // prevents time "drift" between queries, which is crucial for reliable step @@ -97,11 +109,12 @@ export class PrometheusManagerMetrics implements ManagerMetrics { // windows are queried each time, leading to more stable and predictable // results. const end = - Math.ceil(now / (PROMETHEUS_RANGE_QUERY_STEP_SECONDS * 1000)) * - PROMETHEUS_RANGE_QUERY_STEP_SECONDS; + Math.ceil(now / PROMETHEUS_RANGE_QUERY_STEP_SECONDS) * PROMETHEUS_RANGE_QUERY_STEP_SECONDS; const start = end - timeframe.seconds; const [ + totalDataTransferred, + totalDataTransferredRange, dataTransferredByLocation, tunnelTimeByLocation, dataTransferredByAccessKey, @@ -109,6 +122,15 @@ export class PrometheusManagerMetrics implements ManagerMetrics { dataTransferredByAccessKeyRange, tunnelTimeByAccessKeyRange, ] = await Promise.all([ + this.prometheusClient.query( + `sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))` + ), + this.prometheusClient.queryRange( + `sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))`, + start, + end, + `${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s` + ), this.prometheusClient.query( `sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[${timeframe.seconds}s])) by (location, asn, asorg)` ), @@ -135,28 +157,56 @@ export class PrometheusManagerMetrics implements ManagerMetrics { ), ]); - const serverMap = new Map(); - for (const result of tunnelTimeByLocation.result) { - const entry = getServerMetricsServerEntry(serverMap, result.metric); - entry.tunnelTime.seconds = result.value ? parseFloat(result.value[1]) : 0; + const serverMetrics: ServerMetricsServerEntry = { + tunnelTime: {seconds: 0}, + dataTransferred: { + total: {bytes: 0}, + current: {bytes: 0}, + peak: {data: {bytes: 0}, timestamp: null}, + }, + locations: [], + }; + for (const result of totalDataTransferred.result) { + const bytes = result.value ? parseFloat(result.value[1]) : 0; + serverMetrics.dataTransferred.current.bytes = bytes; + break; // There should only be one result. + } + for (const result of totalDataTransferredRange.result) { + const peakDataTransferred = findPeak(result.values ?? []); + if (peakDataTransferred !== null) { + const peakValue = parseFloat(peakDataTransferred[1]); + if (peakValue > 0) { + serverMetrics.dataTransferred.peak.data.bytes = peakValue; + serverMetrics.dataTransferred.peak.timestamp = Math.min(now, peakDataTransferred[0]); + } + } + break; // There should only be one result. } + const locationMap = new Map(); + for (const result of tunnelTimeByLocation.result) { + const entry = getServerMetricsLocationEntry(locationMap, result.metric); + const tunnelTime = result.value ? parseFloat(result.value[1]) : 0; + entry.tunnelTime.seconds = tunnelTime; + serverMetrics.tunnelTime.seconds += tunnelTime; + } for (const result of dataTransferredByLocation.result) { - const entry = getServerMetricsServerEntry(serverMap, result.metric); - entry.dataTransferred.bytes = result.value ? parseFloat(result.value[1]) : 0; + const entry = getServerMetricsLocationEntry(locationMap, result.metric); + const bytes = result.value ? parseFloat(result.value[1]) : 0; + entry.dataTransferred.bytes = bytes; + serverMetrics.dataTransferred.total.bytes += bytes; } + serverMetrics.locations = Array.from(locationMap.values()); const accessKeyMap = new Map(); for (const result of tunnelTimeByAccessKey.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); entry.tunnelTime.seconds = result.value ? parseFloat(result.value[1]) : 0; } - for (const result of dataTransferredByAccessKey.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); entry.dataTransferred.bytes = result.value ? parseFloat(result.value[1]) : 0; } - for (const result of tunnelTimeByAccessKeyRange.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); const lastConnected = findLastNonZero(result.values ?? []); @@ -166,12 +216,11 @@ export class PrometheusManagerMetrics implements ManagerMetrics { const peakValue = parseFloat(peakTunnelTimeSec[1]); if (peakValue > 0) { const peakTunnelTimeOverTime = peakValue / PROMETHEUS_RANGE_QUERY_STEP_SECONDS; - entry.connection.peakDevices.count = Math.ceil(peakTunnelTimeOverTime); - entry.connection.peakDevices.timestamp = Math.min(now, peakTunnelTimeSec[0]); + entry.connection.peakDeviceCount.data = Math.ceil(peakTunnelTimeOverTime); + entry.connection.peakDeviceCount.timestamp = Math.min(now, peakTunnelTimeSec[0]); } } } - for (const result of dataTransferredByAccessKeyRange.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); const lastTrafficSeen = findLastNonZero(result.values ?? []); @@ -179,16 +228,16 @@ export class PrometheusManagerMetrics implements ManagerMetrics { } return { - server: Array.from(serverMap.values()), + server: serverMetrics, accessKeys: Array.from(accessKeyMap.values()), }; } } -function getServerMetricsServerEntry( - map: Map, +function getServerMetricsLocationEntry( + map: Map, metric: PrometheusMetric -): ServerMetricsServerEntry { +): ServerMetricsLocationEntry { const {location, asn, asorg} = metric; const key = `${location},${asn},${asorg}`; let entry = map.get(key); @@ -219,8 +268,8 @@ function getServerMetricsAccessKeyEntry( connection: { lastConnected: null, lastTrafficSeen: null, - peakDevices: { - count: 0, + peakDeviceCount: { + data: 0, timestamp: null, }, },