From 3f9db01279ec8a7fd2c75a1c24d2c41869c81204 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 16 Oct 2024 10:42:02 -0400 Subject: [PATCH 1/9] feat(server): add ASN metric to opt-in server usage report --- src/shadowbox/server/shared_metrics.spec.ts | 45 +++++++++++++++++---- src/shadowbox/server/shared_metrics.ts | 41 +++++++++++-------- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 868a7330c..2cb6a2bc1 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -20,7 +20,7 @@ import {AccessKeyConfigJson} from './server_access_key'; import {ServerConfigJson} from './server_config'; import { - CountryUsage, + LocationUsage, DailyFeatureMetricsReportJson, HourlyServerMetricsReportJson, MetricsCollectorClient, @@ -78,7 +78,7 @@ describe('OutlineSharedMetricsPublisher', () => { ); publisher.startSharing(); - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'BB', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -102,7 +102,7 @@ describe('OutlineSharedMetricsPublisher', () => { }); startTime = clock.nowMs; - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'EE', inboundBytes: 44}, {country: 'FF', inboundBytes: 55}, ]; @@ -121,6 +121,35 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.stopSharing(); }); + + it('reports ASN metrics correctly', async () => { + const clock = new ManualClock(); + const serverConfig = new InMemoryConfig({serverId: 'server-id'}); + const usageMetrics = new ManualUsageMetrics(); + const metricsCollector = new FakeMetricsCollector(); + const publisher = new OutlineSharedMetricsPublisher( + clock, + serverConfig, + null, + usageMetrics, + metricsCollector + ); + publisher.startSharing(); + + usageMetrics.locationUsage = [ + {country: 'DD', asn: 999, inboundBytes: 44}, + {country: 'EE', inboundBytes: 55}, + ]; + clock.nowMs += 60 * 60 * 1000; + await clock.runCallbacks(); + + expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ + {bytesTransferred: 44, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, countries: ['EE']}, + ]); + publisher.stopSharing(); + }); + it('ignores sanctioned countries', async () => { const clock = new ManualClock(); const startTime = clock.nowMs; @@ -136,7 +165,7 @@ describe('OutlineSharedMetricsPublisher', () => { ); publisher.startSharing(); - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'SY', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -257,13 +286,13 @@ class FakeMetricsCollector implements MetricsCollectorClient { } class ManualUsageMetrics implements UsageMetrics { - countryUsage = [] as CountryUsage[]; + locationUsage = [] as LocationUsage[]; - getCountryUsage(): Promise { - return Promise.resolve(this.countryUsage); + getLocationUsage(): Promise { + return Promise.resolve(this.locationUsage); } reset() { - this.countryUsage = [] as CountryUsage[]; + this.locationUsage = [] as LocationUsage[]; } } diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index f1ab67a3a..5e7f8c4fd 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -26,8 +26,9 @@ const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); -export interface CountryUsage { +export interface LocationUsage { country: string; + asn?: number; inboundBytes: number; } @@ -44,6 +45,7 @@ export interface HourlyServerMetricsReportJson { // Field renames will break backwards-compatibility. export interface HourlyUserMetricsReportJson { countries: string[]; + asn?: number; bytesTransferred: number; } @@ -70,7 +72,7 @@ export interface SharedMetricsPublisher { } export interface UsageMetrics { - getCountryUsage(): Promise; + getLocationUsage(): Promise; reset(); } @@ -80,17 +82,18 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} - async getCountryUsage(): Promise { + async getLocationUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); // We measure the traffic to and from the target, since that's what we are protecting. const result = await this.prometheusClient.query( - `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|pt|p; } -export class RestMetricsCollectorClient { +export class RestMetricsCollectorClient implements MetricsCollectorClient { constructor(private serviceUrl: string) {} collectServerUsageMetrics(reportJson: HourlyServerMetricsReportJson): Promise { @@ -163,7 +166,7 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { return; } try { - await this.reportServerUsageMetrics(await usageMetrics.getCountryUsage()); + await this.reportServerUsageMetrics(await usageMetrics.getLocationUsage()); usageMetrics.reset(); } catch (err) { logging.error(`Failed to report server usage metrics: ${err}`); @@ -197,24 +200,28 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { return this.serverConfig.data().metricsEnabled || false; } - private async reportServerUsageMetrics(countryUsageMetrics: CountryUsage[]): Promise { + private async reportServerUsageMetrics(locationUsageMetrics: LocationUsage[]): Promise { const reportEndTimestampMs = this.clock.now(); - const userReports = [] as HourlyUserMetricsReportJson[]; - for (const countryUsage of countryUsageMetrics) { - if (countryUsage.inboundBytes === 0) { + const userReports: HourlyUserMetricsReportJson[] = []; + for (const locationUsage of locationUsageMetrics) { + if (locationUsage.inboundBytes === 0) { continue; } - if (isSanctionedCountry(countryUsage.country)) { + if (isSanctionedCountry(locationUsage.country)) { continue; } // Make sure to always set a country, which is required by the metrics server validation. // It's used to differentiate the row from the legacy key usage rows. - const country = countryUsage.country || 'ZZ'; - userReports.push({ - bytesTransferred: countryUsage.inboundBytes, + const country = locationUsage.country || 'ZZ'; + const report: HourlyUserMetricsReportJson = { + bytesTransferred: locationUsage.inboundBytes, countries: [country], - }); + }; + if (locationUsage.asn) { + report.asn = locationUsage.asn; + } + userReports.push(report); } const report = { serverId: this.serverConfig.data().serverId, From cc3ffe3a8825b0ca72636067eb9e0f1b187850bc Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 16 Oct 2024 11:02:23 -0400 Subject: [PATCH 2/9] feat(server): add tunnel time metric to opt-in server usage report --- .../infrastructure/prometheus_scraper.ts | 8 +- src/shadowbox/server/mocks/mocks.ts | 2 +- src/shadowbox/server/shared_metrics.spec.ts | 54 ++++++------- src/shadowbox/server/shared_metrics.ts | 79 ++++++++++++++++--- 4 files changed, 100 insertions(+), 43 deletions(-) diff --git a/src/shadowbox/infrastructure/prometheus_scraper.ts b/src/shadowbox/infrastructure/prometheus_scraper.ts index aee0239a6..c461cd619 100644 --- a/src/shadowbox/infrastructure/prometheus_scraper.ts +++ b/src/shadowbox/infrastructure/prometheus_scraper.ts @@ -21,10 +21,14 @@ import * as path from 'path'; import * as logging from '../infrastructure/logging'; +export interface QueryResultMetric { + [labelValue: string]: string; +} + export interface QueryResultData { resultType: 'matrix' | 'vector' | 'scalar' | 'string'; result: Array<{ - metric: {[labelValue: string]: string}; + metric: QueryResultMetric; value: [number, string]; }>; } @@ -101,7 +105,7 @@ async function spawnPrometheusSubprocess( prometheusEndpoint: string ): Promise { logging.info('======== Starting Prometheus ========'); - logging.info(`${binaryFilename} ${processArgs.map(a => `"${a}"`).join(' ')}`); + logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`); const runProcess = child_process.spawn(binaryFilename, processArgs); runProcess.on('error', (error) => { logging.error(`Error spawning Prometheus: ${error}`); diff --git a/src/shadowbox/server/mocks/mocks.ts b/src/shadowbox/server/mocks/mocks.ts index 248cd55b0..58f6a6458 100644 --- a/src/shadowbox/server/mocks/mocks.ts +++ b/src/shadowbox/server/mocks/mocks.ts @@ -59,7 +59,7 @@ export class FakePrometheusClient extends PrometheusClient { const bytesTransferred = this.bytesTransferredById[accessKeyId] || 0; queryResultData.result.push({ metric: {access_key: accessKeyId}, - value: [bytesTransferred, `${bytesTransferred}`], + value: [Date.now() / 1000, `${bytesTransferred}`], }); } return queryResultData; diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 2cb6a2bc1..1e750711e 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -79,11 +79,11 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.locationUsage = [ - {country: 'AA', inboundBytes: 11}, - {country: 'BB', inboundBytes: 11}, - {country: 'CC', inboundBytes: 22}, - {country: 'AA', inboundBytes: 33}, - {country: 'DD', inboundBytes: 33}, + {country: 'AA', inboundBytes: 11, tunnelTimeSec: 99}, + {country: 'BB', inboundBytes: 11, tunnelTimeSec: 88}, + {country: 'CC', inboundBytes: 22, tunnelTimeSec: 77}, + {country: 'AA', inboundBytes: 33, tunnelTimeSec: 66}, + {country: 'DD', inboundBytes: 33, tunnelTimeSec: 55}, ]; clock.nowMs += 60 * 60 * 1000; @@ -93,18 +93,18 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 11, countries: ['AA']}, - {bytesTransferred: 11, countries: ['BB']}, - {bytesTransferred: 22, countries: ['CC']}, - {bytesTransferred: 33, countries: ['AA']}, - {bytesTransferred: 33, countries: ['DD']}, + {bytesTransferred: 11, countries: ['AA'], tunnelTimeSec: 99}, + {bytesTransferred: 11, countries: ['BB'], tunnelTimeSec: 88}, + {bytesTransferred: 22, countries: ['CC'], tunnelTimeSec: 77}, + {bytesTransferred: 33, countries: ['AA'], tunnelTimeSec: 66}, + {bytesTransferred: 33, countries: ['DD'], tunnelTimeSec: 55}, ], }); startTime = clock.nowMs; usageMetrics.locationUsage = [ - {country: 'EE', inboundBytes: 44}, - {country: 'FF', inboundBytes: 55}, + {country: 'EE', inboundBytes: 44, tunnelTimeSec: 11}, + {country: 'FF', inboundBytes: 55, tunnelTimeSec: 22}, ]; clock.nowMs += 60 * 60 * 1000; @@ -114,8 +114,8 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 44, countries: ['EE']}, - {bytesTransferred: 55, countries: ['FF']}, + {bytesTransferred: 44, countries: ['EE'], tunnelTimeSec: 11}, + {bytesTransferred: 55, countries: ['FF'], tunnelTimeSec: 22}, ], }); @@ -137,15 +137,15 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.locationUsage = [ - {country: 'DD', asn: 999, inboundBytes: 44}, - {country: 'EE', inboundBytes: 55}, + {country: 'DD', inboundBytes: 44, tunnelTimeSec: 11, asn: 999}, + {country: 'EE', inboundBytes: 55, tunnelTimeSec: 22}, ]; clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ - {bytesTransferred: 44, countries: ['DD'], asn: 999}, - {bytesTransferred: 55, countries: ['EE']}, + {bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, tunnelTimeSec: 22, countries: ['EE']}, ]); publisher.stopSharing(); }); @@ -166,11 +166,11 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.startSharing(); usageMetrics.locationUsage = [ - {country: 'AA', inboundBytes: 11}, - {country: 'SY', inboundBytes: 11}, - {country: 'CC', inboundBytes: 22}, - {country: 'AA', inboundBytes: 33}, - {country: 'DD', inboundBytes: 33}, + {country: 'AA', tunnelTimeSec: 99, inboundBytes: 11}, + {country: 'SY', tunnelTimeSec: 88, inboundBytes: 11}, + {country: 'CC', tunnelTimeSec: 77, inboundBytes: 22}, + {country: 'AA', tunnelTimeSec: 66, inboundBytes: 33}, + {country: 'DD', tunnelTimeSec: 55, inboundBytes: 33}, ]; clock.nowMs += 60 * 60 * 1000; @@ -180,10 +180,10 @@ describe('OutlineSharedMetricsPublisher', () => { startUtcMs: startTime, endUtcMs: clock.nowMs, userReports: [ - {bytesTransferred: 11, countries: ['AA']}, - {bytesTransferred: 22, countries: ['CC']}, - {bytesTransferred: 33, countries: ['AA']}, - {bytesTransferred: 33, countries: ['DD']}, + {bytesTransferred: 11, tunnelTimeSec: 99, countries: ['AA']}, + {bytesTransferred: 22, tunnelTimeSec: 77, countries: ['CC']}, + {bytesTransferred: 33, tunnelTimeSec: 66, countries: ['AA']}, + {bytesTransferred: 33, tunnelTimeSec: 55, countries: ['DD']}, ], }); publisher.stopSharing(); diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 5e7f8c4fd..7534ebfab 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -16,7 +16,7 @@ import {Clock} from '../infrastructure/clock'; import * as follow_redirects from '../infrastructure/follow_redirects'; import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; -import {PrometheusClient} from '../infrastructure/prometheus_scraper'; +import {PrometheusClient, QueryResultMetric} from '../infrastructure/prometheus_scraper'; import * as version from './version'; import {AccessKeyConfigJson} from './server_access_key'; @@ -26,10 +26,21 @@ const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); +const PROMETHEUS_COUNTRY_LABEL = 'location'; +const PROMETHEUS_ASN_LABEL = 'asn'; + +type PrometheusQueryResult = { + [metricKey: string]: { + metric: QueryResultMetric; + value: number; + }; +}; + export interface LocationUsage { country: string; asn?: number; inboundBytes: number; + tunnelTimeSec: number; } // JSON format for the published report. @@ -47,6 +58,7 @@ export interface HourlyUserMetricsReportJson { countries: string[]; asn?: number; bytesTransferred: number; + tunnelTimeSec: number; } // JSON format for the feature metrics report. @@ -82,18 +94,58 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} + private async queryUsage(metric: string, deltaSecs: number): Promise { + const query = ` + sum(increase(${metric}[${deltaSecs}s])) + by (${PROMETHEUS_COUNTRY_LABEL}, ${PROMETHEUS_ASN_LABEL}) + `; + const queryResponse = await this.prometheusClient.query(query); + const result: PrometheusQueryResult = {}; + for (const entry of queryResponse.result) { + const serializedKey = JSON.stringify(entry.metric, Object.keys(entry.metric).sort()); + result[serializedKey] = { + metric: entry.metric, + value: Math.round(parseFloat(entry.value[1])), + }; + } + return result; + } + async getLocationUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); - // We measure the traffic to and from the target, since that's what we are protecting. - const result = await this.prometheusClient.query( - `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|pt|p Date: Wed, 16 Oct 2024 11:09:50 -0400 Subject: [PATCH 3/9] Rename variable. --- src/shadowbox/server/shared_metrics.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 7534ebfab..a02e0e4fa 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -94,9 +94,12 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} - private async queryUsage(metric: string, deltaSecs: number): Promise { + private async queryUsage( + timeSeriesSelector: string, + deltaSecs: number + ): Promise { const query = ` - sum(increase(${metric}[${deltaSecs}s])) + sum(increase(${timeSeriesSelector}[${deltaSecs}s])) by (${PROMETHEUS_COUNTRY_LABEL}, ${PROMETHEUS_ASN_LABEL}) `; const queryResponse = await this.prometheusClient.query(query); From 94c221982ec27f2104119624b5b2053de558429b Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 16 Oct 2024 15:11:44 -0400 Subject: [PATCH 4/9] Let Prometheus join the metrics. --- src/shadowbox/server/shared_metrics.ts | 95 +++++++++----------------- 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index a02e0e4fa..42bb54a9b 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -16,7 +16,7 @@ import {Clock} from '../infrastructure/clock'; import * as follow_redirects from '../infrastructure/follow_redirects'; import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; -import {PrometheusClient, QueryResultMetric} from '../infrastructure/prometheus_scraper'; +import {PrometheusClient} from '../infrastructure/prometheus_scraper'; import * as version from './version'; import {AccessKeyConfigJson} from './server_access_key'; @@ -26,16 +26,6 @@ const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); -const PROMETHEUS_COUNTRY_LABEL = 'location'; -const PROMETHEUS_ASN_LABEL = 'asn'; - -type PrometheusQueryResult = { - [metricKey: string]: { - metric: QueryResultMetric; - value: number; - }; -}; - export interface LocationUsage { country: string; asn?: number; @@ -94,63 +84,44 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} - private async queryUsage( - timeSeriesSelector: string, - deltaSecs: number - ): Promise { - const query = ` - sum(increase(${timeSeriesSelector}[${deltaSecs}s])) - by (${PROMETHEUS_COUNTRY_LABEL}, ${PROMETHEUS_ASN_LABEL}) - `; - const queryResponse = await this.prometheusClient.query(query); - const result: PrometheusQueryResult = {}; - for (const entry of queryResponse.result) { - const serializedKey = JSON.stringify(entry.metric, Object.keys(entry.metric).sort()); - result[serializedKey] = { - metric: entry.metric, - value: Math.round(parseFloat(entry.value[1])), - }; - } - return result; - } - async getLocationUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); - const [dataBytesResult, tunnelTimeResult] = await Promise.all([ - // We measure the traffic to and from the target, since that's what we are protecting. - this.queryUsage('shadowsocks_data_bytes_per_location{dir=~"p>t|pt|p Date: Thu, 17 Oct 2024 12:16:38 -0400 Subject: [PATCH 5/9] Use a Map. --- src/shadowbox/server/shared_metrics.ts | 38 +++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 42bb54a9b..c5576af19 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -101,27 +101,33 @@ export class PrometheusUsageMetrics implements UsageMetrics { ) `); - const usage: {[key: string]: LocationUsage} = {}; - for (const entry of queryResponse.result) { - const country = entry.metric['location'] || ''; - const asn = entry.metric['asn'] ? Number(entry.metric['asn']) : undefined; + const usage = new Map(); + for (const result of queryResponse.result) { + const country = result.metric['location'] || ''; + const asn = result.metric['asn'] ? Number(result.metric['asn']) : undefined; - // Create or update the entry for the country+ASN combination. + // Get or create an entry for the country+ASN combination. const key = `${country}-${asn}`; - usage[key] = { - country, - asn, - inboundBytes: usage[key]?.inboundBytes || 0, - tunnelTimeSec: usage[key]?.tunnelTimeSec || 0, - }; + let entry: LocationUsage; + if (usage.has(key)) { + entry = usage.get(key); + } else { + entry = { + country, + asn, + inboundBytes: 0, + tunnelTimeSec: 0, + }; + } - if (entry.metric['metric_type'] === 'inbound_bytes') { - usage[key].inboundBytes = Math.round(parseFloat(entry.value[1])); - } else if (entry.metric['metric_type'] === 'tunnel_time') { - usage[key].tunnelTimeSec = Math.round(parseFloat(entry.value[1])); + if (result.metric['metric_type'] === 'inbound_bytes') { + entry.inboundBytes = Math.round(parseFloat(result.value[1])); + } else if (result.metric['metric_type'] === 'tunnel_time') { + entry.tunnelTimeSec = Math.round(parseFloat(result.value[1])); } + usage.set(key, entry); } - return Object.values(usage); + return Array.from(usage.values()); } reset() { From 40df2a0c893a1676aec7684e9c06baa6f92ee816 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 17 Oct 2024 12:21:03 -0400 Subject: [PATCH 6/9] Revert changes to `prometheus_scraper.ts`. --- src/shadowbox/infrastructure/prometheus_scraper.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/shadowbox/infrastructure/prometheus_scraper.ts b/src/shadowbox/infrastructure/prometheus_scraper.ts index c461cd619..aee0239a6 100644 --- a/src/shadowbox/infrastructure/prometheus_scraper.ts +++ b/src/shadowbox/infrastructure/prometheus_scraper.ts @@ -21,14 +21,10 @@ import * as path from 'path'; import * as logging from '../infrastructure/logging'; -export interface QueryResultMetric { - [labelValue: string]: string; -} - export interface QueryResultData { resultType: 'matrix' | 'vector' | 'scalar' | 'string'; result: Array<{ - metric: QueryResultMetric; + metric: {[labelValue: string]: string}; value: [number, string]; }>; } @@ -105,7 +101,7 @@ async function spawnPrometheusSubprocess( prometheusEndpoint: string ): Promise { logging.info('======== Starting Prometheus ========'); - logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`); + logging.info(`${binaryFilename} ${processArgs.map(a => `"${a}"`).join(' ')}`); const runProcess = child_process.spawn(binaryFilename, processArgs); runProcess.on('error', (error) => { logging.error(`Error spawning Prometheus: ${error}`); From 24e23a758e458d7ca46bfc836ab4b16c5ac55003 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 17 Oct 2024 12:33:18 -0400 Subject: [PATCH 7/9] Rename `LocationUsage` to `ReportedUsage`. --- src/shadowbox/server/shared_metrics.spec.ts | 18 +++++++++--------- src/shadowbox/server/shared_metrics.ts | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 2cb6a2bc1..a3f910252 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -20,7 +20,7 @@ import {AccessKeyConfigJson} from './server_access_key'; import {ServerConfigJson} from './server_config'; import { - LocationUsage, + ReportedUsage, DailyFeatureMetricsReportJson, HourlyServerMetricsReportJson, MetricsCollectorClient, @@ -78,7 +78,7 @@ describe('OutlineSharedMetricsPublisher', () => { ); publisher.startSharing(); - usageMetrics.locationUsage = [ + usageMetrics.reportedUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'BB', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -102,7 +102,7 @@ describe('OutlineSharedMetricsPublisher', () => { }); startTime = clock.nowMs; - usageMetrics.locationUsage = [ + usageMetrics.reportedUsage = [ {country: 'EE', inboundBytes: 44}, {country: 'FF', inboundBytes: 55}, ]; @@ -136,7 +136,7 @@ describe('OutlineSharedMetricsPublisher', () => { ); publisher.startSharing(); - usageMetrics.locationUsage = [ + usageMetrics.reportedUsage = [ {country: 'DD', asn: 999, inboundBytes: 44}, {country: 'EE', inboundBytes: 55}, ]; @@ -165,7 +165,7 @@ describe('OutlineSharedMetricsPublisher', () => { ); publisher.startSharing(); - usageMetrics.locationUsage = [ + usageMetrics.reportedUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'SY', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -286,13 +286,13 @@ class FakeMetricsCollector implements MetricsCollectorClient { } class ManualUsageMetrics implements UsageMetrics { - locationUsage = [] as LocationUsage[]; + reportedUsage = [] as ReportedUsage[]; - getLocationUsage(): Promise { - return Promise.resolve(this.locationUsage); + getReportedUsage(): Promise { + return Promise.resolve(this.reportedUsage); } reset() { - this.locationUsage = [] as LocationUsage[]; + this.reportedUsage = [] as ReportedUsage[]; } } diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 5e7f8c4fd..c0076e35d 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -26,7 +26,7 @@ const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); -export interface LocationUsage { +export interface ReportedUsage { country: string; asn?: number; inboundBytes: number; @@ -72,7 +72,7 @@ export interface SharedMetricsPublisher { } export interface UsageMetrics { - getLocationUsage(): Promise; + getReportedUsage(): Promise; reset(); } @@ -82,13 +82,13 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} - async getLocationUsage(): Promise { + async getReportedUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); // We measure the traffic to and from the target, since that's what we are protecting. const result = await this.prometheusClient.query( `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p { + private async reportServerUsageMetrics(locationUsageMetrics: ReportedUsage[]): Promise { const reportEndTimestampMs = this.clock.now(); const userReports: HourlyUserMetricsReportJson[] = []; From 540b1e015b55537dbb35179ad360845b5637d5b7 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 17 Oct 2024 12:51:26 -0400 Subject: [PATCH 8/9] Add test cases for different ASN+country combinations. --- src/shadowbox/server/shared_metrics.spec.ts | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index a3f910252..5e21b1ace 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -150,6 +150,62 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.stopSharing(); }); + it('reports different ASNs in the same country correctly', async () => { + const clock = new ManualClock(); + const serverConfig = new InMemoryConfig({serverId: 'server-id'}); + const usageMetrics = new ManualUsageMetrics(); + const metricsCollector = new FakeMetricsCollector(); + const publisher = new OutlineSharedMetricsPublisher( + clock, + serverConfig, + null, + usageMetrics, + metricsCollector + ); + publisher.startSharing(); + + usageMetrics.reportedUsage = [ + {country: 'DD', asn: 999, inboundBytes: 44}, + {country: 'DD', asn: 888, inboundBytes: 55}, + ]; + clock.nowMs += 60 * 60 * 1000; + await clock.runCallbacks(); + + expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ + {bytesTransferred: 44, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, countries: ['DD'], asn: 888}, + ]); + publisher.stopSharing(); + }); + + it('reports the same ASNs across different countries correctly', async () => { + const clock = new ManualClock(); + const serverConfig = new InMemoryConfig({serverId: 'server-id'}); + const usageMetrics = new ManualUsageMetrics(); + const metricsCollector = new FakeMetricsCollector(); + const publisher = new OutlineSharedMetricsPublisher( + clock, + serverConfig, + null, + usageMetrics, + metricsCollector + ); + publisher.startSharing(); + + usageMetrics.reportedUsage = [ + {country: 'DD', asn: 999, inboundBytes: 44}, + {country: 'EE', asn: 999, inboundBytes: 66}, + ]; + clock.nowMs += 60 * 60 * 1000; + await clock.runCallbacks(); + + expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ + {bytesTransferred: 44, countries: ['DD'], asn: 999}, + {bytesTransferred: 66, countries: ['EE'], asn: 999}, + ]); + publisher.stopSharing(); + }); + it('ignores sanctioned countries', async () => { const clock = new ManualClock(); const startTime = clock.nowMs; From 749aca19860914340a2a3c056301aa31378412fe Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 18 Oct 2024 16:36:25 -0400 Subject: [PATCH 9/9] Use a different approach where we don't let Prometheus combine the series. --- src/shadowbox/server/shared_metrics.ts | 64 +++++++++++++------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 48134d4e9..8cb4b9492 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -16,7 +16,7 @@ import {Clock} from '../infrastructure/clock'; import * as follow_redirects from '../infrastructure/follow_redirects'; import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; -import {PrometheusClient} from '../infrastructure/prometheus_scraper'; +import {PrometheusClient, QueryResultData} from '../infrastructure/prometheus_scraper'; import * as version from './version'; import {AccessKeyConfigJson} from './server_access_key'; @@ -86,47 +86,45 @@ export class PrometheusUsageMetrics implements UsageMetrics { async getReportedUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); - // Return both data bytes and tunnel time information with a single - // Prometheus query, by using a custom "metric_type" label. - const queryResponse = await this.prometheusClient.query(` - label_replace( - sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p(); - for (const result of queryResponse.result) { - const country = result.metric['location'] || ''; - const asn = result.metric['asn'] ? Number(result.metric['asn']) : undefined; - - // Get or create an entry for the country+ASN combination. - const key = `${country}-${asn}`; - let entry: ReportedUsage; - if (usage.has(key)) { - entry = usage.get(key); - } else { - entry = { + const processResults = ( + data: QueryResultData, + setValue: (entry: ReportedUsage, value: string) => void + ) => { + for (const result of data.result) { + const country = result.metric['location'] || ''; + const asn = result.metric['asn'] ? Number(result.metric['asn']) : undefined; + const key = `${country}-${asn}`; + const entry = usage.get(key) || { country, asn, inboundBytes: 0, tunnelTimeSec: 0, }; + setValue(entry, result.value[1]); + if (!usage.has(key)) { + usage.set(key, entry); + } } + }; + + // Query and process inbound data bytes by country+ASN. + const dataBytesQueryResponse = await this.prometheusClient.query( + `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p { + entry.inboundBytes = Math.round(parseFloat(value)); + }); + + // Query and process tunneltime by country+ASN. + const tunnelTimeQueryResponse = await this.prometheusClient.query( + `sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeDeltaSecs}s])) by (location, asn)` + ); + processResults(tunnelTimeQueryResponse, (entry, value) => { + entry.tunnelTimeSec = Math.round(parseFloat(value)); + }); - if (result.metric['metric_type'] === 'inbound_bytes') { - entry.inboundBytes = Math.round(parseFloat(result.value[1])); - } else if (result.metric['metric_type'] === 'tunnel_time') { - entry.tunnelTimeSec = Math.round(parseFloat(result.value[1])); - } - usage.set(key, entry); - } return Array.from(usage.values()); }