From 09b33ec1e9ba44606cfda6547c743b7c48307e0a Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Fri, 24 Jan 2025 18:08:49 +0400 Subject: [PATCH] Fix LoadTests Go GBK Flink Batch job (#33754) --- .github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml | 6 +++++- .../load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt | 2 +- .../go_GBK_Flink_Batch_100kb.txt | 2 +- .../load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt | 2 +- .../go_GBK_Flink_Batch_Fanout_4.txt | 2 +- .../go_GBK_Flink_Batch_Fanout_8.txt | 2 +- .../go_GBK_Flink_Batch_Reiteration_10KB.txt | 2 +- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml index 9e1a093b1bca..a7790105f3e9 100644 --- a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml +++ b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml @@ -52,7 +52,7 @@ env: GCS_BUCKET: gs://beam-flink-cluster FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar - FLINK_TASKMANAGER_SLOTS: 1 + FLINK_TASKMANAGER_SLOTS: 5 DETACHED_MODE: true HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest @@ -98,6 +98,8 @@ jobs: - name: Start Flink with parallelism 5 env: FLINK_NUM_WORKERS: 5 + HIGH_MEM_MACHINE: n1-highmem-16 + HIGH_MEM_FLINK_PROPS: flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=12g,flink:taskmanager.memory.jvm-overhead.max=4g,flink:jobmanager.memory.process.size=6g,flink:jobmanager.memory.jvm-overhead.max= 2g,flink:jobmanager.memory.flink.size=4g run: | cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create # The env variables are created and populated in the test-arguments-action as "_test_arguments_" @@ -140,6 +142,8 @@ jobs: - name: Restart Flink with parallelism 16 env: FLINK_NUM_WORKERS: 16 + HIGH_MEM_MACHINE: n1-highmem-16 + HIGH_MEM_FLINK_PROPS: flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=12g,flink:taskmanager.memory.jvm-overhead.max=4g,flink:jobmanager.memory.process.size=6g,flink:jobmanager.memory.jvm-overhead.max= 2g,flink:jobmanager.memory.flink.size=4g run: | cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh restart - name: run GBK Flink Batch Go Load Test 4 (fanout 4) diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt index 78f99fb4e9cc..d5c1d865e18d 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100b.txt @@ -16,7 +16,7 @@ --influx_namespace=flink --influx_measurement=go_batch_gbk_2 ---input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90}'' --iterations=1 --fanout=1 --parallelism=5 diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt index 5772a27ca184..a188f8c09787 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=1 --parallelism=5 ---input_options=''{\"num_records\":20000,\"key_size\":10000,\"value_size\":90000}'' +--input_options=''{\"num_records\":1000,\"key_size\":10000,\"value_size\":90000}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt index e826fbe10dae..0bb35b26436d 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_10b.txt @@ -16,7 +16,7 @@ --influx_namespace=flink --influx_measurement=go_batch_gbk_1 ---input_options=''{\"num_records\":200000000,\"key_size\":1,\"value_size\":9}'' +--input_options=''{\"num_records\":5000000,\"key_size\":1,\"value_size\":9}'' --iterations=1 --fanout=1 --parallelism=5 diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt index a5f6b0934521..4378d56a8f8c 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=4 --parallelism=16 ---input_options=''{\"num_records\":5000000,\"key_size\":10,\"value_size\":90}'' +--input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt index b261ff58c5bf..43292d577170 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=8 --parallelism=16 ---input_options=''{\"num_records\":2500000,\"key_size\":10,\"value_size\":90}'' +--input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt index 7e8d1eaa60d9..c4d33c21482a 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt @@ -19,7 +19,7 @@ --iterations=4 --fanout=1 --parallelism=5 ---input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200,\"hot_key_fraction\":1}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200,\"hot_key_fraction\":1}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest