diff --git a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt index 504b464eb1f5..ec2c5c3e0403 100644 --- a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt +++ b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt @@ -16,8 +16,8 @@ --influx_measurement=go_batch_cogbk_1 --influx_namespace=flink ---input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}'' ---co_input_options=''{\"num_records\":1000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' --iterations=1 --parallelism=5 --endpoint=localhost:8099 diff --git a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt index 7eb9a3c80534..7698c7aa7c75 100644 --- a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt +++ b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_10KB.txt @@ -16,8 +16,8 @@ --influx_measurement=go_batch_cogbk_2 --influx_namespace=flink ---input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' ---co_input_options=''{\"num_records\":1000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":10,\"hot_key_fraction\":1}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":100000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' --iterations=4 --parallelism=5 --endpoint=localhost:8099 diff --git a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt index ccf5ae7cbf28..c6b1f5fcc331 100644 --- a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt +++ b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_Reiteration_2MB.txt @@ -16,8 +16,8 @@ --influx_measurement=go_batch_cogbk_3 --influx_namespace=flink ---input_options=''{\"num_records\":100,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":100,\"hot_key_fraction\":1}'' ---co_input_options=''{\"num_records\":10,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":10,\"hot_key_fraction\":1}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' +--co_input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}'' --iterations=4 --parallelism=5 --endpoint=localhost:8099 diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt index f02e6984c81f..a188f8c09787 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_100kb.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=1 --parallelism=5 ---input_options=''{\"num_records\":50,\"key_size\":10000,\"value_size\":90000}'' +--input_options=''{\"num_records\":1000,\"key_size\":10000,\"value_size\":90000}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt index 0042a9b80f38..4378d56a8f8c 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_4.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=4 --parallelism=16 ---input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90}'' +--input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt index fb14c2da58de..43292d577170 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Fanout_8.txt @@ -19,7 +19,7 @@ --iterations=1 --fanout=8 --parallelism=16 ---input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90}'' +--input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt index ee220853c60c..c4d33c21482a 100644 --- a/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt +++ b/.github/workflows/load-tests-pipeline-options/go_GBK_Flink_Batch_Reiteration_10KB.txt @@ -19,7 +19,7 @@ --iterations=4 --fanout=1 --parallelism=5 ---input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":100,\"hot_key_fraction\":1}'' +--input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200,\"hot_key_fraction\":1}'' --endpoint=localhost:8099 --environment_type=DOCKER --environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest diff --git a/.test-infra/dataproc/flink_cluster.sh b/.test-infra/dataproc/flink_cluster.sh index e146aa4b6ce1..17180ced89da 100755 --- a/.test-infra/dataproc/flink_cluster.sh +++ b/.test-infra/dataproc/flink_cluster.sh @@ -183,6 +183,12 @@ function create() { start_tunnel } +# Connects to an existing Flink cluster and opens an SSH tunnel. +function tunnel() { + get_leader + start_tunnel +} + # Resizes an active Flink cluster. function resize() { if [[ -z "$FLINK_NUM_WORKERS" ]]; then