diff --git a/agent/native/ext/ConfigManager.cpp b/agent/native/ext/ConfigManager.cpp index e1129d51d..60f4a2a4c 100644 --- a/agent/native/ext/ConfigManager.cpp +++ b/agent/native/ext/ConfigManager.cpp @@ -117,6 +117,7 @@ typedef struct DurationOptionAdditionalMetadata DurationOptionAdditionalMetadata struct SizeOptionAdditionalMetadata { SizeUnits defaultUnits = sizeUnits_byte; + Int64 minValidValueInBytes = INT64_MIN; }; typedef struct SizeOptionAdditionalMetadata SizeOptionAdditionalMetadata; @@ -427,7 +428,14 @@ static ResultCode parseSizeValue( const OptionMetadata* optMeta, String rawValue ResultCode parseResultCode = parseSize( stringToView( rawValue ) , optMeta->additionalData.sizeData.defaultUnits , /* out */ &parsedValue->u.sizeValue ); - if ( parseResultCode == resultSuccess ) parsedValue->type = parsedOptionValueType_size; + if ( parseResultCode == resultSuccess ) + { + if ( sizeToBytes( parsedValue->u.sizeValue ) < optMeta->additionalData.sizeData.minValidValueInBytes ) + { + return resultParsingFailed; + } + parsedValue->type = parsedOptionValueType_size; + } return parseResultCode; } @@ -667,7 +675,7 @@ static OptionMetadata buildDurationOptionMetadata( }; } -[[maybe_unused]] static OptionMetadata buildSizeOptionMetadata( +static OptionMetadata buildSizeOptionMetadata( String name , StringView iniName , bool isSecret @@ -676,6 +684,7 @@ static OptionMetadata buildDurationOptionMetadata( , SetConfigSnapshotFieldFunc setFieldFunc , GetConfigSnapshotFieldFunc getFieldFunc , SizeUnits defaultUnits + , Int64 minValidValueInBytes ) { return (OptionMetadata) @@ -692,7 +701,7 @@ static OptionMetadata buildDurationOptionMetadata( .setField = setFieldFunc, .getField = getFieldFunc, .parsedValueToZval = &parsedSizeValueToZval, - .additionalData = (OptionAdditionalMetadata){ .sizeData = (SizeOptionAdditionalMetadata){ .defaultUnits = defaultUnits } } + .additionalData = (OptionAdditionalMetadata){ .sizeData = (SizeOptionAdditionalMetadata){ .defaultUnits = defaultUnits, .minValidValueInBytes = minValidValueInBytes } } }; } @@ -827,6 +836,7 @@ ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( LogLevel, logLevelSyslog ) # ifdef PHP_WIN32 ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( LogLevel, logLevelWinSysDebug ) # endif +ELASTIC_APM_DEFINE_FIELD_ACCESS_FUNCS( sizeValue, maxSendQueueSize ) # if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 ) ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( MemoryTrackingLevel, memoryTrackingLevel ) # endif @@ -882,6 +892,9 @@ ELASTIC_APM_DEFINE_FIELD_ACCESS_FUNCS( stringValue, debugDiagnosticsFile ) #define ELASTIC_APM_INIT_DURATION_METADATA( fieldName, optName, defaultValue, defaultUnits, isNegativeValid ) \ ELASTIC_APM_INIT_METADATA_EX( buildDurationOptionMetadata, fieldName, optName, /* isSecret */ false, /* isDynamic */ false, defaultValue, defaultUnits, isNegativeValid ) +#define ELASTIC_APM_INIT_SIZE_METADATA( fieldName, optName, defaultValue, defaultUnits, minValidValueInBytes ) \ + ELASTIC_APM_INIT_METADATA_EX( buildSizeOptionMetadata, fieldName, optName, /* isSecret */ false, /* isDynamic */ false, defaultValue, defaultUnits, minValidValueInBytes ) + #define ELASTIC_APM_INIT_SECRET_METADATA( buildFunc, fieldName, optName, defaultValue ) \ ELASTIC_APM_INIT_METADATA_EX( buildFunc, fieldName, optName, /* isSecret */ true, /* isDynamic */ false, defaultValue ) @@ -1115,6 +1128,13 @@ static void initOptionsMetadata( OptionMetadata* optsMeta ) ELASTIC_APM_CFG_OPT_NAME_LOG_LEVEL_WIN_SYS_DEBUG ); #endif + ELASTIC_APM_INIT_SIZE_METADATA( + maxSendQueueSize, + ELASTIC_APM_CFG_OPT_NAME_MAX_SEND_QUEUE_SIZE, + /* defaultValue: */ makeSize( 10, sizeUnits_mebibyte ), + /* defaultUnits: */ sizeUnits_mebibyte, + /* minValidValueInBytes: */ 1 ); + #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 ) ELASTIC_APM_ENUM_INIT_METADATA( /* fieldName: */ memoryTrackingLevel, diff --git a/agent/native/ext/ConfigManager.h b/agent/native/ext/ConfigManager.h index f47f867f1..553e351af 100644 --- a/agent/native/ext/ConfigManager.h +++ b/agent/native/ext/ConfigManager.h @@ -96,6 +96,7 @@ enum OptionId #ifdef PHP_WIN32 optionId_logLevelWinSysDebug, #endif + optionId_maxSendQueueSize, #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 ) optionId_memoryTrackingLevel, #endif @@ -303,6 +304,8 @@ const ConfigSnapshot* getGlobalCurrentConfigSnapshot(); #define ELASTIC_APM_CFG_OPT_NAME_LOG_LEVEL_WIN_SYS_DEBUG "log_level_win_sys_debug" # endif +#define ELASTIC_APM_CFG_OPT_NAME_MAX_SEND_QUEUE_SIZE "max_send_queue_size" + /** * Internal configuration option (not included in public documentation) */ diff --git a/agent/native/ext/ConfigSnapshot.h b/agent/native/ext/ConfigSnapshot.h index 8c3397222..b49afb813 100644 --- a/agent/native/ext/ConfigSnapshot.h +++ b/agent/native/ext/ConfigSnapshot.h @@ -25,6 +25,7 @@ #include "LogLevel.h" #include "OptionalBool.h" #include "time_util.h" // Duration +#include "util.h" // Size #include "elastic_apm_assert_enabled.h" struct ConfigSnapshot @@ -68,6 +69,7 @@ struct ConfigSnapshot #ifdef PHP_WIN32 LogLevel logLevelWinSysDebug = logLevel_off; #endif + Size maxSendQueueSize; #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 ) MemoryTrackingLevel memoryTrackingLevel = memoryTrackingLevel_off; #endif diff --git a/agent/native/ext/backend_comm.cpp b/agent/native/ext/backend_comm.cpp index b29044f88..5dd81f510 100644 --- a/agent/native/ext/backend_comm.cpp +++ b/agent/native/ext/backend_comm.cpp @@ -660,8 +660,6 @@ static void freeDataToSendQueue( DataToSendQueue* dataQueue ) } } -#define ELASTIC_APM_MAX_QUEUE_SIZE_IN_BYTES (10 * 1024 * 1024) - struct BackgroundBackendComm { Mutex* mutex; @@ -669,6 +667,7 @@ struct BackgroundBackendComm Thread* thread; DataToSendQueue dataToSendQueue; size_t dataToSendTotalSize; + UInt64 maxQueueSizeInBytes; size_t nextEventsBatchId; bool shouldExit; TimeSpec shouldExitBy; @@ -1051,6 +1050,9 @@ ResultCode newBackgroundBackendComm( const ConfigSnapshot* config, BackgroundBac ResultCode resultCode; BackgroundBackendComm* backgroundBackendComm = NULL; + Int64 maxQueueSizeInBytes = sizeToBytes( config->maxSendQueueSize ); + + ELASTIC_APM_ASSERT( maxQueueSizeInBytes > 0, "maxQueueSizeInBytes: %" PRId64, maxQueueSizeInBytes ); ELASTIC_APM_MALLOC_INSTANCE_IF_FAILED_GOTO( BackgroundBackendComm, /* out */ backgroundBackendComm ); backgroundBackendComm->condVar = NULL; @@ -1058,6 +1060,7 @@ ResultCode newBackgroundBackendComm( const ConfigSnapshot* config, BackgroundBac backgroundBackendComm->thread = NULL; initDataToSendQueue( &( backgroundBackendComm->dataToSendQueue ) ); backgroundBackendComm->dataToSendTotalSize = 0; + backgroundBackendComm->maxQueueSizeInBytes = (UInt64) maxQueueSizeInBytes; backgroundBackendComm->nextEventsBatchId = 1; backgroundBackendComm->shouldExit = false; ELASTIC_APM_CALL_IF_FAILED_GOTO( newMutex( &( backgroundBackendComm->mutex ), /* dbgDesc */ "Background backend communications" ) ); @@ -1179,15 +1182,21 @@ ResultCode enqueueEventsToSendToApmServer( StringView userAgentHttpHeader, Strin bool shouldUnlockMutex = false; UInt64 id; BackgroundBackendComm* backgroundBackendComm = g_backgroundBackendComm; + UInt64 totalSizeAfterEnqueue; ELASTIC_APM_CALL_IF_FAILED_GOTO( lockMutex( backgroundBackendComm->mutex, &shouldUnlockMutex, __FUNCTION__ ) ); - if ( backgroundBackendComm->dataToSendTotalSize >= ELASTIC_APM_MAX_QUEUE_SIZE_IN_BYTES ) + totalSizeAfterEnqueue = (UInt64) backgroundBackendComm->dataToSendTotalSize + (UInt64) serializedEvents.length; + if ( totalSizeAfterEnqueue > backgroundBackendComm->maxQueueSizeInBytes ) { ELASTIC_APM_LOG_ERROR( - "Already queued events are above max queue size - dropping these events" + "Queueing these events would exceed max queue size - dropping these events" "; size of already queued events: %" PRIu64 - , (UInt64) backgroundBackendComm->dataToSendTotalSize ); + "; size of events to queue: %" PRIu64 + "; max queue size: %" PRIu64 + , (UInt64) backgroundBackendComm->dataToSendTotalSize + , (UInt64) serializedEvents.length + , backgroundBackendComm->maxQueueSizeInBytes ); ELASTIC_APM_SET_RESULT_CODE_AND_GOTO_FAILURE(); } diff --git a/agent/native/ext/elastic_apm.cpp b/agent/native/ext/elastic_apm.cpp index 0416de4d7..8efc06657 100644 --- a/agent/native/ext/elastic_apm.cpp +++ b/agent/native/ext/elastic_apm.cpp @@ -173,6 +173,7 @@ PHP_INI_BEGIN() #ifdef PHP_WIN32 ELASTIC_APM_INI_ENTRY( ELASTIC_APM_CFG_OPT_NAME_LOG_LEVEL_WIN_SYS_DEBUG ) #endif + ELASTIC_APM_INI_ENTRY( ELASTIC_APM_CFG_OPT_NAME_MAX_SEND_QUEUE_SIZE ) #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 ) ELASTIC_APM_INI_ENTRY( ELASTIC_APM_CFG_OPT_NAME_MEMORY_TRACKING_LEVEL ) #endif diff --git a/agent/native/ext/tests/config_defaults.phpt b/agent/native/ext/tests/config_defaults.phpt index d949cbaae..a400b89c1 100644 --- a/agent/native/ext/tests/config_defaults.phpt +++ b/agent/native/ext/tests/config_defaults.phpt @@ -8,6 +8,7 @@ ELASTIC_APM_LOG_LEVEL= ELASTIC_APM_LOG_LEVEL_FILE= ELASTIC_APM_LOG_LEVEL_SYSLOG= ELASTIC_APM_LOG_LEVEL_WIN_SYS_DEBUG= +ELASTIC_APM_MAX_SEND_QUEUE_SIZE= ELASTIC_APM_SECRET_TOKEN= ELASTIC_APM_SERVER_URL= ELASTIC_APM_SERVICE_NAME= @@ -78,6 +79,15 @@ if (elasticApmIsOsWindows()) { elasticApmAssertSame("elastic_apm_get_config_option_by_name('log_level_win_sys_debug')", elastic_apm_get_config_option_by_name('log_level_win_sys_debug'), ELASTIC_APM_LOG_LEVEL_NOT_SET); } +////////////////////////////////////////////// +/////////////// max_send_queue_size + +elasticApmAssertSame("getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE')", getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE'), false); + +elasticApmAssertEqual("ini_get('elastic_apm.max_send_queue_size')", ini_get('elastic_apm.max_send_queue_size'), false); + +elasticApmAssertSame("elastic_apm_get_config_option_by_name('max_send_queue_size')", elastic_apm_get_config_option_by_name('max_send_queue_size'), 10.0 * 1024 * 1024); + ////////////////////////////////////////////// /////////////// secret_token diff --git a/agent/native/ext/tests/config_ini_has_higher_precedence_than_env_vars.phpt b/agent/native/ext/tests/config_ini_has_higher_precedence_than_env_vars.phpt index c274be6e5..75c510676 100644 --- a/agent/native/ext/tests/config_ini_has_higher_precedence_than_env_vars.phpt +++ b/agent/native/ext/tests/config_ini_has_higher_precedence_than_env_vars.phpt @@ -4,8 +4,10 @@ Configuration in ini file has higher precedence than environment variables ELASTIC_APM_LOG_LEVEL_STDERR=CRITICAL ELASTIC_APM_LOG_FILE=log_file_from_env_vars.txt ELASTIC_APM_LOG_LEVEL_FILE=off +ELASTIC_APM_MAX_SEND_QUEUE_SIZE=123MB --INI-- elastic_apm.log_file=log_file_from_ini.txt +elastic_apm.max_send_queue_size=456MB elastic_apm.bootstrap_php_part_file=../../php/bootstrap_php_part.php --FILE-- --EXPECT-- diff --git a/agent/native/ext/tests/config_invalid_ini_fallback_default_not_env_var.phpt b/agent/native/ext/tests/config_invalid_ini_fallback_default_not_env_var.phpt index 8ad72e814..1a23658e8 100644 --- a/agent/native/ext/tests/config_invalid_ini_fallback_default_not_env_var.phpt +++ b/agent/native/ext/tests/config_invalid_ini_fallback_default_not_env_var.phpt @@ -3,9 +3,11 @@ When value in ini is invalid the fallback is the default and not environment var --ENV-- ELASTIC_APM_LOG_LEVEL_STDERR=CRITICAL ELASTIC_APM_ASSERT_LEVEL=O_n +ELASTIC_APM_MAX_SEND_QUEUE_SIZE=123MB ELASTIC_APM_MEMORY_TRACKING_LEVEL=ALL ELASTIC_APM_VERIFY_SERVER_CERT=false --INI-- +elastic_apm.max_send_queue_size=0 elastic_apm.memory_tracking_level=not a valid memory tracking level elastic_apm.verify_server_cert=not a valid bool elastic_apm.bootstrap_php_part_file=../../php/bootstrap_php_part.php @@ -18,6 +20,10 @@ require __DIR__ . '/../tests_util/tests_util.php'; elasticApmAssertSame("elastic_apm_get_config_option_by_name('assert_level')", elastic_apm_get_config_option_by_name('assert_level'), ELASTIC_APM_ASSERT_LEVEL_O_N); elasticApmAssertSame("getenv('ELASTIC_APM_ASSERT_LEVEL')", getenv('ELASTIC_APM_ASSERT_LEVEL'), 'O_n'); +// max_send_queue_size is set in ini but the value is invalid so it falls back on default (which is `10MB`) and not the value set by env vars (which is `123MB`) +elasticApmAssertSame("elastic_apm_get_config_option_by_name('max_send_queue_size')", elastic_apm_get_config_option_by_name('max_send_queue_size'), 10.0 * 1024 * 1024); +elasticApmAssertSame("getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE')", getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE'), '123MB'); + // memory_tracking_level is set in ini but the value is invalid so it falls back on default (which is `ELASTIC_APM_MEMORY_TRACKING_LEVEL_NOT_SET) and not the value set by env vars (which is ELASTIC_APM_MEMORY_TRACKING_LEVEL_ALL) elasticApmAssertSame("elastic_apm_get_config_option_by_name('memory_tracking_level')", elastic_apm_get_config_option_by_name('memory_tracking_level'), ELASTIC_APM_MEMORY_TRACKING_LEVEL_NOT_SET); elasticApmAssertSame("getenv('ELASTIC_APM_MEMORY_TRACKING_LEVEL')", getenv('ELASTIC_APM_MEMORY_TRACKING_LEVEL'), 'ALL'); diff --git a/agent/native/ext/tests/config_setting_to_invalid_values_using_env_vars.phpt b/agent/native/ext/tests/config_setting_to_invalid_values_using_env_vars.phpt index 78ee5e4bb..d1ec5842f 100644 --- a/agent/native/ext/tests/config_setting_to_invalid_values_using_env_vars.phpt +++ b/agent/native/ext/tests/config_setting_to_invalid_values_using_env_vars.phpt @@ -4,6 +4,7 @@ Setting configuration option to invalid value via environment variables ELASTIC_APM_LOG_LEVEL_STDERR=CRITICAL ELASTIC_APM_ENABLED=not_valid_boolean_value ELASTIC_APM_ASSERT_LEVEL=|:/:\:| +ELASTIC_APM_MAX_SEND_QUEUE_SIZE=0 ELASTIC_APM_SECRET_TOKEN=\|<>|/ ELASTIC_APM_SERVER_URL=<\/\/> ELASTIC_APM_SERVICE_NAME=/\><\/ @@ -30,6 +31,13 @@ elasticApmAssertSame("getenv('ELASTIC_APM_ASSERT_LEVEL')", getenv('ELASTIC_APM_A elasticApmAssertSame("elastic_apm_get_config_option_by_name('assert_level')", elastic_apm_get_config_option_by_name('assert_level'), ELASTIC_APM_ASSERT_LEVEL_NOT_SET); +////////////////////////////////////////////// +/////////////// max_send_queue_size + +elasticApmAssertSame("getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE')", getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE'), '0'); + +elasticApmAssertSame("elastic_apm_get_config_option_by_name('max_send_queue_size')", elastic_apm_get_config_option_by_name('max_send_queue_size'), 10.0 * 1024 * 1024); + ////////////////////////////////////////////// /////////////// secret_token diff --git a/agent/native/ext/tests/config_setting_to_non-defaults_using_env_vars.phpt b/agent/native/ext/tests/config_setting_to_non-defaults_using_env_vars.phpt index 1d5b3330c..b816447bf 100644 --- a/agent/native/ext/tests/config_setting_to_non-defaults_using_env_vars.phpt +++ b/agent/native/ext/tests/config_setting_to_non-defaults_using_env_vars.phpt @@ -8,6 +8,7 @@ ELASTIC_APM_LOG_LEVEL=CRITICAL ELASTIC_APM_LOG_LEVEL_FILE=TRACE ELASTIC_APM_LOG_LEVEL_SYSLOG=TRACE ELASTIC_APM_LOG_LEVEL_WIN_SYS_DEBUG=CRITICAL +ELASTIC_APM_MAX_SEND_QUEUE_SIZE=123MB ELASTIC_APM_SECRET_TOKEN=non-default_secret_token_123 ELASTIC_APM_SERVER_URL=https://non-default_server_url:4321/some/path ELASTIC_APM_SERVICE_NAME=Non-default Service Name @@ -66,6 +67,13 @@ if (elasticApmIsOsWindows()) { elasticApmAssertSame("elastic_apm_get_config_option_by_name('log_level_win_sys_debug')", elastic_apm_get_config_option_by_name('log_level_win_sys_debug'), ELASTIC_APM_LOG_LEVEL_CRITICAL); } +////////////////////////////////////////////// +/////////////// max_send_queue_size + +elasticApmAssertSame("getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE')", getenv('ELASTIC_APM_MAX_SEND_QUEUE_SIZE'), '123MB'); + +elasticApmAssertSame("elastic_apm_get_config_option_by_name('max_send_queue_size')", elastic_apm_get_config_option_by_name('max_send_queue_size'), 123.0 * 1024 * 1024); + ////////////////////////////////////////////// /////////////// secret_token diff --git a/agent/native/ext/tests/config_setting_to_non-defaults_using_ini.phpt b/agent/native/ext/tests/config_setting_to_non-defaults_using_ini.phpt index f7814d4e1..f16d2c4cb 100644 --- a/agent/native/ext/tests/config_setting_to_non-defaults_using_ini.phpt +++ b/agent/native/ext/tests/config_setting_to_non-defaults_using_ini.phpt @@ -9,6 +9,7 @@ elastic_apm.log_level=CRITICAL elastic_apm.log_level_file=TRACE elastic_apm.log_level_syslog=TRACE elastic_apm.log_level_win_sys_debug=CRITICAL +elastic_apm.max_send_queue_size=456MB elastic_apm.secret_token=non-default_secret_token_123 elastic_apm.server_url=https://non-default_server_url:4321/some/path elastic_apm.service_name=Non-default Service Name @@ -65,6 +66,13 @@ if (elasticApmIsOsWindows()) { elasticApmAssertSame("elastic_apm_get_config_option_by_name('log_level_win_sys_debug')", elastic_apm_get_config_option_by_name('log_level_win_sys_debug'), ELASTIC_APM_LOG_LEVEL_CRITICAL); } +////////////////////////////////////////////// +/////////////// max_send_queue_size + +elasticApmAssertSame("ini_get('elastic_apm.max_send_queue_size')", ini_get('elastic_apm.max_send_queue_size'), '456MB'); + +elasticApmAssertSame("elastic_apm_get_config_option_by_name('max_send_queue_size')", elastic_apm_get_config_option_by_name('max_send_queue_size'), 456.0 * 1024 * 1024); + ////////////////////////////////////////////// /////////////// secret_token diff --git a/agent/php/ElasticApm/Impl/Config/AllOptionsMetadata.php b/agent/php/ElasticApm/Impl/Config/AllOptionsMetadata.php index 50217cc15..1c57792a7 100644 --- a/agent/php/ElasticApm/Impl/Config/AllOptionsMetadata.php +++ b/agent/php/ElasticApm/Impl/Config/AllOptionsMetadata.php @@ -70,6 +70,11 @@ private static function buildPositiveOrZeroIntMetadata(int $default): IntOptionM return new IntOptionMetadata(/* min */ 0, /* max */ null, $default); } + private static function buildPositiveSizeMetadata(int $defaultInBytes, int $defaultUnits): SizeOptionMetadata + { + return new SizeOptionMetadata(/* min */ 1.0, /* max */ null, $defaultUnits, floatval($defaultInBytes)); + } + /** * @return array> Option name to metadata */ @@ -103,6 +108,7 @@ public static function get(): array OptionNames::LOG_LEVEL => new NullableLogLevelOptionMetadata(), OptionNames::LOG_LEVEL_STDERR => new NullableLogLevelOptionMetadata(), OptionNames::LOG_LEVEL_SYSLOG => new NullableLogLevelOptionMetadata(), + OptionNames::MAX_SEND_QUEUE_SIZE => self::buildPositiveSizeMetadata(OptionDefaultValues::MAX_SEND_QUEUE_SIZE, SizeUnits::MEGABYTES), OptionNames::NON_KEYWORD_STRING_MAX_LENGTH => self::buildPositiveOrZeroIntMetadata(/* default */ 10 * 1024), OptionNames::PROFILING_INFERRED_SPANS_ENABLED => new BoolOptionMetadata(/* default */ false), OptionNames::PROFILING_INFERRED_SPANS_MIN_DURATION => self::buildDurationMetadataInMilliseconds(/* default */ 0), diff --git a/agent/php/ElasticApm/Impl/Config/OptionDefaultValues.php b/agent/php/ElasticApm/Impl/Config/OptionDefaultValues.php index b45bb2adb..d2d264e12 100644 --- a/agent/php/ElasticApm/Impl/Config/OptionDefaultValues.php +++ b/agent/php/ElasticApm/Impl/Config/OptionDefaultValues.php @@ -34,6 +34,7 @@ final class OptionDefaultValues { use StaticClassTrait; + public const MAX_SEND_QUEUE_SIZE = 10 * 1024 * 1024; public const SPAN_STACK_TRACE_MIN_DURATION = 5; public const STACK_TRACE_LIMIT = 50; public const TRANSACTION_MAX_SPANS = 500; diff --git a/agent/php/ElasticApm/Impl/Config/OptionNames.php b/agent/php/ElasticApm/Impl/Config/OptionNames.php index addd6c38c..65948ad6d 100644 --- a/agent/php/ElasticApm/Impl/Config/OptionNames.php +++ b/agent/php/ElasticApm/Impl/Config/OptionNames.php @@ -57,6 +57,7 @@ final class OptionNames public const LOG_LEVEL = 'log_level'; public const LOG_LEVEL_SYSLOG = 'log_level_syslog'; public const LOG_LEVEL_STDERR = 'log_level_stderr'; + public const MAX_SEND_QUEUE_SIZE = 'max_send_queue_size'; public const NON_KEYWORD_STRING_MAX_LENGTH = 'non_keyword_string_max_length'; public const PROFILING_INFERRED_SPANS_ENABLED = 'profiling_inferred_spans_enabled'; public const PROFILING_INFERRED_SPANS_MIN_DURATION = 'profiling_inferred_spans_min_duration'; diff --git a/agent/php/ElasticApm/Impl/Config/SizeOptionMetadata.php b/agent/php/ElasticApm/Impl/Config/SizeOptionMetadata.php new file mode 100644 index 000000000..61bc9a8f3 --- /dev/null +++ b/agent/php/ElasticApm/Impl/Config/SizeOptionMetadata.php @@ -0,0 +1,46 @@ + + */ +final class SizeOptionMetadata extends OptionWithDefaultValueMetadata +{ + public function __construct( + ?float $minValidValueInBytes, + ?float $maxValidValueInBytes, + int $defaultUnits, + float $defaultValueInBytes + ) { + parent::__construct( + new SizeOptionParser($minValidValueInBytes, $maxValidValueInBytes, $defaultUnits), + $defaultValueInBytes + ); + } +} diff --git a/agent/php/ElasticApm/Impl/Config/SizeOptionParser.php b/agent/php/ElasticApm/Impl/Config/SizeOptionParser.php new file mode 100644 index 000000000..7c741dcdd --- /dev/null +++ b/agent/php/ElasticApm/Impl/Config/SizeOptionParser.php @@ -0,0 +1,141 @@ + + */ +final class SizeOptionParser extends OptionParser +{ + /** @var ?float */ + private $minValidValueInBytes; + + /** @var ?float */ + private $maxValidValueInBytes; + + /** @var int */ + private $defaultUnits; + + public function __construct( + ?float $minValidValueInBytes, + ?float $maxValidValueInBytes, + int $defaultUnits + ) { + $this->minValidValueInBytes = $minValidValueInBytes; + $this->maxValidValueInBytes = $maxValidValueInBytes; + $this->defaultUnits = $defaultUnits; + } + + /** @inheritDoc */ + public function parse(string $rawValue): float + { + $partWithoutSuffix = ''; + $units = $this->defaultUnits; + self::splitToValueAndUnits($rawValue, /* ref */ $partWithoutSuffix, /* ref */ $units); + + $auxIntOptionParser = new IntOptionParser(null, null); + $parsedValueInBytes = self::convertToBytes($auxIntOptionParser->parse($partWithoutSuffix), $units); + + if ( + (($this->minValidValueInBytes !== null) && ($parsedValueInBytes < $this->minValidValueInBytes)) + || (($this->maxValidValueInBytes !== null) && ($parsedValueInBytes > $this->maxValidValueInBytes)) + ) { + throw new ParseException( + 'Value is not in range between the valid minimum and maximum values.' + . ' Raw option value: `' . $rawValue . "'." + . ' Parsed option value (in bytes): ' . $parsedValueInBytes . '.' + . ' The valid minimum value (in bytes): ' . $this->minValidValueInBytes . '.' + . ' The valid maximum value (in bytes): ' . $this->maxValidValueInBytes . '.' + ); + } + + return $parsedValueInBytes; + } + + public function defaultUnits(): int + { + return $this->defaultUnits; + } + + public function minValidValueInBytes(): ?float + { + return $this->minValidValueInBytes; + } + + public function maxValidValueInBytes(): ?float + { + return $this->maxValidValueInBytes; + } + + private static function splitToValueAndUnits(string $rawValue, string &$partWithoutSuffix, int &$units): void + { + foreach (SizeUnits::$suffixAndIdPairs as $suffixAndIdPair) { + $suffix = $suffixAndIdPair[0]; + if (TextUtil::isSuffixOf($suffix, $rawValue, /* isCaseSensitive */ false)) { + $partWithoutSuffix = trim(substr($rawValue, 0, -strlen($suffix))); + $units = $suffixAndIdPair[1]; + return; + } + } + + $partWithoutSuffix = $rawValue; + } + + public static function convertToBytes(int $srcValue, int $srcValueUnits): float + { + switch ($srcValueUnits) { + case SizeUnits::BYTES: + return floatval($srcValue); + + case SizeUnits::KILOBYTES: + return floatval($srcValue) * 1024; + + case SizeUnits::MEGABYTES: + return floatval($srcValue) * 1024 * 1024; + + case SizeUnits::GIGABYTES: + return floatval($srcValue) * 1024 * 1024 * 1024; + + default: + throw new ParseException( + ExceptionUtil::buildMessage( + 'Not a valid size units ID', + /* context */ + [ + 'srcValueUnits' => $srcValueUnits, + 'srcValue' => $srcValue, + 'valid size units' => SizeUnits::$suffixAndIdPairs, + ] + ) + ); + } + } +} diff --git a/agent/php/ElasticApm/Impl/Config/SizeUnits.php b/agent/php/ElasticApm/Impl/Config/SizeUnits.php new file mode 100644 index 000000000..9f14c7daf --- /dev/null +++ b/agent/php/ElasticApm/Impl/Config/SizeUnits.php @@ -0,0 +1,58 @@ + Array should be in descending order of suffix length + * + * @see \ElasticApmTests\UnitTests\UtilTests\SizeUnitsTest::testSuffixAndIdIsInDescendingOrderOfSuffixLength + */ + public static $suffixAndIdPairs = [ + [self::GIGABYTES_SUFFIX, self::GIGABYTES], + [self::MEGABYTES_SUFFIX, self::MEGABYTES], + [self::KILOBYTES_SUFFIX, self::KILOBYTES], + [self::BYTES_SUFFIX, self::BYTES], + ]; +} diff --git a/agent/php/ElasticApm/Impl/Config/Snapshot.php b/agent/php/ElasticApm/Impl/Config/Snapshot.php index 47f42e685..71c9aba64 100644 --- a/agent/php/ElasticApm/Impl/Config/Snapshot.php +++ b/agent/php/ElasticApm/Impl/Config/Snapshot.php @@ -161,6 +161,9 @@ final class Snapshot implements LoggableInterface /** @var ?int */ private $logLevelSyslog; + /** @var float - In bytes */ + private $maxSendQueueSize; + /** @var int */ private $nonKeywordStringMaxLength; @@ -357,6 +360,11 @@ public function hostname(): ?string return $this->hostname; } + public function maxSendQueueSize(): float + { + return $this->maxSendQueueSize; + } + public function nonKeywordStringMaxLength(): int { return $this->nonKeywordStringMaxLength; diff --git a/docs/reference/configuration-reference.md b/docs/reference/configuration-reference.md index 408cdd821..042fa4fe5 100644 --- a/docs/reference/configuration-reference.md +++ b/docs/reference/configuration-reference.md @@ -258,6 +258,29 @@ The logging level for `stderr` logging sink. See [Logging](/reference/configurat The logging level for `syslog` logging sink. See [Logging](/reference/configuration.md#configure-logging) for details. +## `max_send_queue_size` [config-max-send-queue-size] + +| Environment variable name | Option name in `php.ini` | +| --- | --- | +| `ELASTIC_APM_MAX_SEND_QUEUE_SIZE` | `elastic_apm.max_send_queue_size` | + +| Default | Type | +| --- | --- | +| `10MB` | Size | + +The maximum total size of serialized events that can be queued for background sending to APM Server. + +When adding a new batch of events would make the queue exceed this limit, the agent drops that batch locally and logs an error. + +This option only affects background sending. If the agent sends events synchronously, this limit is not used. + +The value has to be provided in **[size format](/reference/configuration.md#configure-size-format)**. + +This option’s default unit is `MB` (megabytes). + +Values less than `1B` are invalid and result in the default value being used instead. + + ## `profiling_inferred_spans_enabled` [config-profiling-inferred-spans-enabled] ::::{warning} @@ -570,4 +593,3 @@ By default, the agent verifies the SSL certificate if you use an HTTPS connectio With this option, you can group several URL paths together by using wildcard expressions like `/user/*` - this way `/user/Alice` and `/user/Bob` will be mapped to transaction name `/user/*`. See [Wildcard](/reference/configuration.md#configure-wildcard) section for more details on how to use wildcard expressions. - diff --git a/tests/ElasticApmTests/ComponentTests/BackendCommTest.php b/tests/ElasticApmTests/ComponentTests/BackendCommTest.php index 0bd5067eb..fa5635f36 100644 --- a/tests/ElasticApmTests/ComponentTests/BackendCommTest.php +++ b/tests/ElasticApmTests/ComponentTests/BackendCommTest.php @@ -24,8 +24,10 @@ namespace ElasticApmTests\ComponentTests; use Elastic\Apm\ElasticApm; +use Elastic\Apm\Impl\Config\OptionNames; use Elastic\Apm\Impl\Log\Logger; use Elastic\Apm\Impl\Util\RangeUtil; +use ElasticApmTests\ComponentTests\Util\AppCodeHostParams; use ElasticApmTests\ComponentTests\Util\AppCodeRequestParams; use ElasticApmTests\ComponentTests\Util\AppCodeTarget; use ElasticApmTests\ComponentTests\Util\ComponentTestCaseBase; @@ -83,6 +85,34 @@ function (AppCodeRequestParams $appCodeRequestParams) use ($txName): void { } } + public function testSingleOversizedBatchIsDroppedWhenMaxSendQueueSizeIsTooSmall(): void + { + if (self::skipIfMainAppCodeHostIsNotHttp()) { + return; + } + + $testCaseHandle = $this->getTestCaseHandle(); + $appCodeHost = $testCaseHandle->ensureMainAppCodeHost( + function (AppCodeHostParams $appCodeHostParams): void { + $appCodeHostParams->setAgentOption(OptionNames::ASYNC_BACKEND_COMM, 'true'); + $appCodeHostParams->setAgentOption(OptionNames::MAX_SEND_QUEUE_SIZE, '1B'); + } + ); + + $txName = __FUNCTION__; + $appCodeHost->sendRequest( + AppCodeTarget::asRouted([__CLASS__, 'appCodeForTestNumberOfConnections']), + function (AppCodeRequestParams $appCodeRequestParams) use ($txName): void { + $appCodeRequestParams->setAppCodeArgs([self::TRANSACTION_NAME_KEY => $txName]); + } + ); + + // Any serialized batch is larger than 1 byte, so it should be dropped before it reaches the mock APM server. + usleep(500 * 1000); + + self::assertSame([], $testCaseHandle->fetchNewDataFromMockApmServer(/* shouldWait */ false)); + } + private const WAIT_FOR_RECONNECT_COUNT_KEY = 'wait_for_reconnect_count'; /** diff --git a/tests/ElasticApmTests/ComponentTests/ConfigSettingTest.php b/tests/ElasticApmTests/ComponentTests/ConfigSettingTest.php index e28ea3a8c..f0a23421a 100644 --- a/tests/ElasticApmTests/ComponentTests/ConfigSettingTest.php +++ b/tests/ElasticApmTests/ComponentTests/ConfigSettingTest.php @@ -74,6 +74,7 @@ final class ConfigSettingTest extends ComponentTestCaseBase OptionNames::LOG_LEVEL, OptionNames::LOG_LEVEL_STDERR, OptionNames::LOG_LEVEL_SYSLOG, + OptionNames::MAX_SEND_QUEUE_SIZE, OptionNames::PROFILING_INFERRED_SPANS_ENABLED, OptionNames::SERVER_TIMEOUT, OptionNames::SPAN_COMPRESSION_ENABLED, @@ -137,6 +138,16 @@ private static function buildOptionNameToRawToValue(): array 'not valid' => null, ]; + $sizeRawToParsedValues = [ + '100MB' => 100.0 * 1024 * 1024, + "\t 78 mB \n" => 78.0 * 1024 * 1024, + '1MB' => 1.0 * 1024 * 1024, + '10' => 10.0 * 1024 * 1024, + '0' => null, + '-1MB' => null, + 'not valid' => null, + ]; + $intRawToParsedValues = [ "\n\t 123 " => 123, 'not valid' => null, @@ -182,6 +193,7 @@ private static function buildOptionNameToRawToValue(): array OptionNames::LOG_LEVEL => $logLevelRawToParsedValues, OptionNames::LOG_LEVEL_STDERR => $logLevelRawToParsedValues, OptionNames::LOG_LEVEL_SYSLOG => $logLevelRawToParsedValues, + OptionNames::MAX_SEND_QUEUE_SIZE => $sizeRawToParsedValues, OptionNames::NON_KEYWORD_STRING_MAX_LENGTH => $intRawToParsedValues, OptionNames::PROFILING_INFERRED_SPANS_ENABLED => $boolRawToParsedValues(/* valueToExclude: */ true), diff --git a/tests/ElasticApmTests/UnitTests/ConfigTests/SizeOptionTestValuesGenerator.php b/tests/ElasticApmTests/UnitTests/ConfigTests/SizeOptionTestValuesGenerator.php new file mode 100644 index 000000000..ea2624449 --- /dev/null +++ b/tests/ElasticApmTests/UnitTests/ConfigTests/SizeOptionTestValuesGenerator.php @@ -0,0 +1,107 @@ + + */ +final class SizeOptionTestValuesGenerator implements OptionTestValuesGeneratorInterface +{ + /** @var SizeOptionParser */ + private $optionParser; + + public function __construct(SizeOptionParser $optionParser) + { + $this->optionParser = $optionParser; + } + + private function isInValidRange(float $valueInBytes): bool + { + return (($this->optionParser->minValidValueInBytes() === null) || ($valueInBytes >= $this->optionParser->minValidValueInBytes())) + && (($this->optionParser->maxValidValueInBytes() === null) || ($valueInBytes <= $this->optionParser->maxValidValueInBytes())); + } + + /** + * @param int $valueWithoutUnits + */ + private function convertFromDefaultUnitsToBytes(int $valueWithoutUnits): float + { + return SizeOptionParser::convertToBytes($valueWithoutUnits, $this->optionParser->defaultUnits()); + } + + public function validValues(): iterable + { + /** + * @var array + */ + $predefinedValidValues = [ + ['1', $this->convertFromDefaultUnitsToBytes(1)], + ['1B', 1.0], + ['2 KB', 2.0 * 1024], + ['10MB', 10.0 * 1024 * 1024], + ['3 gb', 3.0 * 1024 * 1024 * 1024], + ['123 B', 123.0], + ]; + + foreach ($predefinedValidValues as $rawAndParsedValuesPair) { + if ($this->isInValidRange($rawAndParsedValuesPair[1])) { + yield new OptionTestValidValue($rawAndParsedValuesPair[0], $rawAndParsedValuesPair[1]); + } + } + } + + public function invalidRawValues(): iterable + { + yield from [ + '', + ' ', + '\t', + '\r\n', + 'a', + 'abc', + '123abc', + 'abc123', + '1.0', + '1.5MB', + '1m', + '1MBx', + '3a2MB', + '0', + '0B', + '0 KB', + '-1', + '-1B', + '-1MB', + ]; + + /** @var OptionTestValidValue $validValue */ + foreach ($this->validValues() as $validValue) { + foreach (['a', 'z'] as $invalidSizeUnitsSuffix) { + yield $validValue->rawValue . $invalidSizeUnitsSuffix; + } + } + } +} diff --git a/tests/ElasticApmTests/UnitTests/ConfigTests/VariousOptionsParsingTest.php b/tests/ElasticApmTests/UnitTests/ConfigTests/VariousOptionsParsingTest.php index 981a572e6..971549f8a 100644 --- a/tests/ElasticApmTests/UnitTests/ConfigTests/VariousOptionsParsingTest.php +++ b/tests/ElasticApmTests/UnitTests/ConfigTests/VariousOptionsParsingTest.php @@ -37,6 +37,7 @@ use Elastic\Apm\Impl\Config\OptionParser; use Elastic\Apm\Impl\Config\ParseException; use Elastic\Apm\Impl\Config\Parser; +use Elastic\Apm\Impl\Config\SizeOptionParser; use Elastic\Apm\Impl\Config\StringOptionParser; use Elastic\Apm\Impl\Config\WildcardListOptionParser; use Elastic\Apm\Impl\Log\LoggableToString; @@ -91,6 +92,9 @@ private static function selectTestValuesGenerator(OptionMetadata $optMeta): Opti if ($optionParser instanceof LabelsOptionParser) { return LabelsOptionTestValuesGenerator::singletonInstance(); // @phpstan-ignore-line } + if ($optionParser instanceof SizeOptionParser) { + return new SizeOptionTestValuesGenerator($optionParser); // @phpstan-ignore-line + } throw new RuntimeException('Unknown option metadata type: ' . DbgUtil::getType($optMeta)); } diff --git a/tests/ElasticApmTests/UnitTests/UtilTests/SizeUnitsTest.php b/tests/ElasticApmTests/UnitTests/UtilTests/SizeUnitsTest.php new file mode 100644 index 000000000..7a91e4d6a --- /dev/null +++ b/tests/ElasticApmTests/UnitTests/UtilTests/SizeUnitsTest.php @@ -0,0 +1,44 @@ +