diff --git a/.claude/ci/docker-compose.services.yml b/.claude/ci/docker-compose.services.yml index 49e785f1e5e..557e14743b2 100644 --- a/.claude/ci/docker-compose.services.yml +++ b/.claude/ci/docker-compose.services.yml @@ -35,7 +35,7 @@ services: - SNAPSHOT_REGEX_PLACEHOLDERS=path:/home/circleci/app|/project/dd-trace-php,httpbin:(?<=//)httpbin-integration:8080 request-replayer: - image: datadog/dd-trace-ci:php-request-replayer-2.0 + image: datadog/request-replayer:3.0 httpbin-integration: image: kong/httpbin:0.2.2 diff --git a/.claude/ci/package-native-verification.md b/.claude/ci/package-native-verification.md index 097132f5730..ca523f679a1 100644 --- a/.claude/ci/package-native-verification.md +++ b/.claude/ci/package-native-verification.md @@ -184,7 +184,7 @@ docker network create verify-net 2>/dev/null || true docker rm -f replayer 2>/dev/null || true docker run -d --name replayer --network verify-net \ --network-alias request-replayer \ - datadog/dd-trace-ci:php-request-replayer-2.0 + datadog/request-replayer:3.0 .claude/ci/dockerh --cache verify-debian-83 --overlayfs --root \ debian:bookworm-slim \ diff --git a/.claude/ci/tracer-integration-tests.md b/.claude/ci/tracer-integration-tests.md index 5865fa54591..db1019dae95 100644 --- a/.claude/ci/tracer-integration-tests.md +++ b/.claude/ci/tracer-integration-tests.md @@ -75,7 +75,7 @@ substring. | Service | Image | Alias | Port | Purpose | |---------|-------|-------|------|---------| | test-agent | `ddapm-test-agent:v1.22.1` | `test-agent` | 9126 | Receives traces; validates snapshots | -| request-replayer | `dd-trace-ci:php-request-replayer-*` | `request-replayer` | 80 | HTTP request replay | +| request-replayer | `request-replayer:*` | `request-replayer` | 80 | HTTP request replay | | httpbin | `kong/httpbin:0.2.2` | `httpbin-integration` | 8080 | HTTP echo service | ### Additional services by target substring diff --git a/.claude/ci/tracer-web-tests.md b/.claude/ci/tracer-web-tests.md index b03cc4f4b93..0f7f1b8e6a3 100644 --- a/.claude/ci/tracer-web-tests.md +++ b/.claude/ci/tracer-web-tests.md @@ -62,7 +62,7 @@ All web test jobs use four GitLab service containers: | Service | Image | Alias | Port | Purpose | |---------|-------|-------|------|---------| | test-agent | `ddapm-test-agent:v1.22.1` | `test-agent` | 9126 | Receives traces; validates snapshots | -| request-replayer | `dd-trace-ci:php-request-replayer-*` | `request-replayer` | 80 | Replays HTTP requests for trace forwarding | +| request-replayer | `request-replayer:*` | `request-replayer` | 80 | Replays HTTP requests for trace forwarding | | httpbin | `kong/httpbin:0.2.2` | `httpbin-integration` | 80 | HTTP echo service for curl/guzzle tests | | mysql | `dd-trace-ci:php-mysql-dev-5.6` | `mysql-integration` | 3306 | MySQL for WordPress, Drupal, Magento, etc. | diff --git a/.gitlab/generate-ci-images.php b/.gitlab/generate-ci-images.php index dc3ef385da0..ee9db882d3c 100644 --- a/.gitlab/generate-ci-images.php +++ b/.gitlab/generate-ci-images.php @@ -4,8 +4,8 @@ * Generates the CI image build + publish GitLab child pipeline. * * Source of truth (NO duplication): - * - dockerfiles/ci//docker-compose.yml : service name -> image:TAG - * - dockerfiles/ci/bookworm/.env : $BOOKWORM_NEXT_VERSION etc. + * - each $dirs compose file : service name -> image:TAG + * - dockerfiles/ci/bookworm/.env : $BOOKWORM_NEXT_VERSION etc. * * The compose service name is the `docker buildx bake` target and the build * matrix value; the `image:` tag (with env vars resolved) is the published tag. @@ -39,7 +39,18 @@ function substitute(string $s, array $env): string }, $s); } -// Parse a docker-compose.yml into [service => tag], preserving file order. +// Compose's own filename precedence; the first that exists wins. +function compose_file(string $dir): string +{ + foreach (["compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"] as $name) { + if (is_file("$dir/$name")) { + return "$dir/$name"; + } + } + return "$dir/compose.yaml"; // nonexistent: the caller fails on the empty parse +} + +// Parse a compose file into [service => tag], preserving file order. function parse_compose(string $path, array $env): array { $services = []; @@ -75,14 +86,21 @@ function parse_compose(string $path, array $env): array "Bookworm" => "dockerfiles/ci/bookworm", "CentOS" => "dockerfiles/ci/centos/7", "Alpine" => "dockerfiles/ci/alpine_compile_extension", + "Services" => "dockerfiles/services", +]; + +$registryOverrides = [ + "Services" => "registry.ddbuild.io/ci/dd-trace-php/request-replayer", ]; $osList = []; foreach ($dirs as $os => $dir) { - $services = parse_compose("$root/$dir/docker-compose.yml", parse_env("$root/$dir/.env")); + $services = parse_compose(compose_file("$root/$dir"), parse_env("$root/$dir/.env")); if (!$services) { - fwrite(STDERR, "WARNING: no services parsed for $os ($dir)\n"); - continue; + // Continuing would emit a valid-looking pipeline with this OS's jobs + // silently missing, so fail the generator instead. + fwrite(STDERR, "ERROR: no services parsed for $os ($dir)\n"); + exit(1); } $osList[] = ["name" => $os, "dir" => $dir, "services" => $services]; } @@ -90,9 +108,10 @@ function parse_compose(string $path, array $env): array // Windows is single-arch (no multi-arch manifest) and uses a different build // runner/script, so it is emitted separately from the Linux loop below. It has // no .env, so tags resolve with an empty env map. -$winServices = parse_compose("$root/dockerfiles/ci/windows/docker-compose.yml", []); +$winServices = parse_compose(compose_file("$root/dockerfiles/ci/windows"), []); if (!$winServices) { - fwrite(STDERR, "WARNING: no services parsed for Windows\n"); + fwrite(STDERR, "ERROR: no services parsed for Windows\n"); + exit(1); } ?> # CI image build + publish child pipeline, generated by @@ -302,13 +321,17 @@ function parse_compose(string $path, array $env): array $os, 'dir' => $dir, 'services' => $services]): ?> build: extends: .linux_image_build tags: ["arch:amd64"] + + variables: + CI_REGISTRY_IMAGE: "" + parallel: matrix: $tag): ?> @@ -334,6 +357,11 @@ function parse_compose(string $path, array $env): array publish: extends: .image_publish + + variables: + CI_REGISTRY_IMAGE: "" + IMG_DESTINATIONS: ":${TAG}" + parallel: matrix: - TAG: diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 86678ba917d..fc3dc392848 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -227,11 +227,12 @@ function windows_git_setup_with_packages() { KUBERNETES_SERVICE_MEMORY_LIMIT: 512Mi request-replayer: - name: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-request-replayer-2.0 + name: registry.ddbuild.io/ci/dd-trace-php/request-replayer:3.0 alias: request-replayer command: ["php", "-S", ":80", "index.php"] variables: DD_REQUEST_DUMPER_FILE: dump.json + PHP_CLI_SERVER_WORKERS: "16" KUBERNETES_SERVICE_CPU_REQUEST: 2 KUBERNETES_SERVICE_CPU_LIMIT: 2 KUBERNETES_SERVICE_MEMORY_REQUEST: 1Gi diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index 31852e4afaf..0d50422b08f 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -132,7 +132,7 @@ function before_script_steps($with_docker_auth = false) { # Start the container network and services docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net docker run --network net -d --name httpbin-integration registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:httpbin-windows - docker run --network net -d --name request-replayer registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-request-replayer-2.0-windows + docker run --network net -d --name request-replayer registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:php-request-replayer-3.0_windows docker run -v ${pwd}:C:\Users\ContainerAdministrator\app --network net -d --name ${CONTAINER_NAME} ${IMAGE} ping -t localhost # Enable NTFS long path support so cargo's libgit2-based git checkouts of diff --git a/docker-compose.yml b/docker-compose.yml index 77f3f3aeecd..1a650034967 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -228,7 +228,7 @@ services: - DD_API_KEY=invalid_key_but_its_ok request-replayer: - image: datadog/dd-trace-ci:php-request-replayer-2.0 + image: datadog/request-replayer:3.0 ports: - "8766:80" @@ -295,7 +295,7 @@ services: windows-request-replayer: - image: datadog/dd-trace-ci:php-request-replayer-2.0-windows + image: datadog/dd-trace-ci:php-request-replayer-3.0_windows ports: - "8766:80" networks: diff --git a/dockerfiles/ci/windows/docker-compose.yml b/dockerfiles/ci/windows/docker-compose.yml index 880f3f8033f..dd0620ed9d8 100644 --- a/dockerfiles/ci/windows/docker-compose.yml +++ b/dockerfiles/ci/windows/docker-compose.yml @@ -171,3 +171,11 @@ services: vsVersion: "vc15" phpTarGzUrl: https://www.php.net/distributions/php-7.2.34.tar.gz phpSha256Hash: 8b2777c741e83f188d3ca6d8e98ece7264acafee86787298fae57e05d0dddc78 + + request-replayer: + image: ${CI_REGISTRY_IMAGE:-datadog/dd-trace-ci}:php-request-replayer-3.0_windows + build: + platforms: + - windows/amd64 + context: ../../services/request-replayer + dockerfile: windows.Dockerfile diff --git a/dockerfiles/services/.env b/dockerfiles/services/.env new file mode 100644 index 00000000000..e950c64b4cf --- /dev/null +++ b/dockerfiles/services/.env @@ -0,0 +1,2 @@ +REQUEST_REPLAYER_CURRENT_VERSION=2.0 +REQUEST_REPLAYER_NEXT_VERSION=3.0 diff --git a/dockerfiles/services/Makefile b/dockerfiles/services/Makefile index 281af0a5461..846a2e9d354 100644 --- a/dockerfiles/services/Makefile +++ b/dockerfiles/services/Makefile @@ -11,14 +11,6 @@ redis_publish: redis_build mysql_publish: redis_build docker buildx build --platform=linux/amd64 -t $(MYSQL_IMAGE) mysql --push -# It requires buildx to be able to build cross-architecture images -request-replayer_linux_push: - docker buildx build --platform=linux/arm64,linux/amd64 -t datadog/dd-trace-ci:php-request-replayer-2.0 ./request-replayer -f request-replayer/linux.Dockerfile --push - -request-replayer_windows_push: - docker build -t datadog/dd-trace-ci:php-request-replayer-2.0-windows ./request-replayer -f request-replayer/windows.Dockerfile - docker push datadog/dd-trace-ci:php-request-replayer-2.0-windows - httpbin_windows_push: docker build -t datadog/dd-trace-ci:httpbin-windows ./windows-httpbin -f windows-httpbin/Dockerfile docker push datadog/dd-trace-ci:httpbin-windows diff --git a/dockerfiles/services/compose.yaml b/dockerfiles/services/compose.yaml new file mode 100644 index 00000000000..4ade6072e8d --- /dev/null +++ b/dockerfiles/services/compose.yaml @@ -0,0 +1,11 @@ +services: + + request-replayer: + image: ${CI_REGISTRY_IMAGE:-datadog/request-replayer}:$REQUEST_REPLAYER_NEXT_VERSION + build: + context: request-replayer + dockerfile: linux.Dockerfile + x-bake: + platforms: + - linux/arm64 + - linux/amd64 diff --git a/dockerfiles/services/request-replayer/src/index.php b/dockerfiles/services/request-replayer/src/index.php index ff7de4a3adc..b3374ce6bbe 100644 --- a/dockerfiles/services/request-replayer/src/index.php +++ b/dockerfiles/services/request-replayer/src/index.php @@ -3,6 +3,7 @@ error_reporting(\E_ALL); include __DIR__ . '/vendor/autoload.php'; +include __DIR__ . '/state-lock.php'; use MessagePack\BufferUnpacker; use MessagePack\UnpackOptions; @@ -51,9 +52,22 @@ function decodeDogStatsDMetrics($metrics) $temp_location = sys_get_temp_dir(); -$metricsServerPid = "$temp_location/metrics-server.pid"; -if (!file_exists($metricsServerPid)) { - shell_exec("nohup bash -c 'php metricsserver.php & pid=$!; echo \$pid > $metricsServerPid; wait \$pid; rm $metricsServerPid' > /dev/null 2>&1 &"); +// One UDP metrics server is shared by every PHP_CLI_SERVER_WORKERS worker. +// metricsserver.php holds this lock for its lifetime, so taking it means +// nothing is on udp/80: drop it (the child needs the same lock) and spawn. +// A second worker in that window is harmless -- the loser's own LOCK_NB makes +// it exit rather than bind twice. The stamp caps attempts at one a second. +$metricsServerLock = "$temp_location/metrics-server.lock"; +$metricsServerStamp = "$temp_location/metrics-server.spawned-at"; +$lock = @fopen($metricsServerLock, 'c'); +if ($lock !== false) { + $stopped = flock($lock, LOCK_EX | LOCK_NB); + fclose($lock); + clearstatcache(true, $metricsServerStamp); + if ($stopped && @filemtime($metricsServerStamp) < time()) { + @touch($metricsServerStamp); + shell_exec('nohup php ' . escapeshellarg(__DIR__ . '/metricsserver.php') . ' > /dev/null 2>&1 &'); + } } $token = $_SERVER["HTTP_X_DATADOG_TEST_SESSION_TOKEN"] ?? ""; @@ -81,6 +95,7 @@ function decodeDogStatsDMetrics($metrics) define('REQUEST_METRICS_LOG_FILE', getenv('REQUEST_METRICS_LOG_FILE') ?: ("$temp_location/metrics-log.txt")); define('REQUEST_STATS_FILE', getenv('REQUEST_STATS_FILE') ?: ("$temp_location/stats.json")); define('REQUEST_AGENT_INFO_FILE', getenv('REQUEST_AGENT_INFO_FILE') ?: ("$temp_location/agent-info.txt")); +define('REQUEST_STATE_LOCK_FILE', "$temp_location/.state.lock"); function logRequest($message, $data = '') { @@ -98,144 +113,159 @@ function logRequest($message, $data = '') return true; } logRequest("Triggered error $number $message in $errfile on line $errline: " . (new \Exception)->getTraceAsString()); - trigger_error($message, $number); + // Re-raising via trigger_error() is a ValueError on PHP 8 for any level + // outside E_USER_*, which turns a warning into an HTML fatal served as + // HTTP 200 -- callers then json_decode() that. Let PHP report it instead. + return false; }); -$rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; - switch ($uri) { case '/replay': - if (!file_exists(REQUEST_LATEST_DUMP_FILE)) { - logRequest('Cannot replay last request; request log does not exist'); - break; - } - $request = file_get_contents(REQUEST_LATEST_DUMP_FILE); - echo $request; - unlink(REQUEST_LATEST_DUMP_FILE); - unlink(REQUEST_LOG_FILE); - logRequest('Returned last request and deleted request log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_LATEST_DUMP_FILE)) { + logRequest('Cannot replay last request; request log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_LATEST_DUMP_FILE); + unlink(REQUEST_LATEST_DUMP_FILE); + unlink(REQUEST_LOG_FILE); + logRequest('Returned last request and deleted request log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-metrics': - if (!file_exists(REQUEST_METRICS_FILE)) { - logRequest('Cannot replay last request; metrics log does not exist'); - break; - } - $request = file_get_contents(REQUEST_METRICS_FILE); - echo $request; - unlink(REQUEST_METRICS_FILE); - unlink(REQUEST_METRICS_LOG_FILE); - logRequest('Returned last metrics and deleted metrics log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_METRICS_FILE)) { + logRequest('Cannot replay last request; metrics log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_METRICS_FILE); + unlink(REQUEST_METRICS_FILE); + unlink(REQUEST_METRICS_LOG_FILE); + logRequest('Returned last metrics and deleted metrics log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-stats': - if (!file_exists(REQUEST_STATS_FILE)) { - logRequest('Cannot replay stats; stats log does not exist'); - break; - } - $request = file_get_contents(REQUEST_STATS_FILE); - echo $request; - unlink(REQUEST_STATS_FILE); - logRequest('Returned stats and deleted stats log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_STATS_FILE)) { + logRequest('Cannot replay stats; stats log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_STATS_FILE); + unlink(REQUEST_STATS_FILE); + logRequest('Returned stats and deleted stats log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-rc-requests': - if (!file_exists(REQUEST_RC_REQUESTS_FILE)) { - logRequest('Cannot replay RC requests; RC requests log does not exist'); - break; - } - $request = file_get_contents(REQUEST_RC_REQUESTS_FILE); - echo $request; - unlink(REQUEST_RC_REQUESTS_FILE); - logRequest('Returned RC requests and deleted RC requests log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_RC_REQUESTS_FILE)) { + logRequest('Cannot replay RC requests; RC requests log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_RC_REQUESTS_FILE); + unlink(REQUEST_RC_REQUESTS_FILE); + logRequest('Returned RC requests and deleted RC requests log', $request); + return $request; + }); + echo $request ?? ''; break; case '/clear-dumped-data': - if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_METRICS_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) { - logRequest('Cannot delete request log; request log does not exist'); - break; - } - if (file_exists(REQUEST_RC_CONFIGS_FILE)) { - unlink(REQUEST_RC_CONFIGS_FILE); - } - if (file_exists(REQUEST_LATEST_DUMP_FILE)) { - unlink(REQUEST_LATEST_DUMP_FILE); - unlink(REQUEST_LOG_FILE); - } - if (file_exists(REQUEST_METRICS_FILE)) { - unlink(REQUEST_METRICS_FILE); - unlink(REQUEST_METRICS_LOG_FILE); - } - if (file_exists(REQUEST_STATS_FILE)) { - unlink(REQUEST_STATS_FILE); - } - if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) { - unlink(REQUEST_NEXT_RESPONSE_FILE); - } - if (file_exists(REQUEST_AGENT_INFO_FILE)) { - unlink(REQUEST_AGENT_INFO_FILE); - } - if (file_exists(REQUEST_RC_REQUESTS_FILE)) { - unlink(REQUEST_RC_REQUESTS_FILE); - } - logRequest('Deleted request log'); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_METRICS_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) { + logRequest('Cannot delete request log; request log does not exist'); + return; + } + foreach ([ + REQUEST_RC_CONFIGS_FILE, + REQUEST_LATEST_DUMP_FILE, + REQUEST_LOG_FILE, + REQUEST_METRICS_FILE, + REQUEST_METRICS_LOG_FILE, + REQUEST_STATS_FILE, + REQUEST_NEXT_RESPONSE_FILE, + REQUEST_AGENT_INFO_FILE, + REQUEST_RC_REQUESTS_FILE, + ] as $file) { + if (file_exists($file)) { + unlink($file); + } + } + logRequest('Deleted request log'); + }); break; case '/next-response': $raw = file_get_contents('php://input'); - file_put_contents(REQUEST_NEXT_RESPONSE_FILE, $raw); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + file_put_contents(REQUEST_NEXT_RESPONSE_FILE, $raw); + }); break; case '/add-rc-config-file': - $rc_configs[$_GET["path"]] = ["service" => $_GET["service"], "data" => file_get_contents('php://input')]; - file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + $raw = file_get_contents('php://input'); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + $rc_configs[$_GET["path"]] = ["service" => $_GET["service"], "data" => $raw]; + file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + }); break; case '/del-rc-config-file': - unset($rc_configs[$_GET["path"]]); - file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + unset($rc_configs[$_GET["path"]]); + file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + }); break; case '/v0.7/config': $request = file_get_contents('php://input'); logRequest("Requested remote config", $request); - - if (file_exists(REQUEST_RC_REQUESTS_FILE)) { - $tracesStack = json_decode(file_get_contents(REQUEST_RC_REQUESTS_FILE), true); - } else { - $tracesStack = []; - } - $tracesStack[] = ['uri' => $_SERVER['REQUEST_URI'], 'headers' => getallheaders(), 'body' => $request]; - file_put_contents(REQUEST_RC_REQUESTS_FILE, json_encode($tracesStack)); - - $request = json_decode($request, true); - $recentUpdate = @filemtime(REQUEST_RC_CONFIGS_FILE) > time() - 2; - $response = [ - "roots" => [], - "targets" => [ - "signatures" => [], - "signed" => [ - "_type" => "targets", - "custom" => [ - "opaque_backend_state" => "foobarbaz", - "agent_refresh_interval" => ($recentUpdate ? 10 : 10000) * 1000000, // in ns + $response = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($request) { + $tracesStack = file_exists(REQUEST_RC_REQUESTS_FILE) ? json_decode(file_get_contents(REQUEST_RC_REQUESTS_FILE), true) : []; + $tracesStack[] = ['uri' => $_SERVER['REQUEST_URI'], 'headers' => getallheaders(), 'body' => $request]; + file_put_contents(REQUEST_RC_REQUESTS_FILE, json_encode($tracesStack)); + + $decodedRequest = json_decode($request, true); + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + $recentUpdate = @filemtime(REQUEST_RC_CONFIGS_FILE) > time() - 2; + $response = [ + "roots" => [], + "targets" => [ + "signatures" => [], + "signed" => [ + "_type" => "targets", + "custom" => [ + "opaque_backend_state" => "foobarbaz", + "agent_refresh_interval" => ($recentUpdate ? 10 : 10000) * 1000000, // in ns + ], + "expires" => "9999-12-31T23:59:59Z", + "spec_version" => "1.0.0", + "targets" => new \StdClass, + "version" => 1, ], - "expires" => "9999-12-31T23:59:59Z", - "spec_version" => "1.0.0", - "targets" => new \StdClass, - "version" => 1, ], - ], - "target_files" => [], - "client_configs" => [], - ]; - foreach ($rc_configs as $path => $config) { - if ($config["service"] == $request["client"]["client_tracer"]["service"]) { - $content = $config["data"]; - $response["targets"]["signed"]["targets"]->$path = [ - "custom" => ["v" => strlen($path)], - "hashes" => ["sha256" => hash("sha256", $content)], - "length" => strlen($content), - ]; - $response["target_files"][] = [ - "path" => $path, - "raw" => base64_encode($content), - ]; - $response["client_configs"][] = $path; + "target_files" => [], + "client_configs" => [], + ]; + foreach ($rc_configs as $path => $config) { + if ($config["service"] == $decodedRequest["client"]["client_tracer"]["service"]) { + $content = $config["data"]; + $response["targets"]["signed"]["targets"]->$path = [ + "custom" => ["v" => strlen($path)], + "hashes" => ["sha256" => hash("sha256", $content)], + "length" => strlen($content), + ]; + $response["target_files"][] = [ + "path" => $path, + "raw" => base64_encode($content), + ]; + $response["client_configs"][] = $path; + } } - } + return $response; + }); logRequest("Returned remote config", json_encode($response, JSON_UNESCAPED_SLASHES)); $response["targets"] = base64_encode(json_encode($response["targets"], JSON_UNESCAPED_SLASHES)); echo json_encode($response, JSON_UNESCAPED_SLASHES); @@ -243,24 +273,25 @@ function logRequest($message, $data = '') case "/metrics": $_SERVER['REQUEST_URI'] = $uri; logRequest('Logged new metrics', json_encode($decodedMetrics)); - foreach ($decodedMetrics as $metric) { - file_put_contents(REQUEST_METRICS_LOG_FILE, json_encode($metric) . "\n", FILE_APPEND); - - if (file_exists(REQUEST_METRICS_FILE)) { - $allMetrics = json_decode(file_get_contents(REQUEST_METRICS_FILE), true); - } else { - $allMetrics = []; + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($decodedMetrics) { + $allMetrics = file_exists(REQUEST_METRICS_FILE) ? json_decode(file_get_contents(REQUEST_METRICS_FILE), true) : []; + foreach ($decodedMetrics as $metric) { + file_put_contents(REQUEST_METRICS_LOG_FILE, json_encode($metric) . "\n", FILE_APPEND); + $allMetrics[] = $metric; } - $allMetrics[] = $metric; file_put_contents(REQUEST_METRICS_FILE, json_encode($allMetrics)); - } + }); break; case '/set-agent-info': $raw = file_get_contents('php://input'); - file_put_contents(REQUEST_AGENT_INFO_FILE, $raw); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + file_put_contents(REQUEST_AGENT_INFO_FILE, $raw); + }); break; case '/info': - $file = @file_get_contents(REQUEST_AGENT_INFO_FILE) ?: "{}"; + $file = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + return @file_get_contents(REQUEST_AGENT_INFO_FILE) ?: "{}"; + }); logRequest('Requested /info endpoint, returning ' . $file); header("datadog-agent-state: " . sha1($file)); echo $file; @@ -288,13 +319,11 @@ function logRequest($message, $data = '') 'headers' => getallheaders(), 'body' => $body, ]; - if (file_exists(REQUEST_STATS_FILE)) { - $statsStack = json_decode(file_get_contents(REQUEST_STATS_FILE), true); - } else { - $statsStack = []; - } - $statsStack[] = $newStatsRequest; - file_put_contents(REQUEST_STATS_FILE, json_encode($statsStack)); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($newStatsRequest) { + $statsStack = file_exists(REQUEST_STATS_FILE) ? json_decode(file_get_contents(REQUEST_STATS_FILE), true) : []; + $statsStack[] = $newStatsRequest; + file_put_contents(REQUEST_STATS_FILE, json_encode($statsStack)); + }); logRequest('Logged stats request', $body); break; default: @@ -366,22 +395,21 @@ function logRequest($message, $data = '') $newIncomingRequest["body"] = $body; } - if (file_exists(REQUEST_LATEST_DUMP_FILE)) { - $tracesStack = json_decode(file_get_contents(REQUEST_LATEST_DUMP_FILE), true); - } else { - $tracesStack = []; - } - - $tracesStack[] = $newIncomingRequest; $newIncomingRequestJson = json_encode($newIncomingRequest); - - file_put_contents(REQUEST_LATEST_DUMP_FILE, json_encode($tracesStack)); - file_put_contents(REQUEST_LOG_FILE, $newIncomingRequestJson . "\n", FILE_APPEND); - logRequest('Logged new request', $newIncomingRequestJson); - - if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) { - readfile(REQUEST_NEXT_RESPONSE_FILE); + $nextResponse = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($newIncomingRequest, $newIncomingRequestJson) { + $tracesStack = file_exists(REQUEST_LATEST_DUMP_FILE) ? json_decode(file_get_contents(REQUEST_LATEST_DUMP_FILE), true) : []; + $tracesStack[] = $newIncomingRequest; + file_put_contents(REQUEST_LATEST_DUMP_FILE, json_encode($tracesStack)); + file_put_contents(REQUEST_LOG_FILE, $newIncomingRequestJson . "\n", FILE_APPEND); + logRequest('Logged new request', $newIncomingRequestJson); + + if (!file_exists(REQUEST_NEXT_RESPONSE_FILE)) { + return null; + } + $response = file_get_contents(REQUEST_NEXT_RESPONSE_FILE); unlink(REQUEST_NEXT_RESPONSE_FILE); - } + return $response; + }); + echo $nextResponse ?? ''; break; } diff --git a/dockerfiles/services/request-replayer/src/metricsserver.php b/dockerfiles/services/request-replayer/src/metricsserver.php index dda9bde3bca..ab9e912e070 100644 --- a/dockerfiles/services/request-replayer/src/metricsserver.php +++ b/dockerfiles/services/request-replayer/src/metricsserver.php @@ -1,6 +1,19 @@